DIME

Institution DDN
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.7
Client Kernel Version 3.1

DATA SERVER

Storage Type NVMe-SSD
Volatile Memory 128GB
Storage Interface NVMe
Network IB-EDR
Software Version IME 1.4.1
OS Version Centos 7.7

INFORMATION

Client Nodes 10
Client Total Procs 110
Metadata Nodes 2
Metadata Storage Devices 17
Data Nodes 6
Data Storage Devices 18

METADATA

Easy Write 162.05 kIOP/s
Easy Stat 837.20 kIOP/s
Easy Delete 201.21 kIOP/s
Hard Write 37.49 kIOP/s
Hard Read 326.31 kIOP/s
Hard Stat 884.26 kIOP/s
Hard Delete 23.47 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.
#
# The only parts of the script that may need to be modified are:
#  - setup_paths() to configure the binary locations and MPI parameters
#  - setup_directories() to create/tune the IOR/mdtest output directories
#
# Please visit https://vi4io.org/io500-info-creator/ to help generate the
# "system-information.txt" file, by pasting the output of the info-creator.
# This file contains details of your system hardware for your submission.

# Set the paths to the binaries and how to launch MPI jobs.
# If you ran ./prepare.sh successfully, then binaries are in ./bin/
function setup_paths {
  CNODES="dcn-01,dcn-02,dcn-04,dcn-06,dcn-07,dcn-09,dcn-10,dcn-11,dcn-13,dcn-15"
  BINDING="-genv MV2_CPU_MAPPING 0,20:1,21:2,22:3,23:4,24:5,25:6,26:7,27:8,28:9,29:10,30:11,31:12,32:13,33"

  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mpirun="/opt/ddn/mvapich/bin/mpirun"
  io500_mpiargs=" -np 110 -ppn 11 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_NUM_HCAS 1 -genv MV2_USE_RDMA_CM 0 $BINDING -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_HANDSHAKE_ENABLED 0 --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs -genv CCI_CTP_VERBS_SPINNING_DURATION 1000 -genv IM_CLIENT_DEBUG_LEVEL 5 -genv IM_CLIENT_ENABLE_SGL 1 -genv IM_CLIENT_BFS_PATH /bfs/lustre_1/jyvet/BFS -genv LD_PRELOAD /bfs/lustre/jyvet/posix_2_ime/libposix2ime.so "
}

# Set directories where benchmark files are created and where the results go.
# If you want to set up stripe tuning on your output directories or anything
# similar, then this is the right place to do it.
function setup_directories {
  local workdir
  local resultdir
  local bfspath

  mkdir -p $io500_workdir $io500_resultdir

  eval $(grep "^BFSDIR" /etc/ddn/ime/ime-fuse.conf)
  eval $(grep "^MNTDIR" /etc/ddn/ime/ime-fuse.conf)
  bfspath=$(echo "$BFSDIR${io500_workdir#"$MNTDIR"}")

  sudo lfs setstripe -L mdt -E 1M $bfspath
  sudo lfs setdirstripe -D -c -1 $bfspath
  sudo lfs setstripe -c 1 $bfspath

  #mkdir $io500_workdir/mdtest-hard
  #sudo lfs setdirstripe -c -1  /bfs/lustre_1/jyvet/BFS/datafiles/${ts}/mdtest-easy
  #sudo lfs setdirstripe -c -1 -D /bfs/lustre_1/jyvet/BFS/datafiles/${ts}/mdtest-hard
  #sudo chown jyvet:jyvet /bfs/lustre_1/jyvet/BFS/datafiles/${ts}/mdtest-easy

  # Example commands to create output directories for Lustre.  Creating
  # top-level directories is allowed, but not the whole directory tree.
  #if (( $(lfs df $io500_workdir | grep -c MDT) > 1 )); then
  #  lfs setdirstripe -D -c -1 $io500_workdir
  #fi
  #lfs setstripe -c 1 $io500_workdir
  #mkdir $io500_workdir/ior-easy $io500_workdir/ior-hard
  #mkdir $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard
  #local osts=$(lfs df $io500_workdir | grep -c OST)
  # Try overstriping for ior-hard to improve scaling, or use wide striping
  #lfs setstripe -C $((osts * 4)) $io500_workdir/ior-hard ||
  #  lfs setstripe -c -1 $io500_workdir/ior-hard
  # Try to use DoM if available, otherwise use default for small files
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-easy || true #DoM?
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-hard || true #DoM?
}

# *****  YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE  *****
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

function run_benchmarks {
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_resultdir)
  local fname=$(basename ${io500_resultdir})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_resultdir
  tar czf $tarball -C $sourcedir $fname
  echo "Created result tarball $tarball"
}

function main {
  # These commands extract the 'datadir' and 'resultdir' from .ini file
  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # working directory where the test files will be created
  export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts

  setup_directories
  setup_paths
  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a 'system-information.txt' description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_resultdir
  fi

  create_tarball
}

main
ior-easy-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Oct  8 09:52:28 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.08-09.30.31/ior-easy.stonewall -t 1M -b 9920000m -r -R -a POSIX
Machine             : Linux dcn-01
TestID              : 0
StartTime           : Thu Oct  8 09:52:28 2020
Path                : /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-easy/ior_file_easy
FS                  : 826.5 GiB   Used FS: 0.0%   Inodes: 66.5 Mi   Used Inodes: 90.2%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 110
clients per node    : 11
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 1040.65 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 1144206131200000
WARNING: Stat() of aggregate file size      = 1134116673159168
WARNING: Using actual aggregate bytes moved = 34365746708480
read      101419     101431     0.000033    10158080000 1024.00    0.000843   323.11     0.035778   323.15     0   
ior-easy-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Oct  8 09:30:39 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.08-09.30.31/ior-easy.stonewall -t 1M -b 9920000m -w -D 300 -O stoneWallingWearOut=1 -a POSIX
Machine             : Linux dcn-01
TestID              : 0
StartTime           : Thu Oct  8 09:30:39 2020
Path                : /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-easy/ior_file_easy
FS                  : 826.5 GiB   Used FS: 0.0%   Inodes: 6.6 Mi   Used Inodes: 0.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 110
clients per node    : 11
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 1040.65 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 283409 max: 297943 -- min data: 276.8 GiB mean data: 286.4 GiB time: 300.0s
WARNING: Expected aggregate file size       = 1144206131200000
WARNING: Stat() of aggregate file size      = 1134116673159168
WARNING: Using actual aggregate bytes moved = 34365746708480
WARNING: Maybe caused by deadlineForStonewalling
write     106960     106968     0.000031    10158080000 1024.00    0.023269   306.39     0.000342   306.41     0   
ior-hard-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Oct  8 09:58:53 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.08-09.30.31/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a POSIX
Machine             : Linux dcn-01
TestID              : 0
StartTime           : Thu Oct  8 09:58:53 2020
Path                : /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-hard/file
FS                  : 826.5 GiB   Used FS: 0.0%   Inodes: 66.5 Mi   Used Inodes: 90.2%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 110
clients per node    : 11
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 47.03 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 6658880 max: 6658880 -- min data: 291.5 GiB mean data: 291.5 GiB time: 328.0s
WARNING: Expected aggregate file size       = 51708800000000
WARNING: Stat() of aggregate file size      = 34432269414400
WARNING: Using actual aggregate bytes moved = 34432269414400
read      100027     2231292    328.27      45.91      45.91      0.000839   328.27     0.007610   328.28     0   
ior-hard-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Oct  8 09:41:21 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.08-09.30.31/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a POSIX
Machine             : Linux dcn-01
TestID              : 0
StartTime           : Thu Oct  8 09:41:21 2020
Path                : /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-hard/file
FS                  : 826.5 GiB   Used FS: 0.0%   Inodes: 55.3 Mi   Used Inodes: 88.2%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 110
clients per node    : 11
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 47.03 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 6536612 max: 6658880 -- min data: 286.2 GiB mean data: 288.9 GiB time: 300.0s
WARNING: Expected aggregate file size       = 51708800000000
WARNING: Stat() of aggregate file size      = 34432269414400
WARNING: Using actual aggregate bytes moved = 34432269414400
WARNING: Maybe caused by deadlineForStonewalling
write     107994     2409034    303.24      45.91      45.91      0.010132   304.05     0.000292   304.06     0   
mdtest-easy-delete
-- started at 10/08/2020 10:04:34 --

mdtest-3.4.0+dev was launched with 110 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-easy' '-x' './results/2020.10.08-09.30.31/mdtest-easy.stonewall' '-r' '-a' 'POSIX'
Path: /tmp/fuseMNT/datafiles/2020.10.08-09.30.31
FS: 826.5 GiB   Used FS: 0.0%   Inodes: 66.5 Mi   Used Inodes: 90.2%

Nodemap: 11111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 11 for each phase.
110 tasks, 110000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     201209.816     201209.553     201209.725          0.083
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.561          1.561          1.561          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        253.972        253.972        253.972          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.641          0.641          0.641          0.000
-- finished at 10/08/2020 10:08:49 --

mdtest-easy-stat
-- started at 10/08/2020 09:57:52 --

mdtest-3.4.0+dev was launched with 110 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-easy' '-x' './results/2020.10.08-09.30.31/mdtest-easy.stonewall' '-T' '-a' 'POSIX'
Path: /tmp/fuseMNT/datafiles/2020.10.08-09.30.31
FS: 826.5 GiB   Used FS: 0.0%   Inodes: 66.5 Mi   Used Inodes: 90.2%

Nodemap: 11111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 11 for each phase.
110 tasks, 110000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     837196.640     837196.208     837196.419          0.087
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         61.039         61.039         61.039          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/08/2020 09:58:53 --

mdtest-easy-write
-- started at 10/08/2020 09:35:46 --

mdtest-3.4.0+dev was launched with 110 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-easy' '-x' './results/2020.10.08-09.30.31/mdtest-easy.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /tmp/fuseMNT/datafiles/2020.10.08-09.30.31
FS: 826.5 GiB   Used FS: 0.0%   Inodes: 6.6 Mi   Used Inodes: 0.1%

Nodemap: 11111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 11 for each phase.
110 tasks, 110000000 files
Continue stonewall hit min: 449052 max: 464560 avg: 458667.7 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     162046.218     162046.078     162046.168          0.040
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     168143.195             NA
   Tree creation             :          3.480          3.480          3.480          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        315.352        315.352        315.352          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.062             NA
   Tree creation             :          0.287          0.287          0.287          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/08/2020 09:41:21 --

mdtest-hard-delete
-- started at 10/08/2020 10:09:25 --

mdtest-3.4.0+dev was launched with 110 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-hard' '-x' './results/2020.10.08-09.30.31/mdtest-hard.stonewall' '-r' '-a' 'POSIX'
Path: /tmp/fuseMNT/datafiles/2020.10.08-09.30.31
FS: 826.5 GiB   Used FS: 0.0%   Inodes: 17.8 Mi   Used Inodes: 63.2%

Nodemap: 11111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 11 for each phase.
110 tasks, 110000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      23472.157      23472.136      23472.142          0.006
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          5.390          5.390          5.390          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        501.657        501.656        501.656          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.186          0.186          0.186          0.000
-- finished at 10/08/2020 10:17:47 --

mdtest-hard-read
-- started at 10/08/2020 10:08:49 --

mdtest-3.4.0+dev was launched with 110 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-hard' '-x' './results/2020.10.08-09.30.31/mdtest-hard.stonewall' '-E' '-X' '-a' 'POSIX'
Path: /tmp/fuseMNT/datafiles/2020.10.08-09.30.31
FS: 826.5 GiB   Used FS: 0.0%   Inodes: 17.9 Mi   Used Inodes: 63.4%

Nodemap: 11111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 11 for each phase.
110 tasks, 110000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     326307.761     326305.616     326306.670          0.676
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :         36.086         36.085         36.086          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/08/2020 10:09:25 --

mdtest-hard-stat
-- started at 10/08/2020 10:04:21 --

mdtest-3.4.0+dev was launched with 110 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-hard' '-x' './results/2020.10.08-09.30.31/mdtest-hard.stonewall' '-T' '-a' 'POSIX'
Path: /tmp/fuseMNT/datafiles/2020.10.08-09.30.31
FS: 826.5 GiB   Used FS: 0.0%   Inodes: 66.5 Mi   Used Inodes: 90.2%

Nodemap: 11111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 11 for each phase.
110 tasks, 110000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     884257.773     884256.475     884257.140          0.284
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         13.316         13.316         13.316          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/08/2020 10:04:34 --

mdtest-hard-write
-- started at 10/08/2020 09:46:25 --

mdtest-3.4.0+dev was launched with 110 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-hard' '-x' './results/2020.10.08-09.30.31/mdtest-hard.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /tmp/fuseMNT/datafiles/2020.10.08-09.30.31
FS: 826.5 GiB   Used FS: 0.0%   Inodes: 55.3 Mi   Used Inodes: 88.2%

Nodemap: 11111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 11 for each phase.
110 tasks, 110000000 files
Continue stonewall hit min: 104779 max: 107045 avg: 105452.3 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      37487.878      37487.870      37487.875          0.002
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      38664.838             NA
   Tree creation             :          2.663          2.663          2.663          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        314.100        314.100        314.100          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.008             NA
   Tree creation             :          0.376          0.376          0.376          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/08/2020 09:52:05 --

result
version         = io500-sc20_v2
config-hash     = BB2203FE
result-dir      = ./results/2020.10.08-09.30.31
; START 2020-10-08 09:30:39


[ior-easy-write]
t_start         = 2020-10-08 09:30:39
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.08-09.30.31/ior-easy.stonewall -t 1M -b 9920000m -w -D 300 -O stoneWallingWearOut=1 -a POSIX
throughput-stonewall = 11309.58
score           = 104.452970
t_delta         = 306.5120
t_end           = 2020-10-08 09:35:46

[mdtest-easy-write]
t_start         = 2020-10-08 09:35:46
exe             = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-easy -x ./results/2020.10.08-09.30.31/mdtest-easy.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 168.143195
score           = 162.046187
t_delta         = 335.5016
t_end           = 2020-10-08 09:41:21

[timestamp]
t_start         = 2020-10-08 09:41:21
t_delta         = 0.0008
t_end           = 2020-10-08 09:41:21

[ior-hard-write]
t_start         = 2020-10-08 09:41:21
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.08-09.30.31/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a POSIX
throughput-stonewall = 11495.43
score           = 105.463040
t_delta         = 304.0917
t_end           = 2020-10-08 09:46:25

[mdtest-hard-write]
t_start         = 2020-10-08 09:46:25
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-hard -x ./results/2020.10.08-09.30.31/mdtest-hard.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 38.664838
score           = 37.487877
t_delta         = 340.0631
t_end           = 2020-10-08 09:52:05

[find]
t_start         = 2020-10-08 09:52:05
exe             = ./pfind /tmp/fuseMNT/datafiles/2020.10.08-09.30.31 -newer ./results/2020.10.08-09.30.31/timestampfile -size 3901c -name *01* -C -q 50000
found           = 555035
total-files     = 62876552
score           = 2727.657152
t_delta         = 23.1568
t_end           = 2020-10-08 09:52:28

[ior-easy-read]
t_start         = 2020-10-08 09:52:28
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.08-09.30.31/ior-easy.stonewall -t 1M -b 9920000m -r -R -a POSIX
score           = 99.042243
t_delta         = 323.1569
t_end           = 2020-10-08 09:57:52

[mdtest-easy-stat]
t_start         = 2020-10-08 09:57:52
exe             = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-easy -x ./results/2020.10.08-09.30.31/mdtest-easy.stonewall -T -a POSIX
score           = 837.196401
t_delta         = 61.0451
t_end           = 2020-10-08 09:58:53

[ior-hard-read]
t_start         = 2020-10-08 09:58:53
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.08-09.30.31/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a POSIX
score           = 97.682712
t_delta         = 328.2957
t_end           = 2020-10-08 10:04:21

[mdtest-hard-stat]
t_start         = 2020-10-08 10:04:21
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-hard -x ./results/2020.10.08-09.30.31/mdtest-hard.stonewall -T -a POSIX
score           = 884.257472
t_delta         = 13.3222
t_end           = 2020-10-08 10:04:34

[mdtest-easy-delete]
t_start         = 2020-10-08 10:04:34
exe             = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-easy -x ./results/2020.10.08-09.30.31/mdtest-easy.stonewall -r -a POSIX
score           = 201.209734
t_delta         = 254.6369
t_end           = 2020-10-08 10:08:49

[mdtest-hard-read]
t_start         = 2020-10-08 10:08:49
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-hard -x ./results/2020.10.08-09.30.31/mdtest-hard.stonewall -E -X -a POSIX
score           = 326.307593
t_delta         = 36.0913
t_end           = 2020-10-08 10:09:25

[mdtest-hard-delete]
t_start         = 2020-10-08 10:09:25
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/fuseMNT/datafiles/2020.10.08-09.30.31/mdtest-hard -x ./results/2020.10.08-09.30.31/mdtest-hard.stonewall -r -a POSIX
score           = 23.472137
t_delta         = 501.8947
t_end           = 2020-10-08 10:17:47

[SCORE]
MD              = 256.784787
BW              = 101.604938
SCORE           = 161.525856 
hash            = D1FAEB30
; END 2020-10-08 10:17:48
result_summary
IO500 version io500-sc20_v2
[RESULT]       ior-easy-write      104.452970 GiB/s : time 306.512 seconds
[RESULT]    mdtest-easy-write      162.046187 kIOPS : time 335.502 seconds
[RESULT]       ior-hard-write      105.463040 GiB/s : time 304.092 seconds
[RESULT]    mdtest-hard-write       37.487877 kIOPS : time 340.063 seconds
[RESULT]                 find     2727.657152 kIOPS : time 23.157 seconds
[RESULT]        ior-easy-read       99.042243 GiB/s : time 323.157 seconds
[RESULT]     mdtest-easy-stat      837.196401 kIOPS : time 61.045 seconds
[RESULT]        ior-hard-read       97.682712 GiB/s : time 328.296 seconds
[RESULT]     mdtest-hard-stat      884.257472 kIOPS : time 13.322 seconds
[RESULT]   mdtest-easy-delete      201.209734 kIOPS : time 254.637 seconds
[RESULT]     mdtest-hard-read      326.307593 kIOPS : time 36.091 seconds
[RESULT]   mdtest-hard-delete       23.472137 kIOPS : time 501.895 seconds
[SCORE] Bandwidth 101.604938 GB/s : IOPS 256.784787 kiops : TOTAL 161.525856