bracewell scratch2

Institution CSIRO
Client Procs Per Node
Client Operating System SUSE
Client Operating System Version SLES 12 SP4
Client Kernel Version 4.12

DATA SERVER

Storage Type NVMe
Volatile Memory 192GB
Storage Interface NVMe
Network IB EDR
Software Version 7.1.3
OS Version SLES 12 SP4

INFORMATION

Client Nodes 26
Client Total Procs 260
Metadata Nodes 2
Metadata Storage Devices 24
Data Nodes 12
Data Storage Devices 24

METADATA

Easy Write 143.57 kIOP/s
Easy Stat 882.77 kIOP/s
Easy Delete 231.88 kIOP/s
Hard Write 10.95 kIOP/s
Hard Read 36.15 kIOP/s
Hard Stat 139.09 kIOP/s
Hard Delete 17.56 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
hostfile="host.io500"

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
    mkdir -p ${io500_workdir}/ior_easy ${io500_workdir}/ior_hard
  mkdir -p ${io500_workdir}/mdt_easy ${io500_workdir}/mdt_hard
# for ior_easy.
  beegfs-ctl --cfgFile=/etc/beegfs/beegfs-client-cl2.conf --setpattern --numtargets=4 --chunksize=1024k ${io500_workdir}/ior_easy
# stripe across all OSTs for ior_hard, 256k chunksize
  #beegfs-ctl --setpattern --numtargets=96 --chunksize=256k ${io500_workdir}/ior_hard
  beegfs-ctl --cfgFile=/etc/beegfs/beegfs-client-cl2.conf --setpattern --numtargets=96 --chunksize=256k ${io500_workdir}/ior_hard
# turn off striping and use small chunks for mdtest
  beegfs-ctl --cfgFile=/etc/beegfs/beegfs-client-cl2.conf --setpattern --numtargets=1 --chunksize=64k ${io500_workdir}/mdt_easy
  beegfs-ctl --cfgFile=/etc/beegfs/beegfs-client-cl2.conf --setpattern --numtargets=1 --chunksize=64k ${io500_workdir}/mdt_hard
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="--hostfile ${hostfile} "
  #io500_mpiargs="-x LD_LIBRARY_PATH -np 184 --hostfile hosts_mpich.io500 "
}

function setup_ior_easy {
  #io500_ior_easy_params="-t 2048k -b \${io500_ior_easy_size}m -F"
  io500_ior_easy_size=860
  io500_ior_easy_params="-t 2048k -b ${io500_ior_easy_size}g -F --posix.odirect "
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
  #io500_mdtest_easy_params="-u -L --posix.odirect" # unique dir per thread, files only at leaves
  #io500_mdtest_easy_files_per_proc=430000
}

function setup_ior_hard {
  io500_ior_hard_api="POSIX"
  io500_ior_hard_api_specific_options=""
  io500_ior_hard_writes_per_proc=180000
}

function setup_mdt_hard {
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
  #io500_mdtest_hard_files_per_proc=23500
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind
  #io500_find_cmd_args=""
  io500_find_cmd_args="-s 20000 -r $io500_result_dir/pfind_results"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
  # TODO: Please add your information using the info-creator!
  # EXAMPLE:
  io500_info_system_name='scratch2'      # e.g. Oakforest-PACS
}

echo "Using these Client Nodes:"
echo "========================="
cat $hostfile
echo "Using these storage nodes:"
echo "=========================="
beegfs-ctl --listnodes --nodetype=storage --mount=/scratch2
echo "Using these metadata nodes:"
echo "=================== ======="
beegfs-ctl --listnodes --nodetype=meta --mount=/scratch2
echo start run
main
cp $hostfile $io500_result_dir
cp $0 $io500_result_dir
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 12 15:35:23 2019
Command line        : /scratch2/leh015/io500-sc19/bin/ior -r -R -t 2048k -b 860g -F --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -o /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_easy/ior_file_easy -O stoneWallingStatusFile=/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_easy/stonewall
Machine             : Linux b003
TestID              : 0
StartTime           : Tue Nov 12 15:35:23 2019
Path                : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_easy
FS                  : 838.1 TiB   Used FS: 10.9%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 260
clients per node    : 10
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 860 GiB
aggregate filesize  : 218.36 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
WARNING: Expected aggregate file size       = 240088671846400.
WARNING: Stat() of aggregate file size      = 98518035333120.
WARNING: Using actual aggregate bytes moved = 98518035333120.
read      184765     901775360  2048.00    0.013338   508.48     0.008998   508.51     0   
Max Read:  184764.99 MiB/sec (193740.13 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       184764.99  184764.99  184764.99       0.00   92382.49   92382.49   92382.49       0.00  508.50608         NA            NA     0    260  10    1   1     1        1         0    0      1 923417968640  2097152 93954120.0 POSIX      0
Finished            : Tue Nov 12 15:43:52 2019
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 12 15:05:41 2019
Command line        : /scratch2/leh015/io500-sc19/bin/ior -w -t 2048k -b 860g -F --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -o /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_easy/ior_file_easy -O stoneWallingStatusFile=/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux b003
TestID              : 0
StartTime           : Tue Nov 12 15:05:41 2019
Path                : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_easy
FS                  : 838.1 TiB   Used FS: 0.0%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 260
clients per node    : 10
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 860 GiB
aggregate filesize  : 218.36 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 127130 max: 180681 -- min data: 248.3 GiB mean data: 275.2 GiB time: 300.0s
WARNING: Expected aggregate file size       = 240088671846400.
WARNING: Stat() of aggregate file size      = 98518035333120.
WARNING: Using actual aggregate bytes moved = 98518035333120.
WARNING: maybe caused by deadlineForStonewalling
write     234476     901775360  2048.00    0.010719   400.67     0.015226   400.70     0   
Max Write: 234476.49 MiB/sec (245866.42 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      234476.49  234476.49  234476.49       0.00  117238.25  117238.25  117238.25       0.00  400.69740     300.00     244218.77     0    260  10    1   1     1        1         0    0      1 923417968640  2097152 93954120.0 POSIX      0
Finished            : Tue Nov 12 15:12:22 2019
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 12 15:45:51 2019
Command line        : /scratch2/leh015/io500-sc19/bin/ior -r -R -s 180000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_hard/IOR_file -O stoneWallingStatusFile=/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_hard/stonewall
Machine             : Linux b003
TestID              : 0
StartTime           : Tue Nov 12 15:45:51 2019
Path                : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_hard
FS                  : 838.1 TiB   Used FS: 10.9%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 180000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 260
clients per node    : 10
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 2.00 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
read      104077     45.91      45.91      0.180298   19.95      0.026819   20.16      0   
Max Read:  104076.73 MiB/sec (109132.36 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       104076.73  104076.73  104076.73       0.00 2321570.05 2321570.05 2321570.05       0.00   20.15877         NA            NA     0    260  10    1   0     1        1         0    0 180000    47008    47008 2098059.0 POSIX      0
Finished            : Tue Nov 12 15:46:12 2019
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 12 15:23:24 2019
Command line        : /scratch2/leh015/io500-sc19/bin/ior -w -s 180000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_hard/IOR_file -O stoneWallingStatusFile=/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux b003
TestID              : 0
StartTime           : Tue Nov 12 15:23:24 2019
Path                : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_hard
FS                  : 838.1 TiB   Used FS: 10.7%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 180000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 260
clients per node    : 10
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 2.00 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 39560 max: 180000 -- min data: 1.7 GiB mean data: 6.3 GiB time: 300.0s
write     5560       45.91      45.91      0.179194   377.15     0.024519   377.35     0   
Max Write: 5559.93 MiB/sec (5830.01 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        5559.93    5559.93    5559.93       0.00  124021.75  124021.75  124021.75       0.00  377.35317     300.00       5560.74     0    260  10    1   0     1        1         0    0 180000    47008    47008 2098059.0 POSIX      0
Finished            : Tue Nov 12 15:29:41 2019
mdtest_easy_delete
-- started at 11/12/2019 15:46:39 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 26 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-r' '-F' '-P' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_easy-stonewall' '-N' '1'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40
FS: 838.1 TiB   Used FS: 10.9%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 10 for each phase.
260 tasks, 234000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     231878.490     231878.369     231878.463          0.021
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.053          0.053          0.053          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        407.312        407.312        407.312          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         18.948         18.948         18.948          0.000
-- finished at 11/12/2019 15:53:52 --

mdtest_easy_stat
-- started at 11/12/2019 15:43:54 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 26 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-T' '-F' '-P' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_easy-stonewall' '-N' '1'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40
FS: 838.1 TiB   Used FS: 10.9%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 10 for each phase.
260 tasks, 234000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     882768.143     882767.547     882767.892          0.141
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        106.989        106.989        106.989          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/12/2019 15:45:41 --

mdtest_easy_write
-- started at 11/12/2019 15:12:24 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 26 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-C' '-F' '-P' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40
FS: 838.1 TiB   Used FS: 10.7%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 10 for each phase.
260 tasks, 234000000 files
Continue stonewall hit min: 154964 max: 363257 avg: 208938.2 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     143571.019     143570.943     143570.977          0.020
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     181096.788             NA
   Tree creation             :         49.020         49.020         49.020          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        657.841        657.840        657.841          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.972             NA
   Tree creation             :          0.020          0.020          0.020          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/12/2019 15:23:21 --

mdtest_hard_delete
-- started at 11/12/2019 15:55:28 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 26 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_hard' '-n' '950000' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40
FS: 838.1 TiB   Used FS: 10.9%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 10 for each phase.
260 tasks, 247000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      17560.039      17560.029      17560.035          0.002
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.300          0.300          0.300          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        190.010        190.010        190.010          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.330          3.330          3.330          0.000
-- finished at 11/12/2019 15:58:42 --

mdtest_hard_read
-- started at 11/12/2019 15:53:54 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 26 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_hard' '-n' '950000' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40
FS: 838.1 TiB   Used FS: 10.9%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 10 for each phase.
260 tasks, 247000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      36148.556      36148.529      36148.548          0.004
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :         92.302         92.302         92.302          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/12/2019 15:55:26 --

mdtest_hard_stat
-- started at 11/12/2019 15:46:13 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 26 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_hard' '-n' '950000' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40
FS: 838.1 TiB   Used FS: 10.9%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 10 for each phase.
260 tasks, 247000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     139091.584     139091.427     139091.546          0.026
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         23.988         23.988         23.988          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/12/2019 15:46:37 --

mdtest_hard_write
-- started at 11/12/2019 15:29:43 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 26 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_hard' '-n' '950000' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.12-15.05.40
FS: 838.1 TiB   Used FS: 10.9%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 10 for each phase.
260 tasks, 247000000 files
Continue stonewall hit min: 12554 max: 12833 avg: 12650.3 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      10952.359      10952.353      10952.355          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      10963.475             NA
   Tree creation             :       1758.418       1758.418       1758.418          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        304.645        304.645        304.645          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.004             NA
   Tree creation             :          0.001          0.001          0.001          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/12/2019 15:34:48 --

result_summary
[RESULT] BW   phase 1            ior_easy_write              228.980 GB/s : time 400.70 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              143.571 kiops : time 657.84 seconds
[RESULT] BW   phase 2            ior_hard_write                5.430 GB/s : time 377.35 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               10.952 kiops : time 304.64 seconds
[RESULT] IOPS phase 3                      find             2910.620 kiops : time  33.60 seconds
[RESULT] BW   phase 3             ior_easy_read              180.435 GB/s : time 508.51 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              882.768 kiops : time 106.99 seconds
[RESULT] BW   phase 4             ior_hard_read              101.638 GB/s : time  20.16 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat              139.092 kiops : time  23.99 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              231.878 kiops : time 407.31 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               36.149 kiops : time  92.30 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               17.560 kiops : time 195.50 seconds
[SCORE] Bandwidth 69.1015 GB/s : IOPS 130.226 kiops : TOTAL 94.8621