BEAR-AI

Institution University of Birmingham
Client Procs Per Node
Client Operating System RHEL
Client Operating System Version 7.6
Client Kernel Version 4.14.0-115.7.1.el7a.ppc64le

DATA SERVER

Storage Type NVMe
Volatile Memory 96GB
Storage Interface NVMe
Network Infiniband
Software Version 5.0.2.3
OS Version RHEL 7.6

INFORMATION

Client Nodes 10
Client Total Procs 80

METADATA

Easy Write 103.11 kIOP/s
Easy Stat 188.93 kIOP/s
Easy Delete 51.04 kIOP/s
Hard Write 23.01 kIOP/s
Hard Read 124.97 kIOP/s
Hard Stat 188.01 kIOP/s
Hard Delete 11.72 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="/usr/mpi/gcc/openmpi-4.0.0rc5/bin/mpirun"
  io500_mpiargs="-np 80 --map-by node -machinefile 10nodes --allow-run-as-root"
}

function setup_ior_easy {
  #io500_ior_easy_params="-t 2048k -b \${io500_ior_easy_size}m -F"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
  io500_mdtest_easy_files_per_proc="1344944"
}

function setup_ior_hard {
  io500_ior_hard_api="POSIX"
  io500_ior_hard_api_specific_options=""
}

function setup_mdt_hard {
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind
  io500_find_cmd_args=""

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
  io500_info_submitter="Simon Thompson"
  io500_info_10nodechallenge="true"
  io500_info_institution="University of Birmingham"
  io500_info_system="BEAR-AI"
  io500_info_storage_install_date="2019-04-01"
  io500_info_storage_refresh_date="2019-04-01"
  io500_info_storage_vendor="DDN"
  io500_info_filesystem_type="Spectrum Scale"
  io500_info_filesystem_version="5.0.3.3"
  io500_info_client_nodes="10"
  io500_info_client_procs_per_node="8"
  io500_info_client_operating_system="RHEL"
  io500_info_client_operating_system_version="7.6"
  io500_info_client_kernel_version="4.14.0-115.7.1.el7a.ppc64le"
  io500_info_ds_nodes="4"
  io500_info_ds_storage_devices="20"
  io500_info_ds_storage_type="NVMe"
  io500_info_ds_volatile_memory_capacity="96GB"
  io500_info_ds_storage_interface="NVMe"
  io500_info_ds_network="Infiniband"
  io500_info_ds_software_version="5.0.2.3"
  io500_info_ds_operating_system_version="RHEL 7.6"
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 10 21:12:58 2019
Command line        : /ai/io500-test/2019-11/ppc64le/bin/ior -r -R -t 2048k -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_easy/ior_file_easy -O stoneWallingStatusFile=/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_easy/stonewall
Machine             : Linux bear-pg0305u03a.bear.cluster
TestID              : 0
StartTime           : Sun Nov 10 21:12:58 2019
Path                : /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_easy
FS                  : 109.5 TiB   Used FS: 7.6%   Inodes: 147.7 Mi   Used Inodes: 30.5%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 80
clients per node    : 8
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 756.84 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
WARNING: Expected aggregate file size       = 832149913600000.
WARNING: Stat() of aggregate file size      = 3258974208000.
WARNING: Using actual aggregate bytes moved = 3258974208000.
read      25483      10158080000 2048.00    0.005080   121.96     0.000289   121.96     0   
Max Read:  25483.23 MiB/sec (26721.10 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        25483.23   25483.23   25483.23       0.00   12741.61   12741.61   12741.61       0.00  121.96257         NA            NA     0     80   8    1   1     1        1         0    0      1 10401873920000  2097152 3108000.0 POSIX      0
Finished            : Sun Nov 10 21:15:00 2019
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 10 20:31:26 2019
Command line        : /ai/io500-test/2019-11/ppc64le/bin/ior -w -t 2048k -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_easy/ior_file_easy -O stoneWallingStatusFile=/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux bear-pg0305u03a.bear.cluster
TestID              : 0
StartTime           : Sun Nov 10 20:31:26 2019
Path                : /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_easy
FS                  : 109.5 TiB   Used FS: 4.3%   Inodes: 147.7 Mi   Used Inodes: 0.2%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 80
clients per node    : 8
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 756.84 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 15031 max: 19425 -- min data: 29.4 GiB mean data: 33.6 GiB time: 301.6s
WARNING: Expected aggregate file size       = 832149913600000.
WARNING: Stat() of aggregate file size      = 3258974208000.
WARNING: Using actual aggregate bytes moved = 3258974208000.
WARNING: maybe caused by deadlineForStonewalling
write     9096       10158080000 2048.00    0.116563   341.56     0.002836   341.68     0   
Max Write: 9096.16 MiB/sec (9538.01 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        9096.16    9096.16    9096.16       0.00    4548.08    4548.08    4548.08       0.00  341.68281     301.61       9137.51     0     80   8    1   1     1        1         0    0      1 10401873920000  2097152 3108000.0 POSIX      0
Finished            : Sun Nov 10 20:37:07 2019
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 10 21:18:05 2019
Command line        : /ai/io500-test/2019-11/ppc64le/bin/ior -r -R -s 1900000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_hard/IOR_file -O stoneWallingStatusFile=/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_hard/stonewall
Machine             : Linux bear-pg0305u03a.bear.cluster
TestID              : 0
StartTime           : Sun Nov 10 21:18:05 2019
Path                : /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_hard
FS                  : 109.5 TiB   Used FS: 7.6%   Inodes: 147.7 Mi   Used Inodes: 30.5%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 80
clients per node    : 8
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 6.50 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
WARNING: Expected aggregate file size       = 7145216000000.
WARNING: Stat() of aggregate file size      = 667231552000.
WARNING: Using actual aggregate bytes moved = 667231552000.
read      481.89     45.91      45.91      0.006416   1320.47    0.000449   1320.48    0   
Max Read:  481.89 MiB/sec (505.29 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read          481.89     481.89     481.89       0.00   10749.11   10749.11   10749.11       0.00 1320.48150         NA            NA     0     80   8    1   0     1        1         0    0 1900000    47008    47008  636321.6 POSIX      0
Finished            : Sun Nov 10 21:40:06 2019
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 10 20:42:43 2019
Command line        : /ai/io500-test/2019-11/ppc64le/bin/ior -w -s 1900000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_hard/IOR_file -O stoneWallingStatusFile=/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux bear-pg0305u03a.bear.cluster
TestID              : 0
StartTime           : Sun Nov 10 20:42:43 2019
Path                : /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_hard
FS                  : 109.5 TiB   Used FS: 7.0%   Inodes: 147.7 Mi   Used Inodes: 18.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 80
clients per node    : 8
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 6.50 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 17772 max: 177425 -- min data: 0.8 GiB mean data: 1.3 GiB time: 300.0s
WARNING: Expected aggregate file size       = 7145216000000.
WARNING: Stat() of aggregate file size      = 667231552000.
WARNING: Using actual aggregate bytes moved = 667231552000.
WARNING: maybe caused by deadlineForStonewalling
write     552.52     45.91      45.91      0.034247   1151.62    0.011290   1151.67    0   
Max Write: 552.52 MiB/sec (579.36 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write         552.52     552.52     552.52       0.00   12324.73   12324.73   12324.73       0.00 1151.66833     300.03        346.48     0     80   8    1   0     1        1         0    0 1900000    47008    47008  636321.6 POSIX      0
Finished            : Sun Nov 10 21:01:54 2019
mdtest_easy_delete
-- started at 11/10/2019 21:41:19 --

mdtest-3.3.0+dev was launched with 80 total task(s) on 10 node(s)
Command line used: /ai/io500-test/2019-11/ppc64le/bin/mdtest '-r' '-F' '-P' '-d' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_easy' '-n' '1344944' '-u' '-L' '-x' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_easy-stonewall' '-N' '1'
Path: /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07
FS: 109.5 TiB   Used FS: 7.6%   Inodes: 147.7 Mi   Used Inodes: 30.5%

Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2128 Shifting ranks by 1 for each phase.
80 tasks, 107595520 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      51037.944      51037.934      51037.939          0.003
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.266          0.266          0.266          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        666.704        666.704        666.704          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.753          3.753          3.753          0.000
-- finished at 11/10/2019 21:52:30 --

mdtest_easy_stat
-- started at 11/10/2019 21:15:03 --

mdtest-3.3.0+dev was launched with 80 total task(s) on 10 node(s)
Command line used: /ai/io500-test/2019-11/ppc64le/bin/mdtest '-T' '-F' '-P' '-d' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_easy' '-n' '1344944' '-u' '-L' '-x' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_easy-stonewall' '-N' '1'
Path: /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07
FS: 109.5 TiB   Used FS: 7.6%   Inodes: 147.7 Mi   Used Inodes: 30.5%

Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2128 Shifting ranks by 1 for each phase.
80 tasks, 107595520 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     188932.221     188932.163     188932.197          0.017
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        180.103        180.103        180.103          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/10/2019 21:18:03 --

mdtest_easy_write
-- started at 11/10/2019 20:37:10 --

mdtest-3.3.0+dev was launched with 80 total task(s) on 10 node(s)
Command line used: /ai/io500-test/2019-11/ppc64le/bin/mdtest '-C' '-F' '-P' '-d' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_easy' '-n' '1344944' '-u' '-L' '-x' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07
FS: 109.5 TiB   Used FS: 6.7%   Inodes: 147.7 Mi   Used Inodes: 0.2%

Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2128 Shifting ranks by 1 for each phase.
80 tasks, 107595520 files
Continue stonewall hit min: 359182 max: 425340 avg: 399771.1 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     103110.418     103110.396     103110.409          0.005
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     106730.984             NA
   Tree creation             :          2.562          2.562          2.562          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        330.007        330.007        330.007          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.648             NA
   Tree creation             :          0.390          0.390          0.390          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/10/2019 20:42:40 --

mdtest_hard_delete
-- started at 11/10/2019 21:54:19 --

mdtest-3.3.0+dev was launched with 80 total task(s) on 10 node(s)
Command line used: /ai/io500-test/2019-11/ppc64le/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_hard' '-n' '950000' '-x' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07
FS: 109.5 TiB   Used FS: 7.6%   Inodes: 147.7 Mi   Used Inodes: 8.8%

Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2128 Shifting ranks by 1 for each phase.
80 tasks, 76000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      11723.285      11723.283      11723.284          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         12.512         12.512         12.512          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       1108.208       1108.208       1108.208          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.080          0.080          0.080          0.000
-- finished at 11/10/2019 22:12:47 --

mdtest_hard_read
-- started at 11/10/2019 21:52:32 --

mdtest-3.3.0+dev was launched with 80 total task(s) on 10 node(s)
Command line used: /ai/io500-test/2019-11/ppc64le/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_hard' '-n' '950000' '-x' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07
FS: 109.5 TiB   Used FS: 7.6%   Inodes: 147.7 Mi   Used Inodes: 9.8%

Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2128 Shifting ranks by 1 for each phase.
80 tasks, 76000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     124972.117     124972.063     124972.085          0.012
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        103.958        103.958        103.958          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/10/2019 21:54:16 --

mdtest_hard_stat
-- started at 11/10/2019 21:40:08 --

mdtest-3.3.0+dev was launched with 80 total task(s) on 10 node(s)
Command line used: /ai/io500-test/2019-11/ppc64le/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_hard' '-n' '950000' '-x' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07
FS: 109.5 TiB   Used FS: 7.6%   Inodes: 147.7 Mi   Used Inodes: 30.5%

Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2128 Shifting ranks by 1 for each phase.
80 tasks, 76000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     188008.066     188007.976     188008.014          0.020
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         69.103         69.103         69.103          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/10/2019 21:41:17 --

mdtest_hard_write
-- started at 11/10/2019 21:01:56 --

mdtest-3.3.0+dev was launched with 80 total task(s) on 10 node(s)
Command line used: /ai/io500-test/2019-11/ppc64le/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_hard' '-n' '950000' '-x' '/ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /ai/io500-test/2019-11/ppc64le/datafiles/io500.2019.11.10-20.31.07
FS: 109.5 TiB   Used FS: 7.6%   Inodes: 147.7 Mi   Used Inodes: 22.2%

Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2128 Shifting ranks by 1 for each phase.
80 tasks, 76000000 files
Continue stonewall hit min: 90412 max: 162398 avg: 103444.1 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      23009.911      23009.904      23009.907          0.002
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      27584.710             NA
   Tree creation             :        572.410        572.410        572.410          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        564.619        564.619        564.619          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.004             NA
   Tree creation             :          0.002          0.002          0.002          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/10/2019 21:11:21 --

result_summary
[RESULT] BW   phase 1            ior_easy_write                8.883 GB/s : time 341.68 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              103.110 kiops : time 330.01 seconds
[RESULT] BW   phase 2            ior_hard_write                0.540 GB/s : time 1151.67 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               23.010 kiops : time 564.62 seconds
[RESULT] IOPS phase 3                      find              496.590 kiops : time  94.68 seconds
[RESULT] BW   phase 3             ior_easy_read               24.886 GB/s : time 121.96 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              188.932 kiops : time 180.10 seconds
[RESULT] BW   phase 4             ior_hard_read                0.471 GB/s : time 1320.48 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat              188.008 kiops : time  69.10 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               51.038 kiops : time 666.70 seconds
[RESULT] IOPS phase 7          mdtest_hard_read              124.972 kiops : time 103.96 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               11.723 kiops : time 1110.99 seconds
[SCORE] Bandwidth 2.73715 GB/s : IOPS 86.4831 kiops : TOTAL 15.3856