Oracle Cloud Infrastructure with Block Volume Service running Spectrum Scale

Institution Oracle Cloud Infrastructure
Client Procs Per Node
Client Operating System CentOS Linux
Client Operating System Version CentOS Linux release 7.6.1810 (Core)
Client Kernel Version 3.10.0-957.27.2.el7.x86_64

DATA SERVER

Storage Type NVMe SSD
Volatile Memory 768 GB
Storage Interface iSCSI (network attached)
Network 25Gbps Ethernet
Software Version IBM Spectrum Scale 5.0.3.2
OS Version CentOS Linux release 7.6.1810 (Core) with Kernel (3.10.0-957.27.2.el7.x86_64)

INFORMATION

Client Nodes 30
Client Total Procs 480
Metadata Nodes 22
Metadata Storage Devices 11
Data Nodes 22
Data Storage Devices 11

METADATA

Easy Write 430.84 kIOP/s
Easy Stat 604.52 kIOP/s
Easy Delete 341.63 kIOP/s
Hard Write 17.93 kIOP/s
Hard Read 31.54 kIOP/s
Hard Stat 30.86 kIOP/s
Hard Delete 8.72 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="/gpfs/fs1/openmpi-4.0.1/bin/mpirun"
  io500_mpiargs="--allow-run-as-root -n 480 -npernode 16 --hostfile /gpfs/fs1/hostsfile.cn"
}

function setup_ior_easy {
  #io500_ior_easy_params="-t 2048k -b \${io500_ior_easy_size}m -F"
  io500_ior_easy_size=200
  io500_ior_easy_params="-t 2m -b ${io500_ior_easy_size}g -F"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_files_per_proc=1000000
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
}

function setup_ior_hard {
  io500_ior_hard_writes_per_proc=95000
  io500_ior_hard_api="POSIX"
  io500_ior_hard_api_specific_options=""
}

function setup_mdt_hard {
  io500_mdtest_hard_files_per_proc=30000
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind
  # io500_find_cmd_args=""
  io500_find_cmd_args="-s $io500_stonewall_timer -r $io500_result_dir/pfind_results"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  io500_mdreal_params="-P=5000 -I=1000"
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
  # TODO: Please add your information using the info-creator!
  # EXAMPLE:
  io500_info_system_name='xxx'      # e.g. Oakforest-PACS
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Oct  3 23:05:18 2019
Command line        : /gpfs/fs1/io-500-dev-master/bin/ior -r -R -t 2m -b 200g -F -i 1 -C -Q 1 -g -G 27 -k -e -o /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_easy/ior_file_easy -O stoneWallingStatusFile=/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_easy/stonewall
Machine             : Linux ss-compute-1
TestID              : 0
StartTime           : Thu Oct  3 23:05:18 2019
Path                : /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_easy
FS                  : 189.1 TiB   Used FS: 23.8%   Inodes: 581.7 Mi   Used Inodes: 31.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 480
clients per node    : 16
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 200 GiB
aggregate filesize  : 93.75 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
WARNING: Expected aggregate file size       = 103079215104000.
WARNING: Stat() of aggregate file size      = 44663297802240.
WARNING: Using actual aggregate bytes moved = 44663297802240.
read      54881      209715200  2048.00    0.024867   776.07     0.029764   776.12     0   
Max Read:  54881.00 MiB/sec (57546.89 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        54881.00   54881.00   54881.00       0.00   27440.50   27440.50   27440.50       0.00  776.12003         NA            NA     0    480  16    1   1     1        1         0    0      1 214748364800  2097152 42594240.0 POSIX      0
Finished            : Thu Oct  3 23:18:14 2019
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Oct  3 22:20:20 2019
Command line        : /gpfs/fs1/io-500-dev-master/bin/ior -w -t 2m -b 200g -F -i 1 -C -Q 1 -g -G 27 -k -e -o /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_easy/ior_file_easy -O stoneWallingStatusFile=/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux ss-compute-1
TestID              : 0
StartTime           : Thu Oct  3 22:20:20 2019
Path                : /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_easy
FS                  : 189.1 TiB   Used FS: 16.5%   Inodes: 581.7 Mi   Used Inodes: 0.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 480
clients per node    : 16
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 200 GiB
aggregate filesize  : 93.75 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 4285 max: 44369 -- min data: 8.4 GiB mean data: 24.9 GiB time: 323.3s
WARNING: Expected aggregate file size       = 103079215104000.
WARNING: Stat() of aggregate file size      = 44663297802240.
WARNING: Using actual aggregate bytes moved = 44663297802240.
WARNING: maybe caused by deadlineForStonewalling
write     37258      209715200  2048.00    0.265673   1142.94    0.018921   1143.23    0   
Max Write: 37257.89 MiB/sec (39067.73 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       37257.89   37257.89   37257.89       0.00   18628.94   18628.94   18628.94       0.00 1143.22747     323.30      37911.09     0    480  16    1   1     1        1         0    0      1 214748364800  2097152 42594240.0 POSIX      0
Finished            : Thu Oct  3 22:39:23 2019
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Oct  3 23:23:17 2019
Command line        : /gpfs/fs1/io-500-dev-master/bin/ior -r -R -s 95000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_hard/IOR_file -O stoneWallingStatusFile=/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_hard/stonewall
Machine             : Linux ss-compute-1
TestID              : 0
StartTime           : Thu Oct  3 23:23:17 2019
Path                : /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_hard
FS                  : 189.1 TiB   Used FS: 23.8%   Inodes: 581.7 Mi   Used Inodes: 31.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 95000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 480
clients per node    : 16
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 1.95 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
read      25135      45.91      45.91      0.004740   81.05      0.277650   81.33      0   
Max Read:  25135.34 MiB/sec (26356.31 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        25135.34   25135.34   25135.34       0.00  560677.20  560677.20  560677.20       0.00   81.33022         NA            NA     0    480  16    1   0     1        1         0    0  95000    47008    47008 2044262.8 POSIX      0
Finished            : Thu Oct  3 23:24:38 2019
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Oct  3 22:46:33 2019
Command line        : /gpfs/fs1/io-500-dev-master/bin/ior -w -s 95000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_hard/IOR_file -O stoneWallingStatusFile=/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux ss-compute-1
TestID              : 0
StartTime           : Thu Oct  3 22:46:33 2019
Path                : /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_hard
FS                  : 189.1 TiB   Used FS: 22.7%   Inodes: 581.7 Mi   Used Inodes: 23.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 95000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 480
clients per node    : 16
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 1.95 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 35695 max: 95000 -- min data: 1.6 GiB mean data: 2.4 GiB time: 300.6s
write     3693       45.91      45.91      0.034381   553.38     0.094786   553.51     0   
Max Write: 3693.26 MiB/sec (3872.67 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        3693.26    3693.26    3693.26       0.00   82383.17   82383.17   82383.17       0.00  553.51111     300.57       3990.28     0    480  16    1   0     1        1         0    0  95000    47008    47008 2044262.8 POSIX      0
Finished            : Thu Oct  3 22:55:47 2019
mdtest_easy_delete
-- started at 10/03/2019 23:29:05 --

mdtest-3.3.0+dev was launched with 480 total task(s) on 30 node(s)
Command line used: /gpfs/fs1/io-500-dev-master/bin/mdtest '-r' '-F' '-P' '-d' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_easy' '-n' '1000000' '-u' '-L' '-x' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_easy-stonewall' '-N' '1'
Path: /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19
FS: 189.1 TiB   Used FS: 23.8%   Inodes: 581.7 Mi   Used Inodes: 31.1%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 16 for each phase.
480 tasks, 480000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     341629.691     341629.400     341629.533          0.055
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.087          0.087          0.087          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        531.936        531.936        531.936          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         11.499         11.499         11.499          0.000
-- finished at 10/03/2019 23:38:09 --

mdtest_easy_stat
-- started at 10/03/2019 23:18:15 --

mdtest-3.3.0+dev was launched with 480 total task(s) on 30 node(s)
Command line used: /gpfs/fs1/io-500-dev-master/bin/mdtest '-T' '-F' '-P' '-d' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_easy' '-n' '1000000' '-u' '-L' '-x' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_easy-stonewall' '-N' '1'
Path: /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19
FS: 189.1 TiB   Used FS: 23.8%   Inodes: 581.7 Mi   Used Inodes: 31.1%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 16 for each phase.
480 tasks, 480000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     604522.078     604521.030     604521.521          0.221
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        300.610        300.610        300.610          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/03/2019 23:23:16 --

mdtest_easy_write
-- started at 10/03/2019 22:39:24 --

mdtest-3.3.0+dev was launched with 480 total task(s) on 30 node(s)
Command line used: /gpfs/fs1/io-500-dev-master/bin/mdtest '-C' '-F' '-P' '-d' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_easy' '-n' '1000000' '-u' '-L' '-x' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19
FS: 189.1 TiB   Used FS: 16.7%   Inodes: 581.7 Mi   Used Inodes: 0.0%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 16 for each phase.
480 tasks, 480000000 files
Continue stonewall hit min: 240321 max: 378594 avg: 265437.5 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     430838.865     430838.335     430838.581          0.108
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     432853.160             NA
   Tree creation             :          0.174          0.174          0.174          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        421.794        421.794        421.794          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        294.349             NA
   Tree creation             :          5.743          5.743          5.743          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/03/2019 22:46:32 --

mdtest_hard_delete
-- started at 10/03/2019 23:42:30 --

mdtest-3.3.0+dev was launched with 480 total task(s) on 30 node(s)
Command line used: /gpfs/fs1/io-500-dev-master/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_hard' '-n' '30000' '-x' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19
FS: 189.1 TiB   Used FS: 23.8%   Inodes: 581.7 Mi   Used Inodes: 1.3%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 16 for each phase.
480 tasks, 14400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       8722.237       8722.232       8722.234          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         20.493         20.493         20.493          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        937.356        937.356        937.356          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.049          0.049          0.049          0.000
-- finished at 10/03/2019 23:58:07 --

mdtest_hard_read
-- started at 10/03/2019 23:38:10 --

mdtest-3.3.0+dev was launched with 480 total task(s) on 30 node(s)
Command line used: /gpfs/fs1/io-500-dev-master/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_hard' '-n' '30000' '-x' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19
FS: 189.1 TiB   Used FS: 23.8%   Inodes: 581.7 Mi   Used Inodes: 2.1%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 16 for each phase.
480 tasks, 14400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      31543.087      31543.029      31543.057          0.011
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        259.196        259.196        259.196          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/03/2019 23:42:29 --

mdtest_hard_stat
-- started at 10/03/2019 23:24:39 --

mdtest-3.3.0+dev was launched with 480 total task(s) on 30 node(s)
Command line used: /gpfs/fs1/io-500-dev-master/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_hard' '-n' '30000' '-x' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19
FS: 189.1 TiB   Used FS: 23.8%   Inodes: 581.7 Mi   Used Inodes: 31.1%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 16 for each phase.
480 tasks, 14400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      30864.983      30864.926      30864.950          0.011
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        264.891        264.890        264.891          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/03/2019 23:29:04 --

mdtest_hard_write
-- started at 10/03/2019 22:55:48 --

mdtest-3.3.0+dev was launched with 480 total task(s) on 30 node(s)
Command line used: /gpfs/fs1/io-500-dev-master/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_hard' '-n' '30000' '-x' '/gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /gpfs/fs1/io-500-dev-master/datafiles/io500.2019.10.03-22.20.19
FS: 189.1 TiB   Used FS: 23.8%   Inodes: 581.7 Mi   Used Inodes: 29.8%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 16 for each phase.
480 tasks, 14400000 files
Continue stonewall hit min: 11187 max: 17033 avg: 11658.8 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      17927.804      17927.780      17927.794          0.004
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      18633.718             NA
   Tree creation             :       1986.401       1986.401       1986.401          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        456.043        456.042        456.043          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.328             NA
   Tree creation             :          0.001          0.001          0.001          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/03/2019 23:03:24 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               36.385 GB/s : time 1143.23 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              430.839 kiops : time 421.79 seconds
[RESULT] BW   phase 2            ior_hard_write                3.606 GB/s : time 553.51 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               17.928 kiops : time 456.04 seconds
[RESULT] IOPS phase 3                      find             1692.210 kiops : time 112.89 seconds
[RESULT] BW   phase 3             ior_easy_read               53.595 GB/s : time 776.12 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              604.522 kiops : time 300.61 seconds
[RESULT] BW   phase 4             ior_hard_read               24.546 GB/s : time  81.33 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               30.865 kiops : time 264.89 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              341.630 kiops : time 531.94 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               31.543 kiops : time 259.20 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                8.722 kiops : time 938.30 seconds
[SCORE] Bandwidth 20.3833 GB/s : IOPS 110.926 kiops : TOTAL 47.5504