Vast

Institution Janelia Research Campus, HHMI
Client Procs Per Node
Client Operating System Scientific-Linux
Client Operating System Version 7.6
Client Kernel Version 3.10.0-957.10.1.el7.x86_64

DATA SERVER

Storage Type Flash
Volatile Memory 64GB
Storage Interface NVMe
Network 100GbE
Software Version 2.0.6
OS Version 7.6

INFORMATION

Client Nodes 88
Client Total Procs 2,816
Metadata Nodes 12
Metadata Storage Devices 72
Data Nodes 6
Data Storage Devices 1,320

METADATA

Easy Write 44.70 kIOP/s
Easy Stat 306.30 kIOP/s
Easy Delete 65.34 kIOP/s
Hard Write 1.32 kIOP/s
Hard Read 33.21 kIOP/s
Hard Stat 52.11 kIOP/s
Hard Delete 1.72 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 2816 -ppn 32 -f /misc/local/admin/hostlist-cpu --bootstrap ssh"
}

function setup_ior_easy {
  io500_ior_easy_params="-t 1024k -b 9920000m -F"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
}

function setup_ior_hard {
  io500_ior_hard_api="POSIX"
  io500_ior_hard_api_specific_options="--posix.odirect"
}

function setup_mdt_hard {
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="False"
  io500_find_cmd="mpirun -np 880 -ppn 10 -f /misc/local/admin/hostlist-cpu --bootstrap ssh $PWD/bin/pfind"
  #io500_find_cmd=$PWD/bin/pfind
  # uses stonewalling, run pfind
  io500_find_cmd_args="-N"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
  # TODO: Please add your information using the info-creator!
  # EXAMPLE:
  echo io500_info_submitter='"Janelia Scientific Computing Systems"'
  echo io500_info_institution='"Janelia Research Campus, Howard Hughes Medical Institute"'
  echo io500_info_system='"Vast"'
  echo io500_info_storage_install_date='"1/2019"'
  echo io500_info_storage_refresh_date='"8/2019"'
  echo io500_info_storage_vendor='"Vast Data"'
  echo io500_info_filesystem_type='"Scale-out NAS"'
  echo io500_info_filesystem_version='"2.0.6"'
  echo io500_info_client_nodes='"88"'
  echo io500_info_client_procs_per_node='"32"'
  echo io500_info_client_operating_system='"Scientific-Linux"'
  echo io500_info_client_operating_system_version='"7.6"'
  echo io500_info_client_kernel_version='"3.10.0-957.10.1.el7.x86_64"'
  echo io500_info_md_nodes='"24"'
  echo io500_info_md_storage_devices='"3"'
  echo io500_info_md_storage_type='"3DXPNVMeSSD"'
  echo io500_info_md_storage_interface='"NVMeoF"'
  echo io500_info_md_network='"50GbE"'
  echo io500_info_ds_nodes='"68"'
  echo io500_info_ds_storage_devices='"220"'
  echo io500_info_ds_storage_type='"NVMe-SSD"'
  echo io500_info_ds_storage_interface='"NVMe"'
  echo io500_info_ds_network='"Ethernet"'
  echo io500_info_whatever='"Metadata server == Vast cnode Data server == Vast dbox 3DXP listed in Metadata Server are actually in dboxes (12 per dbox)"'
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Nov 11 08:32:54 2019
Command line        : /io500/io500-sc19/bin/ior -r -R -t 1024k -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_easy/ior_file_easy -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_easy/stonewall
Machine             : Linux e10u01.int.janelia.org
TestID              : 0
StartTime           : Mon Nov 11 08:32:54 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_easy
FS                  : 3132.7 TiB   Used FS: 49.5%   Inodes: 8270.0 Mi   Used Inodes: 12.2%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 2816
clients per node    : 32
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 26640.62 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
WARNING: Expected aggregate file size       = 29291676958720000.
WARNING: Stat() of aggregate file size      = 22125255589888.
WARNING: Using actual aggregate bytes moved = 22125255589888.
read      39976      10158080000 1024.00    0.121871   527.68     0.020933   527.82     0   
Max Read:  39976.10 MiB/sec (41917.98 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        39976.10   39976.10   39976.10       0.00   39976.10   39976.10   39976.10       0.00  527.82255         NA            NA     0   2816  32    1   1     1        1         0    0      1 10401873920000  1048576 21100288.0 POSIX      0
Finished            : Mon Nov 11 08:41:42 2019
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Nov 11 06:21:32 2019
Command line        : /io500/io500-sc19/bin/ior -w -t 1024k -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_easy/ior_file_easy -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux e10u01.int.janelia.org
TestID              : 0
StartTime           : Mon Nov 11 06:21:32 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_easy
FS                  : 3091.4 TiB   Used FS: 49.5%   Inodes: 8270.0 Mi   Used Inodes: 12.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 2816
clients per node    : 32
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 26640.62 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 4978 max: 7493 -- min data: 4.9 GiB mean data: 6.4 GiB time: 300.5s
WARNING: Expected aggregate file size       = 29291676958720000.
WARNING: Stat() of aggregate file size      = 22125255589888.
WARNING: Using actual aggregate bytes moved = 22125255589888.
WARNING: maybe caused by deadlineForStonewalling
write     12648      10158080000 1024.00    1.63       1666.55    0.059573   1668.24    0   
Max Write: 12648.23 MiB/sec (13262.64 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       12648.23   12648.23   12648.23       0.00   12648.23   12648.23   12648.23       0.00 1668.23974     300.50      61157.42     0   2816  32    1   1     1        1         0    0      1 10401873920000  1048576 21100288.0 POSIX      0
Finished            : Mon Nov 11 06:49:20 2019
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Nov 11 08:42:56 2019
Command line        : /io500/io500-sc19/bin/ior -r -R -s 1900000 -a POSIX --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_hard/IOR_file -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_hard/stonewall
Machine             : Linux e10u01.int.janelia.org
TestID              : 0
StartTime           : Mon Nov 11 08:42:56 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_hard
FS                  : 3132.7 TiB   Used FS: 49.5%   Inodes: 8270.0 Mi   Used Inodes: 12.2%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 2816
clients per node    : 32
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 228.75 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
WARNING: Expected aggregate file size       = 251511603200000.
WARNING: Stat() of aggregate file size      = 1045494022144.
WARNING: Using actual aggregate bytes moved = 1045494022144.
read      4491       45.91      45.91      0.081487   221.91     0.002025   221.99     0   
Max Read:  4491.46 MiB/sec (4709.64 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         4491.46    4491.46    4491.46       0.00  100188.00  100188.00  100188.00       0.00  221.99034         NA            NA     0   2816  32    1   0     1        1         0    0 1900000    47008    47008  997060.8 POSIX      0
Finished            : Mon Nov 11 08:46:38 2019
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Nov 11 06:57:16 2019
Command line        : /io500/io500-sc19/bin/ior -w -s 1900000 -a POSIX --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_hard/IOR_file -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux e10u01.int.janelia.org
TestID              : 0
StartTime           : Mon Nov 11 06:57:16 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_hard
FS                  : 3125.1 TiB   Used FS: 49.6%   Inodes: 8270.0 Mi   Used Inodes: 12.2%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 2816
clients per node    : 32
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 228.75 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 3678 max: 7898 -- min data: 0.2 GiB mean data: 0.2 GiB time: 300.1s
WARNING: Expected aggregate file size       = 251511603200000.
WARNING: Stat() of aggregate file size      = 1045494022144.
WARNING: Using actual aggregate bytes moved = 1045494022144.
WARNING: maybe caused by deadlineForStonewalling
write     2310.76    45.91      45.91      0.127985   431.36     0.002341   431.49     0   
Max Write: 2310.76 MiB/sec (2423.01 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        2310.76    2310.76    2310.76       0.00   51544.64   51544.64   51544.64       0.00  431.48559     300.06       2378.84     0   2816  32    1   0     1        1         0    0 1900000    47008    47008  997060.8 POSIX      0
Finished            : Mon Nov 11 07:04:28 2019
mdtest_easy_delete
-- started at 11/11/2019 08:46:53 --

mdtest-3.3.0+dev was launched with 2816 total task(s) on 88 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-r' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28
FS: 3132.7 TiB   Used FS: 49.5%   Inodes: 8270.0 Mi   Used Inodes: 12.2%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 32 for each phase.
2816 tasks, 2534400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      65342.439      65342.129      65342.282          0.044
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.492          0.492          0.492          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        318.740        318.738        318.739          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.034          2.034          2.034          0.000
-- finished at 11/11/2019 08:52:15 --

mdtest_easy_stat
-- started at 11/11/2019 08:41:44 --

mdtest-3.3.0+dev was launched with 2816 total task(s) on 88 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-T' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28
FS: 3132.7 TiB   Used FS: 49.5%   Inodes: 8270.0 Mi   Used Inodes: 12.2%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 32 for each phase.
2816 tasks, 2534400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     306300.186     306296.294     306299.030          0.401
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         67.997         67.996         67.996          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/11/2019 08:42:52 --

mdtest_easy_write
-- started at 11/11/2019 06:49:22 --

mdtest-3.3.0+dev was launched with 2816 total task(s) on 88 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-C' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28
FS: 3124.4 TiB   Used FS: 49.6%   Inodes: 8270.0 Mi   Used Inodes: 12.0%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 32 for each phase.
2816 tasks, 2534400000 files
Continue stonewall hit min: 4140 max: 7396 avg: 5664.6 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      44703.853      44703.437      44703.556          0.038
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      53650.478             NA
   Tree creation             :          0.345          0.345          0.345          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        465.896        465.891        465.894          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        297.325             NA
   Tree creation             :          2.897          2.897          2.897          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/11/2019 06:57:12 --

mdtest_hard_delete
-- started at 11/11/2019 08:52:38 --

mdtest-3.3.0+dev was launched with 2816 total task(s) on 88 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28
FS: 3130.7 TiB   Used FS: 49.5%   Inodes: 8270.0 Mi   Used Inodes: 12.0%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 32 for each phase.
2816 tasks, 2675200000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       1720.844       1720.835       1720.837          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.570          3.570          3.570          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        340.374        340.372        340.374          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.280          0.280          0.280          0.000
-- finished at 11/11/2019 08:58:18 --

mdtest_hard_read
-- started at 11/11/2019 08:52:18 --

mdtest-3.3.0+dev was launched with 2816 total task(s) on 88 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28
FS: 3131.4 TiB   Used FS: 49.5%   Inodes: 8270.0 Mi   Used Inodes: 12.0%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 32 for each phase.
2816 tasks, 2675200000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      33214.396      33086.095      33213.959          2.812
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :         17.703         17.635         17.635          0.001
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/11/2019 08:52:36 --

mdtest_hard_stat
-- started at 11/11/2019 08:46:40 --

mdtest-3.3.0+dev was launched with 2816 total task(s) on 88 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28
FS: 3132.7 TiB   Used FS: 49.5%   Inodes: 8270.0 Mi   Used Inodes: 12.2%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 32 for each phase.
2816 tasks, 2675200000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      52109.437      52108.171      52108.856          0.276
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         11.241         11.240         11.240          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/11/2019 08:46:52 --

mdtest_hard_write
-- started at 11/11/2019 07:04:30 --

mdtest-3.3.0+dev was launched with 2816 total task(s) on 88 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /io500/io500-sc19/datafiles/io500.2019.11.11-06.21.28
FS: 3128.3 TiB   Used FS: 49.6%   Inodes: 8270.0 Mi   Used Inodes: 12.2%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 32 for each phase.
2816 tasks, 2675200000 files
Continue stonewall hit min: 124 max: 208 avg: 182.2 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       1323.655       1323.651       1323.652          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA       1700.242             NA
   Tree creation             :        156.861        156.861        156.861          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        442.509        442.508        442.509          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        301.798             NA
   Tree creation             :          0.006          0.006          0.006          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/11/2019 07:11:54 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               12.352 GB/s : time 1668.24 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               44.704 kiops : time 465.90 seconds
[RESULT] BW   phase 2            ior_hard_write                2.257 GB/s : time 431.49 seconds
[RESULT] IOPS phase 2         mdtest_hard_write                1.324 kiops : time 442.51 seconds
[RESULT] IOPS phase 3                      find                4.400 kiops : time 4843.85 seconds
[RESULT] BW   phase 3             ior_easy_read               39.039 GB/s : time 527.82 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              306.300 kiops : time  68.00 seconds
[RESULT] BW   phase 4             ior_hard_read                4.386 GB/s : time 221.99 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               52.109 kiops : time  11.24 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               65.342 kiops : time 318.74 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               33.214 kiops : time  17.70 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                1.721 kiops : time 342.01 seconds
[SCORE] Bandwidth 8.31151 GB/s : IOPS 18.7872 kiops : TOTAL 12.496