Institution WekaIO
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.4.1708
Client Kernel Version 3.10.0-693

DATA SERVER

Storage Type SSD
Volatile Memory
Storage Interface NVMe
Network Ethernet
Software Version 3.1.8.1
OS Version CentOS 7.4.1708

INFORMATION

Client Nodes 10
Client Total Procs 700
Metadata Nodes 32
Metadata Storage Devices 6
Data Nodes 32
Data Storage Devices 6

METADATA

Easy Write 68.66 kIOP/s
Easy Stat 2,525.10 kIOP/s
Easy Delete 31.27 kIOP/s
Hard Write 3.94 kIOP/s
Hard Read 745.45 kIOP/s
Hard Stat 2,212.21 kIOP/s
Hard Delete 3.67 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_stonewall_timer=300 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="--oversubscribe -n 700 --allow-run-as-root -H 172.172.3.96,172.172.4.101,172.172.4.102,172.172.4.103,172.172.4.104,172.172.4.105,172.172.4.106,172.172.4.107,172.172.6.82,172.172.6.84 --mca pml ob1 --mca btl self,tcp"
}

function setup_ior_easy {
  # io500_ior_easy_size is the amount of data written per rank in MiB units,
  # but it can be any number as long as it is somehow used to scale the IOR
  # runtime as part of io500_ior_easy_params
  io500_ior_easy_size=20993
  io500_ior_easy_params="-a POSIX -B -t 1024k -b ${io500_ior_easy_size}m -F"
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
  io500_mdtest_easy_files_per_proc=41638
}

function setup_ior_hard {
  io500_ior_hard_writes_per_proc=59213
  io500_ior_hard_other_options="-B -a POSIX" #e.g., -E to keep precreated files using lfs setstripe, or -a MPIIO
}

function setup_mdt_hard {
  io500_mdtest_hard_files_per_proc=20000
  #io500_mdtest_hard_files_per_proc=8000
  io500_mdtest_hard_other_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind 
  io500_find_cmd_args="-s $io500_stonewall_timer -r $io500_result_dir/pfind_results"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  io500_mdreal_params="-P=5000 -I=1000"
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Add key/value pairs defining your system
# Feel free to add extra ones if you'd like
function extra_description {
  # top level info
  io500_info_system_name='WekaIO'      # e.g. Oakforest-PACS
  io500_info_institute_name='WekaIO'   # e.g. JCAHPC
  io500_info_storage_age_in_months='1' # not install date but age since last refresh
  io500_info_storage_install_date='10/18'  # MM/YY
  io500_info_filesystem='WekaIO Matrix'     # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
  io500_info_filesystem_version='3.1.8.1'
  io500_info_filesystem_vendor='WekaIO'
  # client side info
  io500_info_num_client_nodes='10'
  io500_info_procs_per_node='70'
  # server side info
  io500_info_num_metadata_server_nodes='32'
  io500_info_num_data_server_nodes='32'
  io500_info_num_data_storage_devices='192'  # if you have 5 data servers, and each has 5 drives, then this number is 25
  io500_info_num_metadata_storage_devices='192'  # if you have 2 metadata servers, and each has 5 drives, then this number is 10
  io500_info_data_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_metadata_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_storage_network='Ethernet' # infiniband, omnipath, ethernet, etc
  io500_info_storage_interface='NVMe' # SAS, SATA, NVMe, etc
  # miscellaneous
  io500_info_whatever="Note: We don't have a notion of a traditional metadata server architecture.  Our metadata is distributed evenly across all storage nodes"
}

main
ior_easy_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Wed Nov 14 04:52:29 2018
Command line        : /mnt/weka/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -a POSIX -B -t 1024k -b 20993m -F -o /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_easy/stonewall
Machine             : Linux client8
TestID              : 0
StartTime           : Wed Nov 14 04:52:29 2018
Path                : /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_easy
FS                  : 461.0 TiB   Used FS: 49.0%   Inodes: 5802.5 Mi   Used Inodes: 9.8%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 700
clients per node    : 70
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 20.50 GiB
aggregate filesize  : 14.01 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      110341     21496832   1024.00    0.068194   133.11     0.003233   133.18     0   
Max Read:  110340.72 MiB/sec (115700.63 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       110340.72  110340.72  110340.72       0.00  110340.72  110340.72  110340.72       0.00  133.17931     0    700  70    1   1     1        1         0    0      1 22012755968  1048576 14695100.0 POSIX      0
Finished            : Wed Nov 14 04:54:42 2018
ior_easy_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Wed Nov 14 04:25:02 2018
Command line        : /mnt/weka/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -a POSIX -B -t 1024k -b 20993m -F -o /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux client8
TestID              : 0
StartTime           : Wed Nov 14 04:25:02 2018
Path                : /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_easy
FS                  : 461.0 TiB   Used FS: 45.5%   Inodes: 5802.5 Mi   Used Inodes: 8.9%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 700
clients per node    : 70
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 20.50 GiB
aggregate filesize  : 14.01 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 9836 max: 20993 -- min data: 9.6 GiB mean data: 17.2 GiB time: 300.1s
write     31874      21496832   1024.00    0.407689   460.32     0.069488   461.03     0   
Max Write: 31874.45 MiB/sec (33422.78 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       31874.45   31874.45   31874.45       0.00   31874.45   31874.45   31874.45       0.00  461.03073     0    700  70    1   1     1        1         0    0      1 22012755968  1048576 14695100.0 POSIX      0
Finished            : Wed Nov 14 04:32:43 2018
ior_hard_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Wed Nov 14 04:54:59 2018
Command line        : /mnt/weka/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 59213 -B -a POSIX -o /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_hard/stonewall
Machine             : Linux client8
TestID              : 0
StartTime           : Wed Nov 14 04:54:59 2018
Path                : /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_hard
FS                  : 461.0 TiB   Used FS: 49.0%   Inodes: 5802.5 Mi   Used Inodes: 9.8%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 59213
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 700
clients per node    : 70
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 1.77 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      33589      45.91      45.91      0.436710   54.88      0.002980   55.32      0   
Max Read:  33588.88 MiB/sec (35220.49 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        33588.88   33588.88   33588.88       0.00  749244.69  749244.69  749244.69       0.00   55.32118     0    700  70    1   0     1        1         0    0  59213    47008    47008 1858176.5 POSIX      0
Finished            : Wed Nov 14 04:55:55 2018
ior_hard_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Wed Nov 14 04:39:53 2018
Command line        : /mnt/weka/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 59213 -B -a POSIX -o /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux client8
TestID              : 0
StartTime           : Wed Nov 14 04:39:53 2018
Path                : /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_hard
FS                  : 461.0 TiB   Used FS: 48.6%   Inodes: 5802.5 Mi   Used Inodes: 9.7%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 59213
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 700
clients per node    : 70
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 1.77 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 41575 max: 59213 -- min data: 1.8 GiB mean data: 1.8 GiB time: 300.1s
write     4981       45.91      45.91      0.101981   372.90     0.033510   373.04     0   
Max Write: 4981.17 MiB/sec (5223.14 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        4981.17    4981.17    4981.17       0.00  111111.71  111111.71  111111.71       0.00  373.03988     0    700  70    1   0     1        1         0    0  59213    47008    47008 1858176.5 POSIX      0
Finished            : Wed Nov 14 04:46:07 2018
mdtest_easy_delete
-- started at 11/14/2018 04:56:00 --

mdtest-1.9.3 was launched with 700 total task(s) on 10 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-r" "-F" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_easy" "-n" "41638" "-u" "-L" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_easy-stonewall"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00
FS: 461.0 TiB   Used FS: 49.0%   Inodes: 5802.5 Mi   Used Inodes: 9.8%

700 tasks, 29146600 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :      31269.135      31269.135      31269.135          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.940          0.940          0.940          0.000

-- finished at 11/14/2018 05:11:34 --
mdtest_easy_stat
-- started at 11/14/2018 04:54:45 --

mdtest-1.9.3 was launched with 700 total task(s) on 10 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-T" "-F" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_easy" "-n" "41638" "-u" "-L" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_easy-stonewall"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00
FS: 461.0 TiB   Used FS: 49.0%   Inodes: 5802.5 Mi   Used Inodes: 9.8%

700 tasks, 29146600 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :    2525100.637    2525100.637    2525100.637          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/14/2018 04:54:56 --
mdtest_easy_write
-- started at 11/14/2018 04:32:46 --

mdtest-1.9.3 was launched with 700 total task(s) on 10 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-C" "-F" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_easy" "-n" "41638" "-u" "-L" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_easy-stonewall" "-W" "300"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00
FS: 461.0 TiB   Used FS: 48.6%   Inodes: 5802.5 Mi   Used Inodes: 9.2%

700 tasks, 29146600 files
stonewall rank 267: 21948 of 41638 
stonewall rank 227: 22092 of 41638 
stonewall rank 59: 35854 of 41638 
stonewall rank 537: 34430 of 41638 
stonewall rank 235: 34392 of 41638 
stonewall rank 123: 40377 of 41638 
stonewall rank 385: 37047 of 41638 
stonewall rank 298: 36242 of 41638 
stonewall rank 545: 33980 of 41638 
stonewall rank 266: 22208 of 41638 
stonewall rank 51: 24538 of 41638 
stonewall rank 539: 33818 of 41638 
stonewall rank 271: 22339 of 41638 
stonewall rank 409: 37652 of 41638 
stonewall rank 3: 24719 of 41638 
stonewall rank 507: 34043 of 41638 
stonewall rank 211: 34289 of 41638 
stonewall rank 641: 27168 of 41638 
stonewall rank 234: 22175 of 41638 
stonewall rank 371: 37664 of 41638 
stonewall rank 35: 35903 of 41638 
stonewall rank 514: 33825 of 41638 
stonewall rank 107: 28792 of 41638 
stonewall rank 597: 32988 of 41638 
stonewall rank 527: 34551 of 41638 
stonewall rank 210: 21402 of 41638 
stonewall rank 299: 35875 of 41638 
stonewall rank 226: 21922 of 41638 
stonewall rank 387: 37886 of 41638 
stonewall rank 19: 24703 of 41638 
stonewall rank 517: 34097 of 41638 
stonewall rank 243: 22557 of 41638 
stonewall rank 75: 29144 of 41638 
stonewall rank 223: 34089 of 41638 
stonewall rank 667: 26539 of 41638 
stonewall rank 46: 24248 of 41638 
stonewall rank 519: 34364 of 41638 
stonewall rank 627: 33325 of 41638 
stonewall rank 521: 34949 of 41638 
stonewall rank 269: 21570 of 41638 
stonewall rank 407: 37152 of 41638 
stonewall rank 303: 24028 of 41638 
stonewall rank 526: 33899 of 41638 
stonewall rank 268: 22397 of 41638 
stonewall rank 99: 28439 of 41638 
stonewall rank 513: 34194 of 41638 
stonewall rank 238: 33535 of 41638 
stonewall rank 665: 25952 of 41638 
stonewall rank 433: 22907 of 41638 
stonewall rank 10: 24794 of 41638 
stonewall rank 554: 34448 of 41638 
stonewall rank 245: 22230 of 41638 
stonewall rank 637: 27031 of 41638 
stonewall rank 14: 34627 of 41638 
stonewall rank 548: 34152 of 41638 
stonewall rank 249: 33508 of 41638 
stonewall rank 359: 37543 of 41638 
stonewall rank 577: 32978 of 41638 
stonewall rank 522: 34539 of 41638 
stonewall rank 255: 34555 of 41638 
stonewall rank 115: 29322 of 41638 
stonewall rank 302: 24654 of 41638 
stonewall rank 518: 34033 of 41638 
stonewall rank 258: 22292 of 41638 
stonewall rank 689: 26462 of 41638 
stonewall rank 467: 23379 of 41638 
stonewall rank 532: 33988 of 41638 
stonewall rank 260: 21979 of 41638 
stonewall rank 373: 36438 of 41638 
stonewall rank 529: 33206 of 41638 
stonewall rank 277: 22041 of 41638 
stonewall rank 697: 26046 of 41638 
stonewall rank 58: 36005 of 41638 
stonewall rank 553: 34499 of 41638 
stonewall rank 244: 22012 of 41638 
stonewall rank 391: 37571 of 41638 
stonewall rank 581: 32778 of 41638 
stonewall rank 556: 34585 of 41638 
stonewall rank 278: 21573 of 41638 
stonewall rank 131: 40874 of 41638 
stonewall rank 330: 35220 of 41638 
stonewall rank 504: 34700 of 41638 
stonewall rank 252: 22397 of 41638 
stonewall rank 633: 26988 of 41638 
stonewall rank 473: 23472 of 41638 
stonewall rank 34: 25010 of 41638 
stonewall rank 552: 33579 of 41638 
stonewall rank 248: 34342 of 41638 
stonewall rank 399: 36848 of 41638 
stonewall rank 677: 38502 of 41638 
stonewall rank 1: 24756 of 41638 
stonewall rank 544: 33151 of 41638 
stonewall rank 220: 22340 of 41638 
stonewall rank 619: 32750 of 41638 
stonewall rank 496: 34573 of 41638 
stonewall rank 265: 21871 of 41638 
stonewall rank 693: 27361 of 41638 
stonewall rank 295: 35215 of 41638 
stonewall rank 512: 34328 of 41638 
stonewall rank 272: 34137 of 41638 
stonewall rank 358: 37679 of 41638 
stonewall rank 421: 23022 of 41638 
stonewall rank 233: 22036 of 41638 
stonewall rank 79: 28702 of 41638 
stonewall rank 230: 21555 of 41638 
stonewall rank 81: 29073 of 41638 
stonewall rank 27: 36436 of 41638 
stonewall rank 53: 35555 of 41638 
stonewall rank 219: 33800 of 41638 
stonewall rank 683: 38935 of 41638 
stonewall rank 364: 37522 of 41638 
stonewall rank 57: 24560 of 41638 
stonewall rank 256: 22328 of 41638 
stonewall rank 65: 35529 of 41638 
stonewall rank 110: 40386 of 41638 
stonewall rank 569: 33192 of 41638 
stonewall rank 675: 26651 of 41638 
stonewall rank 305: 24753 of 41638 
stonewall rank 393: 37751 of 41638 
stonewall rank 437: 23324 of 41638 
stonewall rank 82: 28817 of 41638 
stonewall rank 635: 38535 of 41638 
stonewall rank 21: 24881 of 41638 
stonewall rank 25: 35756 of 41638 
stonewall rank 390: 37683 of 41638 
stonewall rank 122: 40475 of 41638 
stonewall rank 39: 24513 of 41638 
stonewall rank 598: 32854 of 41638 
stonewall rank 643: 38488 of 41638 
stonewall rank 281: 24763 of 41638 
stonewall rank 398: 37387 of 41638 
stonewall rank 77: 29186 of 41638 
stonewall rank 441: 23179 of 41638 
stonewall rank 50: 34852 of 41638 
stonewall rank 685: 26471 of 41638 
stonewall rank 582: 33367 of 41638 
stonewall rank 415: 37816 of 41638 
stonewall rank 396: 36757 of 41638 
stonewall rank 283: 35605 of 41638 
stonewall rank 469: 22788 of 41638 
stonewall rank 91: 28931 of 41638 
stonewall rank 5: 24345 of 41638 
stonewall rank 645: 27120 of 41638 
stonewall rank 585: 32850 of 41638 
stonewall rank 413: 37912 of 41638 
stonewall rank 333: 35488 of 41638 
stonewall rank 121: 28891 of 41638 
stonewall rank 455: 23267 of 41638 
stonewall rank 699: 38716 of 41638 
stonewall rank 61: 25009 of 41638 
stonewall rank 394: 37629 of 41638 
stonewall rank 69: 24092 of 41638 
stonewall rank 101: 39779 of 41638 
stonewall rank 663: 27019 of 41638 
stonewall rank 594: 32888 of 41638 
stonewall rank 412: 37612 of 41638 
stonewall rank 347: 24273 of 41638 
stonewall rank 434: 35600 of 41638 
stonewall rank 71: 28708 of 41638 
stonewall rank 41: 24877 of 41638 
stonewall rank 655: 38108 of 41638 
stonewall rank 578: 33265 of 41638 
stonewall rank 135: 29420 of 41638 
stonewall rank 308: 34843 of 41638 
stonewall rank 695: 27019 of 41638 
stonewall rank 87: 41191 of 41638 
stonewall rank 453: 36866 of 41638 
stonewall rank 66: 35927 of 41638 
stonewall rank 646: 38367 of 41638 
stonewall rank 392: 37111 of 41638 
stonewall rank 12: 25186 of 41638 
stonewall rank 590: 32901 of 41638 
stonewall rank 89: 29304 of 41638 
stonewall rank 334: 24617 of 41638 
stonewall rank 630: 36927 of 41638 
stonewall rank 430: 37323 of 41638 
stonewall rank 638: 38536 of 41638 
stonewall rank 28: 24483 of 41638 
stonewall rank 691: 38610 of 41638 
stonewall rank 4: 24507 of 41638 
stonewall rank 93: 41278 of 41638 
stonewall rank 575: 32982 of 41638 
stonewall rank 649: 26438 of 41638 
stonewall rank 85: 29447 of 41638 
stonewall rank 345: 36200 of 41638 
stonewall rank 125: 29132 of 41638 
stonewall rank 436: 36955 of 41638 
stonewall rank 9: 24075 of 41638 
stonewall rank 679: 26806 of 41638 
stonewall rank 626: 33064 of 41638 
stonewall rank 401: 37910 of 41638 
stonewall rank 423: 23097 of 41638 
stonewall rank 642: 26699 of 41638 
stonewall rank 571: 32440 of 41638 
stonewall rank 103: 28892 of 41638 
stonewall rank 422: 22769 of 41638 
stonewall rank 694: 27291 of 41638 
stonewall rank 8: 24203 of 41638 
stonewall rank 698: 26058 of 41638 
stonewall rank 74: 38800 of 41638 
stonewall rank 580: 32736 of 41638 
stonewall rank 652: 26485 of 41638 
stonewall rank 327: 24030 of 41638 
stonewall rank 470: 36765 of 41638 
stonewall rank 118: 40919 of 41638 
stonewall rank 138: 41576 of 41638 
stonewall rank 615: 33273 of 41638 
stonewall rank 332: 24357 of 41638 
stonewall rank 676: 38906 of 41638 
stonewall rank 690: 26641 of 41638 
stonewall rank 596: 32754 of 41638 
stonewall rank 692: 37664 of 41638 
stonewall rank 326: 34907 of 41638 
stonewall rank 426: 36141 of 41638 
stonewall rank 644: 27007 of 41638 
stonewall rank 602: 33061 of 41638 
stonewall rank 681: 38710 of 41638 
stonewall rank 282: 24441 of 41638 
stonewall rank 117: 40535 of 41638 
stonewall rank 659: 27075 of 41638 
stonewall rank 460: 22380 of 41638 
stonewall rank 40: 35480 of 41638 
stonewall rank 668: 27095 of 41638 
stonewall rank 566: 32963 of 41638 
stonewall rank 658: 38477 of 41638 
stonewall rank 408: 37864 of 41638 
stonewall rank 349: 36478 of 41638 
stonewall rank 416: 37604 of 41638 
stonewall rank 472: 23251 of 41638 
stonewall rank 343: 24273 of 41638 
stonewall rank 648: 37994 of 41638 
stonewall rank 483: 23212 of 41638 
stonewall rank 400: 37553 of 41638 
stonewall rank 318: 24201 of 41638 
stonewall rank 684: 26840 of 41638 
stonewall rank 457: 23061 of 41638 
stonewall rank 674: 38086 of 41638 
stonewall rank 612: 32798 of 41638 
stonewall rank 672: 26719 of 41638 
stonewall rank 286: 36135 of 41638 
stonewall rank 688: 38261 of 41638 
stonewall rank 661: 26945 of 41638 
stonewall rank 425: 22797 of 41638 
stonewall rank 560: 33112 of 41638 
stonewall rank 632: 37752 of 41638 
stonewall rank 297: 24669 of 41638 
stonewall rank 128: 29267 of 41638 
stonewall rank 452: 23673 of 41638 
stonewall rank 136: 28461 of 41638 
stonewall rank 337: 23999 of 41638 
stonewall rank 137: 28737 of 41638 
stonewall rank 458: 23210 of 41638 
stonewall rank 139: 28391 of 41638 
stonewall rank 70: 40019 of 41638 
stonewall rank 342: 24762 of 41638 
stonewall rank 465: 23261 of 41638 
stonewall rank 73: 28648 of 41638 
stonewall rank 329: 23988 of 41638 
stonewall rank 76: 28999 of 41638 
stonewall rank 456: 23137 of 41638 
stonewall rank 80: 39660 of 41638 
stonewall rank 338: 24698 of 41638 
stonewall rank 84: 28942 of 41638 
stonewall rank 86: 40706 of 41638 
stonewall rank 428: 23418 of 41638 
stonewall rank 317: 24072 of 41638 
stonewall rank 96: 40354 of 41638 
stonewall rank 108: 40296 of 41638 
stonewall rank 424: 23136 of 41638 
stonewall rank 114: 29333 of 41638 
stonewall rank 325: 23561 of 41638 
stonewall rank 116: 28803 of 41638 
stonewall rank 488: 37450 of 41638 
stonewall rank 316: 36110 of 41638 
stonewall rank 124: 29054 of 41638 
stonewall rank 72: 28907 of 41638 
stonewall rank 344: 35971 of 41638 
stonewall rank 324: 24300 of 41638 
stonewall rank 664: 26572 of 41638 
stonewall rank 656: 26885 of 41638 
stonewall rank 328: 24676 of 41638 
stonewall rank 320: 35825 of 41638 
stonewall rank 312: 35872 of 41638 
stonewall rank 608: 32802 of 41638 
stonewall rank 288: 23687 of 41638 
stonewall rank 624: 32571 of 41638 
stonewall rank 68: 24796 of 41638 
stonewall rank 64: 36079 of 41638 
stonewall rank 32: 35415 of 41638 
Continue stonewall hit min: 21402 max: 41638 avg: 37175.2 
stonewall rank 0: 35655 of 41638 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :      68656.689      68656.689      68656.689          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          1.697          1.697          1.697          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/14/2018 04:39:51 --
mdtest_hard_delete
-- started at 11/14/2018 05:11:41 --

mdtest-1.9.3 was launched with 700 total task(s) on 10 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-r" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_hard" "-n" "20000" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_hard-stonewall"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00
FS: 461.0 TiB   Used FS: 49.0%   Inodes: 5802.5 Mi   Used Inodes: 9.2%

700 tasks, 14000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :       3673.491       3673.491       3673.491          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          1.845          1.845          1.845          0.000

-- finished at 11/14/2018 05:17:17 --
mdtest_hard_read
-- started at 11/14/2018 05:11:36 --

mdtest-1.9.3 was launched with 700 total task(s) on 10 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-E" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_hard" "-n" "20000" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_hard-stonewall"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00
FS: 461.0 TiB   Used FS: 49.0%   Inodes: 5802.5 Mi   Used Inodes: 9.2%

700 tasks, 14000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :     745446.482     745446.482     745446.482          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/14/2018 05:11:38 --
mdtest_hard_stat
-- started at 11/14/2018 04:55:57 --

mdtest-1.9.3 was launched with 700 total task(s) on 10 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-T" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_hard" "-n" "20000" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_hard-stonewall"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00
FS: 461.0 TiB   Used FS: 49.0%   Inodes: 5802.5 Mi   Used Inodes: 9.8%

700 tasks, 14000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :    2212209.779    2212209.779    2212209.779          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/14/2018 04:55:58 --
mdtest_hard_write
-- started at 11/14/2018 04:46:09 --

mdtest-1.9.3 was launched with 700 total task(s) on 10 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-C" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_hard" "-n" "20000" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00/mdt_hard-stonewall" "-W" "300"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.14-04.25.00
FS: 461.0 TiB   Used FS: 49.0%   Inodes: 5802.5 Mi   Used Inodes: 9.8%

700 tasks, 14000000 files
stonewall rank 491: 1705 of 1762 
stonewall rank 94: 1703 of 1762 
stonewall rank 63: 1750 of 1762 
stonewall rank 503: 1704 of 1762 
stonewall rank 95: 1709 of 1762 
stonewall rank 31: 1750 of 1762 
stonewall rank 539: 1707 of 1762 
stonewall rank 255: 1712 of 1762 
stonewall rank 62: 1749 of 1762 
stonewall rank 505: 1705 of 1762 
stonewall rank 246: 1709 of 1762 
stonewall rank 123: 1703 of 1762 
stonewall rank 30: 1748 of 1762 
stonewall rank 537: 1703 of 1762 
stonewall rank 247: 1707 of 1762 
stonewall rank 631: 1748 of 1762 
stonewall rank 119: 1704 of 1762 
stonewall rank 59: 1748 of 1762 
stonewall rank 551: 1702 of 1762 
stonewall rank 251: 1704 of 1762 
stonewall rank 279: 1712 of 1762 
stonewall rank 667: 1750 of 1762 
stonewall rank 58: 1747 of 1762 
stonewall rank 535: 1705 of 1762 
stonewall rank 87: 1706 of 1762 
stonewall rank 567: 1749 of 1762 
stonewall rank 547: 1705 of 1762 
stonewall rank 278: 1708 of 1762 
stonewall rank 571: 1753 of 1762 
stonewall rank 499: 1704 of 1762 
stonewall rank 215: 1706 of 1762 
stonewall rank 86: 1708 of 1762 
stonewall rank 314: 1709 of 1762 
stonewall rank 543: 1706 of 1762 
stonewall rank 254: 1709 of 1762 
stonewall rank 695: 1746 of 1762 
stonewall rank 514: 1705 of 1762 
stonewall rank 241: 1706 of 1762 
stonewall rank 351: 1677 of 1762 
stonewall rank 587: 1754 of 1762 
stonewall rank 315: 1704 of 1762 
stonewall rank 515: 1706 of 1762 
stonewall rank 250: 1715 of 1762 
stonewall rank 699: 1748 of 1762 
stonewall rank 459: 1710 of 1762 
stonewall rank 523: 1700 of 1762 
stonewall rank 271: 1707 of 1762 
stonewall rank 375: 1680 of 1762 
stonewall rank 603: 1760 of 1762 
stonewall rank 495: 1702 of 1762 
stonewall rank 237: 1704 of 1762 
stonewall rank 633: 1748 of 1762 
stonewall rank 319: 1704 of 1762 
stonewall rank 531: 1708 of 1762 
stonewall rank 243: 1707 of 1762 
stonewall rank 407: 1678 of 1762 
stonewall rank 475: 1709 of 1762 
stonewall rank 498: 1704 of 1762 
stonewall rank 275: 1705 of 1762 
stonewall rank 665: 1748 of 1762 
stonewall rank 619: 1749 of 1762 
stonewall rank 546: 1705 of 1762 
stonewall rank 219: 1712 of 1762 
stonewall rank 697: 1753 of 1762 
stonewall rank 287: 1705 of 1762 
stonewall rank 519: 1704 of 1762 
stonewall rank 245: 1706 of 1762 
stonewall rank 679: 1746 of 1762 
stonewall rank 187: 1700 of 1762 
stonewall rank 511: 1704 of 1762 
stonewall rank 274: 1712 of 1762 
stonewall rank 118: 1699 of 1762 
stonewall rank 22: 1750 of 1762 
stonewall rank 501: 1704 of 1762 
stonewall rank 214: 1707 of 1762 
stonewall rank 663: 1747 of 1762 
stonewall rank 218: 1708 of 1762 
stonewall rank 122: 1705 of 1762 
stonewall rank 23: 1745 of 1762 
stonewall rank 533: 1708 of 1762 
stonewall rank 601: 1753 of 1762 
stonewall rank 549: 1705 of 1762 
stonewall rank 244: 1710 of 1762 
stonewall rank 350: 1680 of 1762 
stonewall rank 318: 1704 of 1762 
stonewall rank 517: 1703 of 1762 
stonewall rank 249: 1705 of 1762 
stonewall rank 377: 1686 of 1762 
stonewall rank 675: 1753 of 1762 
stonewall rank 473: 1709 of 1762 
stonewall rank 555: 1706 of 1762 
stonewall rank 263: 1708 of 1762 
stonewall rank 569: 1757 of 1762 
stonewall rank 493: 1706 of 1762 
stonewall rank 269: 1709 of 1762 
stonewall rank 111: 1702 of 1762 
stonewall rank 286: 1706 of 1762 
stonewall rank 497: 1710 of 1762 
stonewall rank 231: 1705 of 1762 
stonewall rank 79: 1706 of 1762 
stonewall rank 233: 1704 of 1762 
stonewall rank 110: 1703 of 1762 
stonewall rank 443: 1712 of 1762 
stonewall rank 521: 1705 of 1762 
stonewall rank 71: 1704 of 1762 
stonewall rank 615: 1754 of 1762 
stonewall rank 502: 1703 of 1762 
stonewall rank 223: 1708 of 1762 
stonewall rank 55: 1745 of 1762 
stonewall rank 548: 1706 of 1762 
stonewall rank 234: 1710 of 1762 
stonewall rank 671: 1752 of 1762 
stonewall rank 441: 1706 of 1762 
stonewall rank 550: 1703 of 1762 
stonewall rank 235: 1707 of 1762 
stonewall rank 78: 1706 of 1762 
stonewall rank 211: 1705 of 1762 
stonewall rank 77: 1706 of 1762 
stonewall rank 343: 1710 of 1762 
stonewall rank 553: 1701 of 1762 
stonewall rank 183: 1705 of 1762 
stonewall rank 507: 1708 of 1762 
stonewall rank 257: 1707 of 1762 
stonewall rank 642: 1750 of 1762 
stonewall rank 599: 1759 of 1762 
stonewall rank 490: 1706 of 1762 
stonewall rank 222: 1706 of 1762 
stonewall rank 411: 1682 of 1762 
stonewall rank 559: 1708 of 1762 
stonewall rank 270: 1706 of 1762 
stonewall rank 76: 1712 of 1762 
stonewall rank 487: 1713 of 1762 
stonewall rank 627: 1759 of 1762 
stonewall rank 525: 1710 of 1762 
stonewall rank 273: 1712 of 1762 
stonewall rank 643: 1745 of 1762 
stonewall rank 191: 1702 of 1762 
stonewall rank 527: 1708 of 1762 
stonewall rank 225: 1713 of 1762 
stonewall rank 415: 1680 of 1762 
stonewall rank 611: 1750 of 1762 
stonewall rank 534: 1704 of 1762 
stonewall rank 230: 1705 of 1762 
stonewall rank 70: 1707 of 1762 
stonewall rank 190: 1701 of 1762 
stonewall rank 516: 1707 of 1762 
stonewall rank 268: 1702 of 1762 
stonewall rank 103: 1706 of 1762 
stonewall rank 455: 1712 of 1762 
stonewall rank 518: 1703 of 1762 
stonewall rank 210: 1709 of 1762 
stonewall rank 651: 1754 of 1762 
stonewall rank 541: 1704 of 1762 
stonewall rank 229: 1708 of 1762 
stonewall rank 419: 1683 of 1762 
stonewall rank 15: 1749 of 1762 
stonewall rank 500: 1708 of 1762 
stonewall rank 239: 1707 of 1762 
stonewall rank 102: 1705 of 1762 
stonewall rank 607: 1757 of 1762 
stonewall rank 522: 1705 of 1762 
stonewall rank 213: 1706 of 1762 
stonewall rank 659: 1749 of 1762 
stonewall rank 335: 1703 of 1762 
stonewall rank 557: 1705 of 1762 
stonewall rank 228: 1708 of 1762 
stonewall rank 386: 1678 of 1762 
stonewall rank 423: 1709 of 1762 
stonewall rank 49: 1746 of 1762 
stonewall rank 558: 1711 of 1762 
stonewall rank 242: 1706 of 1762 
stonewall rank 135: 1709 of 1762 
stonewall rank 109: 1705 of 1762 
stonewall rank 14: 1744 of 1762 
stonewall rank 556: 1705 of 1762 
stonewall rank 217: 1715 of 1762 
stonewall rank 579: 1759 of 1762 
stonewall rank 494: 1705 of 1762 
stonewall rank 253: 1705 of 1762 
stonewall rank 674: 1754 of 1762 
stonewall rank 439: 1710 of 1762 
stonewall rank 513: 1706 of 1762 
stonewall rank 240: 1713 of 1762 
stonewall rank 387: 1684 of 1762 
stonewall rank 47: 1747 of 1762 
stonewall rank 554: 1706 of 1762 
stonewall rank 248: 1706 of 1762 
stonewall rank 129: 1704 of 1762 
stonewall rank 252: 1709 of 1762 
stonewall rank 106: 1704 of 1762 
stonewall rank 610: 1755 of 1762 
stonewall rank 492: 1704 of 1762 
stonewall rank 471: 1711 of 1762 
stonewall rank 506: 1701 of 1762 
stonewall rank 262: 1708 of 1762 
stonewall rank 107: 1707 of 1762 
stonewall rank 108: 1706 of 1762 
stonewall rank 46: 1746 of 1762 
stonewall rank 509: 1703 of 1762 
stonewall rank 261: 1709 of 1762 
stonewall rank 1: 1741 of 1762 
stonewall rank 538: 1707 of 1762 
stonewall rank 265: 1704 of 1762 
stonewall rank 639: 1752 of 1762 
stonewall rank 578: 1755 of 1762 
stonewall rank 552: 1706 of 1762 
stonewall rank 277: 1709 of 1762 
stonewall rank 365: 1677 of 1762 
stonewall rank 483: 1708 of 1762 
stonewall rank 529: 1703 of 1762 
stonewall rank 259: 1710 of 1762 
stonewall rank 83: 1709 of 1762 
stonewall rank 51: 1746 of 1762 
stonewall rank 542: 1707 of 1762 
stonewall rank 221: 1704 of 1762 
stonewall rank 85: 1706 of 1762 
stonewall rank 267: 1708 of 1762 
stonewall rank 647: 1750 of 1762 
stonewall rank 6: 1757 of 1762 
stonewall rank 530: 1702 of 1762 
stonewall rank 7: 1748 of 1762 
stonewall rank 545: 1704 of 1762 
stonewall rank 266: 1705 of 1762 
stonewall rank 409: 1680 of 1762 
stonewall rank 508: 1706 of 1762 
stonewall rank 227: 1709 of 1762 
stonewall rank 97: 1709 of 1762 
stonewall rank 33: 1750 of 1762 
stonewall rank 623: 1753 of 1762 
stonewall rank 510: 1706 of 1762 
stonewall rank 264: 1707 of 1762 
stonewall rank 134: 1707 of 1762 
stonewall rank 311: 1711 of 1762 
stonewall rank 532: 1703 of 1762 
stonewall rank 226: 1707 of 1762 
stonewall rank 374: 1680 of 1762 
stonewall rank 238: 1709 of 1762 
stonewall rank 81: 1703 of 1762 
stonewall rank 207: 1703 of 1762 
stonewall rank 526: 1705 of 1762 
stonewall rank 17: 1750 of 1762 
stonewall rank 496: 1710 of 1762 
stonewall rank 276: 1706 of 1762 
stonewall rank 82: 1709 of 1762 
stonewall rank 18: 1749 of 1762 
stonewall rank 504: 1708 of 1762 
stonewall rank 212: 1708 of 1762 
stonewall rank 115: 1710 of 1762 
stonewall rank 258: 1709 of 1762 
stonewall rank 371: 1682 of 1762 
stonewall rank 19: 1746 of 1762 
stonewall rank 536: 1702 of 1762 
stonewall rank 27: 1746 of 1762 
stonewall rank 540: 1708 of 1762 
stonewall rank 272: 1702 of 1762 
stonewall rank 99: 1704 of 1762 
stonewall rank 50: 1748 of 1762 
stonewall rank 544: 1704 of 1762 
stonewall rank 220: 1710 of 1762 
stonewall rank 677: 1754 of 1762 
stonewall rank 65: 1749 of 1762 
stonewall rank 520: 1700 of 1762 
stonewall rank 260: 1711 of 1762 
stonewall rank 353: 1682 of 1762 
stonewall rank 626: 1758 of 1762 
stonewall rank 524: 1708 of 1762 
stonewall rank 216: 1705 of 1762 
stonewall rank 73: 1707 of 1762 
stonewall rank 321: 1707 of 1762 
stonewall rank 528: 1706 of 1762 
stonewall rank 232: 1710 of 1762 
stonewall rank 98: 1708 of 1762 
stonewall rank 141: 1701 of 1762 
stonewall rank 512: 1704 of 1762 
stonewall rank 236: 1710 of 1762 
stonewall rank 105: 1706 of 1762 
stonewall rank 9: 1748 of 1762 
stonewall rank 224: 1711 of 1762 
stonewall rank 114: 1705 of 1762 
stonewall rank 393: 1684 of 1762 
stonewall rank 26: 1749 of 1762 
stonewall rank 256: 1711 of 1762 
stonewall rank 137: 1709 of 1762 
stonewall rank 38: 1747 of 1762 
stonewall rank 39: 1748 of 1762 
stonewall rank 391: 1680 of 1762 
stonewall rank 44: 1744 of 1762 
stonewall rank 683: 1754 of 1762 
stonewall rank 45: 1746 of 1762 
stonewall rank 117: 1713 of 1762 
stonewall rank 57: 1748 of 1762 
stonewall rank 693: 1748 of 1762 
stonewall rank 91: 1710 of 1762 
stonewall rank 575: 1760 of 1762 
stonewall rank 337: 1702 of 1762 
stonewall rank 113: 1710 of 1762 
stonewall rank 355: 1680 of 1762 
stonewall rank 186: 1703 of 1762 
stonewall rank 13: 1748 of 1762 
stonewall rank 121: 1707 of 1762 
stonewall rank 20: 1751 of 1762 
stonewall rank 395: 1678 of 1762 
stonewall rank 21: 1750 of 1762 
stonewall rank 84: 1707 of 1762 
stonewall rank 52: 1751 of 1762 
stonewall rank 116: 1706 of 1762 
stonewall rank 53: 1742 of 1762 
stonewall rank 689: 1749 of 1762 
stonewall rank 54: 1751 of 1762 
stonewall rank 93: 1709 of 1762 
stonewall rank 101: 1710 of 1762 
stonewall rank 66: 1748 of 1762 
stonewall rank 139: 1705 of 1762 
stonewall rank 67: 1748 of 1762 
stonewall rank 597: 1759 of 1762 
stonewall rank 383: 1684 of 1762 
stonewall rank 327: 1707 of 1762 
stonewall rank 127: 1707 of 1762 
stonewall rank 182: 1705 of 1762 
stonewall rank 89: 1707 of 1762 
stonewall rank 10: 1752 of 1762 
stonewall rank 90: 1710 of 1762 
stonewall rank 11: 1753 of 1762 
stonewall rank 92: 1705 of 1762 
stonewall rank 12: 1751 of 1762 
stonewall rank 645: 1748 of 1762 
stonewall rank 366: 1681 of 1762 
stonewall rank 28: 1750 of 1762 
stonewall rank 100: 1704 of 1762 
stonewall rank 29: 1750 of 1762 
stonewall rank 35: 1755 of 1762 
stonewall rank 126: 1706 of 1762 
stonewall rank 136: 1706 of 1762 
stonewall rank 42: 1749 of 1762 
stonewall rank 43: 1745 of 1762 
stonewall rank 138: 1707 of 1762 
stonewall rank 88: 1706 of 1762 
stonewall rank 8: 1742 of 1762 
stonewall rank 131: 1705 of 1762 
stonewall rank 613: 1754 of 1762 
stonewall rank 334: 1706 of 1762 
stonewall rank 133: 1704 of 1762 
stonewall rank 479: 1709 of 1762 
stonewall rank 367: 1684 of 1762 
stonewall rank 2: 1756 of 1762 
stonewall rank 125: 1706 of 1762 
stonewall rank 130: 1707 of 1762 
stonewall rank 3: 1754 of 1762 
stonewall rank 4: 1746 of 1762 
stonewall rank 104: 1707 of 1762 
stonewall rank 5: 1743 of 1762 
stonewall rank 649: 1746 of 1762 
stonewall rank 389: 1678 of 1762 
stonewall rank 24: 1749 of 1762 
stonewall rank 75: 1712 of 1762 
stonewall rank 25: 1754 of 1762 
stonewall rank 34: 1752 of 1762 
stonewall rank 397: 1679 of 1762 
stonewall rank 36: 1753 of 1762 
stonewall rank 132: 1708 of 1762 
stonewall rank 37: 1752 of 1762 
stonewall rank 74: 1705 of 1762 
stonewall rank 40: 1744 of 1762 
stonewall rank 406: 1675 of 1762 
stonewall rank 41: 1748 of 1762 
stonewall rank 80: 1701 of 1762 
stonewall rank 48: 1751 of 1762 
stonewall rank 661: 1746 of 1762 
stonewall rank 361: 1681 of 1762 
stonewall rank 56: 1746 of 1762 
stonewall rank 60: 1747 of 1762 
stonewall rank 72: 1709 of 1762 
stonewall rank 61: 1748 of 1762 
stonewall rank 124: 1706 of 1762 
stonewall rank 68: 1745 of 1762 
stonewall rank 96: 1712 of 1762 
stonewall rank 69: 1747 of 1762 
stonewall rank 112: 1706 of 1762 
stonewall rank 621: 1756 of 1762 
stonewall rank 120: 1708 of 1762 
stonewall rank 678: 1749 of 1762 
stonewall rank 341: 1706 of 1762 
stonewall rank 451: 1704 of 1762 
stonewall rank 363: 1681 of 1762 
stonewall rank 16: 1746 of 1762 
stonewall rank 630: 1744 of 1762 
stonewall rank 32: 1748 of 1762 
stonewall rank 364: 1678 of 1762 
stonewall rank 629: 1759 of 1762 
stonewall rank 635: 1753 of 1762 
stonewall rank 362: 1680 of 1762 
stonewall rank 342: 1703 of 1762 
stonewall rank 482: 1710 of 1762 
stonewall rank 676: 1754 of 1762 
stonewall rank 370: 1679 of 1762 
stonewall rank 64: 1747 of 1762 
stonewall rank 681: 1749 of 1762 
stonewall rank 326: 1705 of 1762 
stonewall rank 692: 1750 of 1762 
stonewall rank 467: 1707 of 1762 
stonewall rank 305: 1704 of 1762 
stonewall rank 373: 1681 of 1762 
stonewall rank 694: 1755 of 1762 
stonewall rank 205: 1704 of 1762 
stonewall rank 565: 1758 of 1762 
stonewall rank 379: 1679 of 1762 
stonewall rank 653: 1750 of 1762 
stonewall rank 333: 1704 of 1762 
stonewall rank 447: 1710 of 1762 
stonewall rank 662: 1752 of 1762 
stonewall rank 614: 1757 of 1762 
stonewall rank 128: 1709 of 1762 
stonewall rank 687: 1753 of 1762 
stonewall rank 147: 1699 of 1762 
stonewall rank 625: 1752 of 1762 
stonewall rank 394: 1679 of 1762 
stonewall rank 469: 1710 of 1762 
stonewall rank 646: 1750 of 1762 
stonewall rank 655: 1751 of 1762 
stonewall rank 561: 1758 of 1762 
stonewall rank 206: 1700 of 1762 
stonewall rank 418: 1681 of 1762 
stonewall rank 357: 1683 of 1762 
stonewall rank 581: 1754 of 1762 
stonewall rank 313: 1705 of 1762 
stonewall rank 658: 1746 of 1762 
stonewall rank 485: 1708 of 1762 
stonewall rank 669: 1746 of 1762 
stonewall rank 585: 1752 of 1762 
stonewall rank 378: 1677 of 1762 
stonewall rank 340: 1703 of 1762 
stonewall rank 644: 1748 of 1762 
stonewall rank 143: 1699 of 1762 
stonewall rank 413: 1685 of 1762 
stonewall rank 612: 1758 of 1762 
stonewall rank 650: 1755 of 1762 
stonewall rank 347: 1706 of 1762 
stonewall rank 372: 1681 of 1762 
stonewall rank 145: 1700 of 1762 
stonewall rank 686: 1753 of 1762 
stonewall rank 618: 1755 of 1762 
stonewall rank 369: 1682 of 1762 
stonewall rank 309: 1705 of 1762 
stonewall rank 690: 1751 of 1762 
stonewall rank 453: 1709 of 1762 
stonewall rank 401: 1683 of 1762 
stonewall rank 691: 1750 of 1762 
stonewall rank 564: 1754 of 1762 
stonewall rank 339: 1710 of 1762 
stonewall rank 410: 1677 of 1762 
stonewall rank 204: 1699 of 1762 
stonewall rank 660: 1754 of 1762 
Continue stonewall hit min: 1675 max: 1762 avg: 1717.1 
stonewall rank 0: 1746 of 1762 
stonewall rank 414: 1680 of 1762 
stonewall rank 566: 1752 of 1762 
stonewall rank 666: 1751 of 1762 
stonewall rank 289: 1703 of 1762 
stonewall rank 382: 1677 of 1762 
stonewall rank 682: 1746 of 1762 
stonewall rank 155: 1702 of 1762 
stonewall rank 591: 1756 of 1762 
stonewall rank 360: 1679 of 1762 
stonewall rank 284: 1703 of 1762 
stonewall rank 684: 1750 of 1762 
stonewall rank 450: 1709 of 1762 
stonewall rank 685: 1751 of 1762 
stonewall rank 596: 1756 of 1762 
stonewall rank 696: 1754 of 1762 
stonewall rank 285: 1705 of 1762 
stonewall rank 698: 1751 of 1762 
stonewall rank 402: 1680 of 1762 
stonewall rank 199: 1704 of 1762 
stonewall rank 634: 1751 of 1762 
stonewall rank 598: 1753 of 1762 
stonewall rank 403: 1677 of 1762 
stonewall rank 346: 1708 of 1762 
stonewall rank 641: 1752 of 1762 
stonewall rank 486: 1708 of 1762 
stonewall rank 617: 1755 of 1762 
stonewall rank 412: 1683 of 1762 
stonewall rank 283: 1707 of 1762 
stonewall rank 638: 1751 of 1762 
stonewall rank 358: 1679 of 1762 
stonewall rank 573: 1751 of 1762 
stonewall rank 656: 1757 of 1762 
stonewall rank 307: 1709 of 1762 
stonewall rank 580: 1760 of 1762 
stonewall rank 359: 1680 of 1762 
stonewall rank 657: 1743 of 1762 
stonewall rank 310: 1710 of 1762 
stonewall rank 582: 1752 of 1762 
stonewall rank 398: 1679 of 1762 
stonewall rank 664: 1751 of 1762 
stonewall rank 332: 1706 of 1762 
stonewall rank 399: 1678 of 1762 
stonewall rank 158: 1700 of 1762 
stonewall rank 594: 1745 of 1762 
stonewall rank 668: 1752 of 1762 
stonewall rank 295: 1707 of 1762 
stonewall rank 408: 1685 of 1762 
stonewall rank 159: 1704 of 1762 
stonewall rank 670: 1751 of 1762 
stonewall rank 595: 1760 of 1762 
stonewall rank 354: 1679 of 1762 
stonewall rank 291: 1706 of 1762 
stonewall rank 672: 1750 of 1762 
stonewall rank 201: 1701 of 1762 
stonewall rank 396: 1680 of 1762 
stonewall rank 605: 1759 of 1762 
stonewall rank 673: 1747 of 1762 
stonewall rank 297: 1703 of 1762 
stonewall rank 390: 1685 of 1762 
stonewall rank 680: 1750 of 1762 
stonewall rank 175: 1702 of 1762 
stonewall rank 562: 1755 of 1762 
stonewall rank 688: 1758 of 1762 
stonewall rank 301: 1710 of 1762 
stonewall rank 637: 1750 of 1762 
stonewall rank 209: 1705 of 1762 
stonewall rank 648: 1748 of 1762 
stonewall rank 652: 1745 of 1762 
stonewall rank 563: 1757 of 1762 
stonewall rank 345: 1711 of 1762 
stonewall rank 388: 1682 of 1762 
stonewall rank 457: 1711 of 1762 
stonewall rank 654: 1750 of 1762 
stonewall rank 577: 1756 of 1762 
stonewall rank 405: 1678 of 1762 
stonewall rank 349: 1704 of 1762 
stonewall rank 632: 1752 of 1762 
stonewall rank 380: 1680 of 1762 
stonewall rank 149: 1701 of 1762 
stonewall rank 586: 1756 of 1762 
stonewall rank 636: 1752 of 1762 
stonewall rank 294: 1709 of 1762 
stonewall rank 381: 1678 of 1762 
stonewall rank 151: 1702 of 1762 
stonewall rank 640: 1746 of 1762 
stonewall rank 404: 1684 of 1762 
stonewall rank 593: 1756 of 1762 
stonewall rank 329: 1704 of 1762 
stonewall rank 417: 1681 of 1762 
stonewall rank 181: 1702 of 1762 
stonewall rank 356: 1680 of 1762 
stonewall rank 622: 1752 of 1762 
stonewall rank 368: 1680 of 1762 
stonewall rank 331: 1702 of 1762 
stonewall rank 376: 1679 of 1762 
stonewall rank 385: 1680 of 1762 
stonewall rank 463: 1716 of 1762 
stonewall rank 628: 1760 of 1762 
stonewall rank 392: 1681 of 1762 
stonewall rank 293: 1708 of 1762 
stonewall rank 416: 1676 of 1762 
stonewall rank 167: 1698 of 1762 
stonewall rank 400: 1685 of 1762 
stonewall rank 352: 1683 of 1762 
stonewall rank 589: 1761 of 1762 
stonewall rank 303: 1710 of 1762 
stonewall rank 384: 1680 of 1762 
stonewall rank 180: 1700 of 1762 
stonewall rank 616: 1757 of 1762 
stonewall rank 316: 1706 of 1762 
stonewall rank 466: 1708 of 1762 
stonewall rank 620: 1756 of 1762 
stonewall rank 317: 1706 of 1762 
stonewall rank 157: 1700 of 1762 
stonewall rank 570: 1755 of 1762 
stonewall rank 312: 1709 of 1762 
stonewall rank 161: 1698 of 1762 
stonewall rank 602: 1752 of 1762 
stonewall rank 323: 1704 of 1762 
stonewall rank 166: 1707 of 1762 
stonewall rank 606: 1757 of 1762 
stonewall rank 330: 1705 of 1762 
stonewall rank 177: 1700 of 1762 
stonewall rank 572: 1752 of 1762 
stonewall rank 282: 1707 of 1762 
stonewall rank 179: 1702 of 1762 
stonewall rank 574: 1755 of 1762 
stonewall rank 292: 1704 of 1762 
stonewall rank 193: 1705 of 1762 
stonewall rank 604: 1757 of 1762 
stonewall rank 308: 1709 of 1762 
stonewall rank 422: 1709 of 1762 
stonewall rank 609: 1756 of 1762 
stonewall rank 328: 1707 of 1762 
stonewall rank 198: 1700 of 1762 
stonewall rank 568: 1757 of 1762 
stonewall rank 299: 1712 of 1762 
stonewall rank 146: 1699 of 1762 
stonewall rank 298: 1707 of 1762 
stonewall rank 154: 1706 of 1762 
stonewall rank 584: 1749 of 1762 
stonewall rank 588: 1755 of 1762 
stonewall rank 338: 1710 of 1762 
stonewall rank 156: 1700 of 1762 
stonewall rank 590: 1752 of 1762 
stonewall rank 281: 1708 of 1762 
stonewall rank 489: 1705 of 1762 
stonewall rank 592: 1755 of 1762 
stonewall rank 290: 1708 of 1762 
stonewall rank 140: 1697 of 1762 
stonewall rank 600: 1758 of 1762 
stonewall rank 306: 1703 of 1762 
stonewall rank 142: 1699 of 1762 
stonewall rank 560: 1754 of 1762 
stonewall rank 280: 1707 of 1762 
stonewall rank 169: 1703 of 1762 
stonewall rank 624: 1758 of 1762 
stonewall rank 304: 1705 of 1762 
stonewall rank 427: 1709 of 1762 
stonewall rank 576: 1753 of 1762 
stonewall rank 325: 1711 of 1762 
stonewall rank 163: 1703 of 1762 
stonewall rank 608: 1754 of 1762 
stonewall rank 344: 1708 of 1762 
stonewall rank 421: 1710 of 1762 
stonewall rank 348: 1710 of 1762 
stonewall rank 162: 1701 of 1762 
stonewall rank 302: 1708 of 1762 
stonewall rank 173: 1702 of 1762 
stonewall rank 324: 1705 of 1762 
stonewall rank 438: 1710 of 1762 
stonewall rank 336: 1707 of 1762 
stonewall rank 172: 1710 of 1762 
stonewall rank 300: 1704 of 1762 
stonewall rank 174: 1700 of 1762 
stonewall rank 322: 1705 of 1762 
stonewall rank 194: 1702 of 1762 
stonewall rank 296: 1705 of 1762 
stonewall rank 195: 1701 of 1762 
stonewall rank 288: 1705 of 1762 
stonewall rank 470: 1714 of 1762 
stonewall rank 320: 1708 of 1762 
stonewall rank 150: 1703 of 1762 
stonewall rank 165: 1702 of 1762 
stonewall rank 185: 1701 of 1762 
stonewall rank 454: 1712 of 1762 
stonewall rank 148: 1700 of 1762 
stonewall rank 164: 1698 of 1762 
stonewall rank 468: 1706 of 1762 
stonewall rank 178: 1705 of 1762 
stonewall rank 197: 1702 of 1762 
stonewall rank 477: 1709 of 1762 
stonewall rank 189: 1700 of 1762 
stonewall rank 484: 1710 of 1762 
stonewall rank 184: 1704 of 1762 
stonewall rank 188: 1699 of 1762 
stonewall rank 203: 1700 of 1762 
stonewall rank 433: 1706 of 1762 
stonewall rank 152: 1700 of 1762 
stonewall rank 153: 1703 of 1762 
stonewall rank 171: 1702 of 1762 
stonewall rank 176: 1701 of 1762 
stonewall rank 144: 1700 of 1762 
stonewall rank 196: 1700 of 1762 
stonewall rank 170: 1701 of 1762 
stonewall rank 437: 1707 of 1762 
stonewall rank 202: 1703 of 1762 
stonewall rank 445: 1709 of 1762 
stonewall rank 200: 1702 of 1762 
stonewall rank 452: 1713 of 1762 
stonewall rank 168: 1696 of 1762 
stonewall rank 458: 1707 of 1762 
stonewall rank 208: 1698 of 1762 
stonewall rank 160: 1698 of 1762 
stonewall rank 425: 1712 of 1762 
stonewall rank 192: 1702 of 1762 
stonewall rank 465: 1710 of 1762 
stonewall rank 449: 1712 of 1762 
stonewall rank 431: 1709 of 1762 
stonewall rank 461: 1707 of 1762 
stonewall rank 474: 1714 of 1762 
stonewall rank 435: 1707 of 1762 
stonewall rank 442: 1711 of 1762 
stonewall rank 476: 1707 of 1762 
stonewall rank 478: 1712 of 1762 
stonewall rank 420: 1707 of 1762 
stonewall rank 429: 1707 of 1762 
stonewall rank 436: 1709 of 1762 
stonewall rank 426: 1707 of 1762 
stonewall rank 444: 1708 of 1762 
stonewall rank 446: 1708 of 1762 
stonewall rank 462: 1710 of 1762 
stonewall rank 472: 1709 of 1762 
stonewall rank 481: 1707 of 1762 
stonewall rank 430: 1704 of 1762 
stonewall rank 440: 1712 of 1762 
stonewall rank 460: 1708 of 1762 
stonewall rank 464: 1712 of 1762 
stonewall rank 424: 1706 of 1762 
stonewall rank 428: 1706 of 1762 
stonewall rank 434: 1705 of 1762 
stonewall rank 456: 1708 of 1762 
stonewall rank 488: 1706 of 1762 
stonewall rank 432: 1708 of 1762 
stonewall rank 448: 1710 of 1762 
stonewall rank 480: 1711 of 1762 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :      44703.144      44703.144      44703.144          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :        232.794        232.794        232.794          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/14/2018 04:51:22 --
result_summary
[RESULT] BW   phase 1            ior_easy_write               31.127 GB/s : time 461.03 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               68.657 kiops : time 427.68 seconds
[RESULT] BW   phase 2            ior_hard_write                4.864 GB/s : time 373.04 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               44.703 kiops : time 315.60 seconds
[RESULT] IOPS phase 3                      find              473.600 kiops : time  64.15 seconds
[RESULT] BW   phase 3             ior_easy_read              107.755 GB/s : time 133.18 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             2525.100 kiops : time  14.02 seconds
[RESULT] BW   phase 4             ior_hard_read               32.802 GB/s : time  55.32 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat             2212.210 kiops : time   3.21 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               31.269 kiops : time 935.90 seconds
[RESULT] IOPS phase 7          mdtest_hard_read              745.446 kiops : time   4.42 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                3.673 kiops : time 339.17 seconds
[SCORE] Bandwidth 27.0472 GB/s : IOPS 169.929 kiops : TOTAL 67.7946