WekaIO

Institution WekaIO
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.6.1810
Client Kernel Version 3.10.0-957.el7.x86_64

DATA SERVER

Storage Type NVMe
Volatile Memory 352.0 GB
Storage Interface NVMe
Network Infiniband HDR
Software Version 3.5
OS Version CentOS 7.6.1810

INFORMATION

Client Nodes 10
Client Total Procs 2,610
Metadata Nodes 26
Metadata Storage Devices 6
Data Nodes 26
Data Storage Devices 156

METADATA

Easy Write 789.96 kIOP/s
Easy Stat 4,155.24 kIOP/s
Easy Delete 921.60 kIOP/s
Hard Write 112.61 kIOP/s
Hard Read 489.30 kIOP/s
Hard Stat 32.50 kIOP/s
Hard Delete 35.05 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="--allow-run-as-root --oversubscribe -n 2616 -H 10.0.11.64,10.0.8.63,10.0.4.65,10.0.4.68,10.0.11.65,10.0.11.69,10.0.6.66,10.0.9.68,10.0.11.67,10.0.2.64,10.0.11.68,10.0.3.62,10.0.6.61,10.0.11.66,10.0.8.69,10.0.6.65,10.0.9.62,10.0.9.65,10.0.6.70,10.0.6.60,10.0.3.69,10.0.6.64,10.0.4.61,10.0.4.66,10.0.8.67,10.0.6.62,10.0.7.62,10.0.10.61,10.0.7.63,10.0.10.64,10.0.7.69,10.0.7.64,10.0.7.70,10.0.3.68,10.0.3.61,10.0.9.66,10.0.10.65,10.0.11.62,10.0.3.67,10.0.8.61,10.0.9.61,10.0.2.69,10.0.7.60,10.0.2.63,10.0.3.64,10.0.10.63,10.0.7.61,10.0.3.70,10.0.1.63,10.0.11.63,10.0.1.68,10.0.2.68,10.0.2.67,10.0.2.61,10.0.6.63,10.0.1.62,10.0.9.64,10.0.8.70,10.0.7.65,10.0.1.60,10.0.4.63,10.0.2.66,10.0.3.65,10.0.10.68,10.0.10.70,10.0.10.62,10.0.4.60,10.0.2.62,10.0.1.64,10.0.4.62,10.0.9.70,10.0.8.68,10.0.3.60,10.0.4.67,10.0.8.66,10.0.2.60,10.0.9.63,10.0.8.62,10.0.3.66,10.0.6.68,10.0.11.70,10.0.9.60,10.0.11.60,10.0.8.65,10.0.6.69,10.0.11.61,10.0.10.67,10.0.6.67,10.0.3.63,10.0.4.70,10.0.10.60,10.0.8.64,10.0.1.69,10.0.9.67,10.0.8.60,10.0.4.64,10.0.2.70,10.0.9.69,10.0.10.66,10.0.7.66,10.0.2.65,10.0.1.67,10.0.7.68,10.0.7.67,10.0.1.65,10.0.4.69,10.0.10.69,10.0.1.66,10.0.1.61 --mca pml ob1 --mca btl self,tcp --mca btl_tcp_if_include 10.0.0.0/16 --mca routed direct"
}

function setup_ior_easy {
  io500_ior_easy_size="11000"
  io500_ior_easy_params="--posix.odirect -u -t 1024k "
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_files_per_proc="90000"
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
}

function setup_ior_hard {
  io500_ior_hard_writes_per_proc="80000"
  io500_ior_hard_api="POSIX"
  io500_ior_hard_api_specific_options="--posix.odirect"
}

function setup_mdt_hard {
  io500_mdtest_hard_files_per_proc="70000"
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="False"
  io500_find_cmd="/mnt/weka/io-500-dev/bin/custom_pfind.sh"
  # uses stonewalling, run pfind
  io500_find_cmd_args="-s $io500_stonewall_timer -q 50000 -r $io500_result_dir/pfind_results"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
  # TODO: Please add your information using the info-creator!
  # EXAMPLE:
  io500_info_system_name="WekaIO"      # e.g. Oakforest-PACS
  io500_info_submitter="Alexander Howard"
  io500_info_institution="WekaIO"
  io500_info_system="WekaIO"
  io500_info_system_classification="production"
  io500_info_storage_install_date="2019-11-11"
  io500_info_storage_vendor="WekaIO"
  io500_info_filesystem_name="WekaIO"
  io500_info_filesystem_type="WekaIO Matrix"
  io500_info_filesystem_version="3.5"
  io500_info_client_nodes="10"
  io500_info_client_procs_per_node="261"
  io500_info_client_operating_system="CentOS"
  io500_info_client_operating_system_version="7.6.1810"
  io500_info_client_kernel_version="3.10.0-957.el7.x86_64"
  io500_info_md_nodes="26"
  io500_info_md_storage_devices="6"
  io500_info_md_storage_type="NVMe"
  io500_info_md_volatile_memory_capacity="352.0 GB"
  io500_info_md_storage_interface="NVMe"
  io500_info_md_network="Infiniband HDR"
  io500_info_md_software_version="3.5"
  io500_info_md_operating_system_version="CentOS 7.6.1810"
  io500_info_md_overhead="12.5"
  io500_info_ds_nodes="26"
  io500_info_ds_storage_devices="156"
  io500_info_ds_storage_type="NVMe"
  io500_info_ds_volatile_memory_capacity="352.0 GB"
  io500_info_ds_storage_interface="NVMe"
  io500_info_ds_network="Infiniband HDR"
  io500_info_ds_software_version="3.5"
  io500_info_ds_operating_system_version="CentOS 7.6.1810"
  io500_info_ds_overhead="12.5"
  io500_info_whatever="We modified pfind to implement a FIFO for the pending work instead of a stack, broke a single large directory to multiple work items and made some changes to the stealing policy."
  io500_info_comment="We do not have a notion of a traditional metadata server architecture. Our metadata is distributed evenly across all storage nodes."
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 12 02:49:19 2019
Command line        : /mnt/weka/io-500-dev/bin/ior -r -R --posix.odirect -u -t 1024k -b 11000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_easy/stonewall
Machine             : Linux client104
TestID              : 0
StartTime           : Tue Nov 12 02:49:19 2019
Path                : /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_easy/0
FS                  : 400.0 TiB   Used FS: 27.8%   Inodes: 13135.3 Mi   Used Inodes: 5.6%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 109
tasks               : 2616
clients per node    : 24
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 10.74 GiB
aggregate filesize  : 27.44 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
read      87596      94819      0.025146    11264000   1024.00    27.84      303.48     30.92      328.51     0   
Max Read:  87596.28 MiB/sec (91851.36 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        87596.28   87596.28   87596.28       0.00   87596.28   87596.28   87596.28       0.00  328.50710         NA            NA     0   2616  24    1   1     1        1         0    0      1 11534336000  1048576 28776000.0 POSIX      0
Finished            : Tue Nov 12 02:55:49 2019
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 12 02:02:41 2019
Command line        : /mnt/weka/io-500-dev/bin/ior -w --posix.odirect -u -t 1024k -b 11000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux client104
TestID              : 0
StartTime           : Tue Nov 12 02:02:41 2019
Path                : /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_easy/0
FS                  : 400.0 TiB   Used FS: 18.3%   Inodes: 13135.3 Mi   Used Inodes: 3.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 109
tasks               : 2616
clients per node    : 24
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 10.74 GiB
aggregate filesize  : 27.44 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 11000 max: 11000 -- min data: 10.7 GiB mean data: 10.7 GiB time: 305.4s
write     83007      87405      0.028678    11264000   1024.00    20.14      329.23     22.51      346.67     0   
Max Write: 83007.24 MiB/sec (87039.40 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       83007.24   83007.24   83007.24       0.00   83007.24   83007.24   83007.24       0.00  346.66856     305.35      94239.04     0   2616  24    1   1     1        1         0    0      1 11534336000  1048576 28776000.0 POSIX      0
Finished            : Tue Nov 12 02:09:45 2019
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 12 03:03:54 2019
Command line        : /mnt/weka/io-500-dev/bin/ior -r -R -s 80000 -a POSIX --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_hard/stonewall
Machine             : Linux client104
TestID              : 0
StartTime           : Tue Nov 12 03:03:54 2019
Path                : /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_hard
FS                  : 400.0 TiB   Used FS: 27.8%   Inodes: 13135.3 Mi   Used Inodes: 5.6%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 80000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 109
tasks               : 2616
clients per node    : 24
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 8.95 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
read      57508      1497217    124.24      45.91      45.91      18.17      139.78     20.58      163.14     0   
Max Read:  57508.23 MiB/sec (60301.75 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        57508.23   57508.23   57508.23       0.00 1282797.55 1282797.55 1282797.55       0.00  163.14344         NA            NA     0   2616  24    1   0     1        1         0    0  80000    47008    47008 9382090.0 POSIX      0
Finished            : Tue Nov 12 03:08:21 2019
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 12 02:22:33 2019
Command line        : /mnt/weka/io-500-dev/bin/ior -w -s 80000 -a POSIX --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux client104
TestID              : 0
StartTime           : Tue Nov 12 02:22:33 2019
Path                : /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_hard
FS                  : 400.0 TiB   Used FS: 25.5%   Inodes: 13135.3 Mi   Used Inodes: 5.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 80000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 109
tasks               : 2616
clients per node    : 24
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 8.95 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 69066 max: 80000 -- min data: 3.0 GiB mean data: 3.5 GiB time: 301.6s
write     26259      622917     316.69      45.91      45.91      22.19      335.97     21.36      357.29     0   
Max Write: 26258.94 MiB/sec (27534.50 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       26258.94   26258.94   26258.94       0.00  585740.68  585740.68  585740.68       0.00  357.29121     301.62      31094.59     0   2616  24    1   0     1        1         0    0  80000    47008    47008 9382090.0 POSIX      0
Finished            : Tue Nov 12 02:30:06 2019
mdtest_easy_delete
-- started at 11/12/2019 03:53:14 --

mdtest-3.3.0+dev was launched with 2616 total task(s) on 109 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest '-r' '-F' '-P' '-d' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_easy' '-n' '90000' '-u' '-L' '-x' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_easy-stonewall' '-N' '1'
Path: /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25
FS: 400.0 TiB   Used FS: 27.8%   Inodes: 13135.3 Mi   Used Inodes: 5.6%

Nodemap: 111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 24 for each phase.
2616 tasks, 235440000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     921601.517     890976.561     916520.100       5165.166
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.068          0.068          0.068          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        264.249        255.468        256.893          1.466
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         14.658         14.658         14.658          0.000
-- finished at 11/12/2019 03:59:55 --

mdtest_easy_stat
-- started at 11/12/2019 02:58:03 --

mdtest-3.3.0+dev was launched with 2616 total task(s) on 109 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest '-T' '-F' '-P' '-d' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_easy' '-n' '90000' '-u' '-L' '-x' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_easy-stonewall' '-N' '1'
Path: /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25
FS: 400.0 TiB   Used FS: 27.8%   Inodes: 13135.3 Mi   Used Inodes: 5.6%

Nodemap: 111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 24 for each phase.
2616 tasks, 235440000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    4155237.204    3503872.341    3943250.210     126506.778
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         67.194         56.661         59.770          1.953
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/12/2019 03:01:10 --

mdtest_easy_write
-- started at 11/12/2019 02:11:58 --

mdtest-3.3.0+dev was launched with 2616 total task(s) on 109 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest '-C' '-F' '-P' '-d' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_easy' '-n' '90000' '-u' '-L' '-x' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25
FS: 400.0 TiB   Used FS: 25.2%   Inodes: 13135.3 Mi   Used Inodes: 3.3%

Nodemap: 111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 24 for each phase.
2616 tasks, 235440000 files
Continue stonewall hit min: 72296 max: 90000 avg: 88042.2 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     789962.748     758995.576     784777.130       7099.814
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     890904.266             NA
   Tree creation             :          0.065          0.065          0.065          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        310.199        298.039        300.034          2.753
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        258.522             NA
   Tree creation             :         15.335         15.335         15.335          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/12/2019 02:19:52 --

mdtest_hard_delete
-- started at 11/12/2019 04:09:13 --

mdtest-3.3.0+dev was launched with 2616 total task(s) on 109 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_hard' '-n' '70000' '-x' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25
FS: 400.0 TiB   Used FS: 27.6%   Inodes: 13135.3 Mi   Used Inodes: 3.9%

Nodemap: 111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 24 for each phase.
2616 tasks, 183120000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      35047.932      34850.765      35031.987         20.174
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.076          0.076          0.076          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       2091.103       2079.339       2080.286          1.200
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         13.236         13.236         13.236          0.000
-- finished at 11/12/2019 04:46:16 --

mdtest_hard_read
-- started at 11/12/2019 04:02:12 --

mdtest-3.3.0+dev was launched with 2616 total task(s) on 109 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_hard' '-n' '70000' '-x' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25
FS: 400.0 TiB   Used FS: 27.6%   Inodes: 13135.3 Mi   Used Inodes: 3.9%

Nodemap: 111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 24 for each phase.
2616 tasks, 183120000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     489297.422     462607.205     486101.679       2576.296
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        157.534        148.941        149.925          0.807
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/12/2019 04:07:07 --

mdtest_hard_stat
-- started at 11/12/2019 03:10:23 --

mdtest-3.3.0+dev was launched with 2616 total task(s) on 109 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_hard' '-n' '70000' '-x' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25
FS: 400.0 TiB   Used FS: 27.8%   Inodes: 13135.3 Mi   Used Inodes: 5.6%

Nodemap: 111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 24 for each phase.
2616 tasks, 183120000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      32504.262      32339.256      32482.972         20.865
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :       2253.500       2242.061       2243.531          1.443
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/12/2019 03:50:54 --

mdtest_hard_write
-- started at 11/12/2019 02:32:21 --

mdtest-3.3.0+dev was launched with 2616 total task(s) on 109 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_hard' '-n' '70000' '-x' '/mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /mnt/weka/io-500-dev/datafiles/io500.2019.11.11-23.00.25
FS: 400.0 TiB   Used FS: 27.7%   Inodes: 13135.3 Mi   Used Inodes: 5.1%

Nodemap: 111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 24 for each phase.
2616 tasks, 183120000 files
Continue stonewall hit min: 20545 max: 27858 avg: 22517.6 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     112614.929     110950.694     112329.940        257.582
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     215490.735             NA
   Tree creation             :          0.074          0.074          0.074          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        656.837        647.130        648.775          1.494
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        273.358             NA
   Tree creation             :         13.447         13.447         13.447          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/12/2019 02:46:00 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               81.061 GB/s : time 329.23 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              789.963 kiops : time 310.20 seconds
[RESULT] BW   phase 2            ior_hard_write               25.644 GB/s : time 335.97 seconds
[RESULT] IOPS phase 2         mdtest_hard_write              112.615 kiops : time 656.84 seconds
[RESULT] IOPS phase 3                      find             6846.390 kiops : time  44.99 seconds
[RESULT] BW   phase 3             ior_easy_read               85.543 GB/s : time 303.48 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             4155.240 kiops : time  67.19 seconds
[RESULT] BW   phase 4             ior_hard_read               56.160 GB/s : time 139.78 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               32.504 kiops : time 2253.50 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              921.602 kiops : time 264.25 seconds
[RESULT] IOPS phase 7          mdtest_hard_read              489.297 kiops : time 157.53 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               35.048 kiops : time 2349.41 seconds
[SCORE] Bandwidth 56.2149 GB/s : IOPS 435.76 kiops : TOTAL 156.513