WekaIO

Institution WekaIO
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.4.1708
Client Kernel Version 3.10.0-693

DATA SERVER

Storage Type SSD
Volatile Memory
Storage Interface NVMe
Network Ethernet
Software Version 3.1.8.1
OS Version CentOS 7.4.1708

INFORMATION

Client Nodes 17
Client Total Procs 935
Metadata Nodes 32
Metadata Storage Devices 6
Data Nodes 32
Data Storage Devices 6

METADATA

Easy Write 89.62 kIOP/s
Easy Stat 3,738.01 kIOP/s
Easy Delete 44.63 kIOP/s
Hard Write 3.99 kIOP/s
Hard Read 943.33 kIOP/s
Hard Stat 3,504.02 kIOP/s
Hard Delete 3.98 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_stonewall_timer=300 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="--oversubscribe -n 950 --allow-run-as-root -H 172.172.3.82,172.172.3.84,172.172.3.86,172.172.3.88,172.172.3.90,172.172.3.92,172.172.3.94,172.172.3.96,172.172.4.101,172.172.4.102,172.172.4.103,172.172.4.104,172.172.4.105,172.172.4.106,172.172.4.107,172.172.6.82,172.172.6.84 --mca pml ob1 --mca btl self,tcp"
}

function setup_ior_easy {
  # io500_ior_easy_size is the amount of data written per rank in MiB units,
  # but it can be any number as long as it is somehow used to scale the IOR
  # runtime as part of io500_ior_easy_params
  io500_ior_easy_size=13238
  io500_ior_easy_params="-a POSIX -B -t 1024k -b ${io500_ior_easy_size}m -F"
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
  io500_mdtest_easy_files_per_proc=41638
}

function setup_ior_hard {
  io500_ior_hard_writes_per_proc=45915
  io500_ior_hard_other_options="-B -a POSIX" #e.g., -E to keep precreated files using lfs setstripe, or -a MPIIO
}

function setup_mdt_hard {
  io500_mdtest_hard_files_per_proc=20000
  io500_mdtest_hard_other_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind 
  io500_find_cmd_args="-s $io500_stonewall_timer -r $io500_result_dir/pfind_results"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  io500_mdreal_params="-P=5000 -I=1000"
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Add key/value pairs defining your system
# Feel free to add extra ones if you'd like
function extra_description {
  # top level info
  io500_info_system_name='WekaIO'      # e.g. Oakforest-PACS
  io500_info_institute_name='WekaIO'   # e.g. JCAHPC
  io500_info_storage_age_in_months='1' # not install date but age since last refresh
  io500_info_storage_install_date='10/18'  # MM/YY
  io500_info_filesystem='WekaIO Matrix'     # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
  io500_info_filesystem_version='3.1.8.1'
  io500_info_filesystem_vendor='WekaIO'
  # client side info
  io500_info_num_client_nodes='17'
  io500_info_procs_per_node='55'
  # server side info
  io500_info_num_metadata_server_nodes='32'
  io500_info_num_data_server_nodes='32'
  io500_info_num_data_storage_devices='192'  # if you have 5 data servers, and each has 5 drives, then this number is 25
  io500_info_num_metadata_storage_devices='192'  # if you have 2 metadata servers, and each has 5 drives, then this number is 10
  io500_info_data_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_metadata_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_storage_network='Ethernet' # infiniband, omnipath, ethernet, etc
  io500_info_storage_interface='NVMe' # SAS, SATA, NVMe, etc
  # miscellaneous
  io500_info_whatever="Note: We don't have a notion of a traditional metadata server architecture.  Our metadata is distributed evenly across all storage nodes"
}

main
ior_easy_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 13 20:27:51 2018
Command line        : /mnt/weka/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -a POSIX -B -t 1024k -b 13238m -F -o /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_easy/stonewall
Machine             : Linux client1
TestID              : 0
StartTime           : Tue Nov 13 20:27:51 2018
Path                : /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_easy
FS                  : 461.0 TiB   Used FS: 73.4%   Inodes: 5802.5 Mi   Used Inodes: 10.6%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 950
clients per node    : 56
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 12.93 GiB
aggregate filesize  : 11.99 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      185272     13555712   1024.00    0.052606   67.82      0.006404   67.88      0   
Max Read:  185271.63 MiB/sec (194271.38 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       185271.63  185271.63  185271.63       0.00  185271.63  185271.63  185271.63       0.00   67.87925     0    950  56    1   1     1        1         0    0      1 13881049088  1048576 12576100.0 POSIX      0
Finished            : Tue Nov 13 20:28:59 2018
ior_easy_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 13 20:02:42 2018
Command line        : /mnt/weka/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -a POSIX -B -t 1024k -b 13238m -F -o /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux client1
TestID              : 0
StartTime           : Tue Nov 13 20:02:42 2018
Path                : /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_easy
FS                  : 461.0 TiB   Used FS: 70.4%   Inodes: 5802.5 Mi   Used Inodes: 9.6%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 950
clients per node    : 56
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 12.93 GiB
aggregate filesize  : 11.99 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 9593 max: 13238 -- min data: 9.4 GiB mean data: 12.7 GiB time: 300.1s
write     38598      13555712   1024.00    0.449778   325.27     0.064681   325.82     0   
Max Write: 38598.19 MiB/sec (40473.14 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       38598.19   38598.19   38598.19       0.00   38598.19   38598.19   38598.19       0.00  325.82095     0    950  56    1   1     1        1         0    0      1 13881049088  1048576 12576100.0 POSIX      0
Finished            : Tue Nov 13 20:08:08 2018
ior_hard_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 13 20:29:14 2018
Command line        : /mnt/weka/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 45915 -B -a POSIX -o /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_hard/stonewall
Machine             : Linux client1
TestID              : 0
StartTime           : Tue Nov 13 20:29:14 2018
Path                : /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_hard
FS                  : 461.0 TiB   Used FS: 73.4%   Inodes: 5802.5 Mi   Used Inodes: 10.6%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 45915
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 950
clients per node    : 56
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 1.86 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      55268      45.91      45.91      0.024802   35.36      0.008872   35.38      0   
Max Read:  55267.52 MiB/sec (57952.19 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        55267.52   55267.52   55267.52       0.00 1232815.56 1232815.56 1232815.56       0.00   35.38181     0    950  56    1   0     1        1         0    0  45915    47008    47008 1955465.0 POSIX      0
Finished            : Tue Nov 13 20:29:50 2018
ior_hard_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Tue Nov 13 20:15:35 2018
Command line        : /mnt/weka/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 45915 -B -a POSIX -o /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux client1
TestID              : 0
StartTime           : Tue Nov 13 20:15:35 2018
Path                : /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_hard
FS                  : 461.0 TiB   Used FS: 73.0%   Inodes: 5802.5 Mi   Used Inodes: 10.6%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 45915
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 950
clients per node    : 56
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 1.86 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 34495 max: 45915 -- min data: 1.5 GiB mean data: 1.5 GiB time: 300.1s
write     5439       45.91      45.91      0.068147   359.46     0.016455   359.55     0   
Max Write: 5438.66 MiB/sec (5702.85 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        5438.66    5438.66    5438.66       0.00  121316.64  121316.64  121316.64       0.00  359.54878     0    950  56    1   0     1        1         0    0  45915    47008    47008 1955465.0 POSIX      0
Finished            : Tue Nov 13 20:21:35 2018
mdtest_easy_delete
-- started at 11/13/2018 20:29:55 --

mdtest-1.9.3 was launched with 950 total task(s) on 16 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-r" "-F" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_easy" "-n" "41638" "-u" "-L" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_easy-stonewall"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39
FS: 461.0 TiB   Used FS: 73.4%   Inodes: 5802.5 Mi   Used Inodes: 10.6%

950 tasks, 39556100 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :      44630.543      44630.543      44630.543          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.910          0.910          0.910          0.000

-- finished at 11/13/2018 20:44:43 --
mdtest_easy_stat
-- started at 11/13/2018 20:29:01 --

mdtest-1.9.3 was launched with 950 total task(s) on 16 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-T" "-F" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_easy" "-n" "41638" "-u" "-L" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_easy-stonewall"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39
FS: 461.0 TiB   Used FS: 73.4%   Inodes: 5802.5 Mi   Used Inodes: 10.6%

950 tasks, 39556100 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :    3738005.966    3738005.966    3738005.966          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/13/2018 20:29:12 --
mdtest_easy_write
-- started at 11/13/2018 20:08:11 --

mdtest-1.9.3 was launched with 950 total task(s) on 16 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-C" "-F" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_easy" "-n" "41638" "-u" "-L" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_easy-stonewall" "-W" "300"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39
FS: 461.0 TiB   Used FS: 73.0%   Inodes: 5802.5 Mi   Used Inodes: 9.8%

950 tasks, 39556100 files
stonewall rank 1: 28132 of 41638 
stonewall rank 41: 28346 of 41638 
stonewall rank 9: 28044 of 41638 
stonewall rank 57: 22627 of 41638 
stonewall rank 121: 25080 of 41638 
stonewall rank 249: 28922 of 41638 
stonewall rank 465: 27236 of 41638 
stonewall rank 97: 38317 of 41638 
stonewall rank 161: 38537 of 41638 
stonewall rank 265: 29043 of 41638 
stonewall rank 473: 39028 of 41638 
stonewall rank 85: 35896 of 41638 
stonewall rank 133: 38744 of 41638 
stonewall rank 233: 29603 of 41638 
stonewall rank 449: 41315 of 41638 
stonewall rank 181: 39607 of 41638 
stonewall rank 165: 25115 of 41638 
stonewall rank 689: 21634 of 41638 
stonewall rank 453: 41141 of 41638 
stonewall rank 217: 24384 of 41638 
stonewall rank 337: 29297 of 41638 
stonewall rank 693: 21658 of 41638 
stonewall rank 481: 27559 of 41638 
stonewall rank 169: 23703 of 41638 
stonewall rank 597: 30693 of 41638 
stonewall rank 697: 32189 of 41638 
stonewall rank 489: 41513 of 41638 
stonewall rank 73: 38720 of 41638 
stonewall rank 361: 29179 of 41638 
stonewall rank 725: 21359 of 41638 
stonewall rank 457: 27207 of 41638 
stonewall rank 197: 24148 of 41638 
stonewall rank 601: 30922 of 41638 
stonewall rank 673: 21561 of 41638 
stonewall rank 477: 41256 of 41638 
stonewall rank 313: 31974 of 41638 
stonewall rank 389: 28679 of 41638 
stonewall rank 681: 32556 of 41638 
stonewall rank 40: 28437 of 41638 
stonewall rank 44: 28143 of 41638 
stonewall rank 50: 27337 of 41638 
stonewall rank 52: 28550 of 41638 
stonewall rank 54: 27597 of 41638 
stonewall rank 55: 28110 of 41638 
stonewall rank 12: 28583 of 41638 
stonewall rank 15: 28130 of 41638 
stonewall rank 35: 28220 of 41638 
stonewall rank 525: 34976 of 41638 
stonewall rank 589: 31055 of 41638 
stonewall rank 898: 28581 of 41638 
stonewall rank 709: 33344 of 41638 
stonewall rank 459: 39507 of 41638 
Continue stonewall hit min: 20600 max: 41638 avg: 38128.6 
stonewall rank 621: 32641 of 41638 
stonewall rank 785: 39741 of 41638 
stonewall rank 915: 28993 of 41638 
stonewall rank 677: 31331 of 41638 
stonewall rank 483: 27614 of 41638 
stonewall rank 505: 35700 of 41638 
stonewall rank 113: 38396 of 41638 
stonewall rank 910: 28629 of 41638 
stonewall rank 685: 20815 of 41638 
stonewall rank 491: 27238 of 41638 
stonewall rank 325: 32791 of 41638 
stonewall rank 789: 27224 of 41638 
stonewall rank 895: 27680 of 41638 
stonewall rank 705: 32739 of 41638 
stonewall rank 451: 40177 of 41638 
stonewall rank 629: 32206 of 41638 
stonewall rank 793: 26477 of 41638 
stonewall rank 929: 28147 of 41638 
stonewall rank 235: 29881 of 41638 
stonewall rank 463: 41407 of 41638 
stonewall rank 333: 32064 of 41638 
stonewall rank 801: 37913 of 41638 
stonewall rank 909: 28686 of 41638 
stonewall rank 227: 29703 of 41638 
stonewall rank 490: 26994 of 41638 
stonewall rank 757: 31432 of 41638 
stonewall rank 821: 27086 of 41638 
stonewall rank 943: 28718 of 41638 
stonewall rank 279: 29854 of 41638 
stonewall rank 462: 27509 of 41638 
stonewall rank 429: 32049 of 41638 
stonewall rank 825: 37328 of 41638 
stonewall rank 900: 29112 of 41638 
stonewall rank 238: 29663 of 41638 
stonewall rank 471: 27463 of 41638 
stonewall rank 329: 33115 of 41638 
stonewall rank 813: 27020 of 41638 
stonewall rank 941: 28232 of 41638 
stonewall rank 699: 32159 of 41638 
stonewall rank 460: 27834 of 41638 
stonewall rank 749: 31366 of 41638 
stonewall rank 137: 25017 of 41638 
stonewall rank 921: 28580 of 41638 
stonewall rank 244: 29826 of 41638 
stonewall rank 455: 27342 of 41638 
stonewall rank 521: 35773 of 41638 
stonewall rank 573: 31267 of 41638 
stonewall rank 933: 27410 of 41638 
stonewall rank 719: 31792 of 41638 
stonewall rank 503: 27515 of 41638 
stonewall rank 857: 33873 of 41638 
stonewall rank 797: 27326 of 41638 
stonewall rank 905: 28895 of 41638 
stonewall rank 230: 29957 of 41638 
stonewall rank 454: 26952 of 41638 
stonewall rank 845: 33625 of 41638 
stonewall rank 829: 39856 of 41638 
stonewall rank 916: 28426 of 41638 
stonewall rank 718: 30983 of 41638 
stonewall rank 467: 27352 of 41638 
stonewall rank 633: 32003 of 41638 
stonewall rank 163: 38948 of 41638 
stonewall rank 904: 28774 of 41638 
stonewall rank 727: 32572 of 41638 
stonewall rank 492: 40235 of 41638 
stonewall rank 753: 31307 of 41638 
stonewall rank 123: 25440 of 41638 
stonewall rank 949: 28104 of 41638 
stonewall rank 228: 29434 of 41638 
stonewall rank 500: 40231 of 41638 
stonewall rank 625: 32702 of 41638 
stonewall rank 611: 31129 of 41638 
stonewall rank 682: 21454 of 41638 
stonewall rank 502: 41545 of 41638 
stonewall rank 645: 32622 of 41638 
stonewall rank 119: 25727 of 41638 
stonewall rank 255: 29846 of 41638 
stonewall rank 466: 41538 of 41638 
stonewall rank 393: 30786 of 41638 
stonewall rank 135: 25287 of 41638 
stonewall rank 231: 29996 of 41638 
stonewall rank 487: 40900 of 41638 
stonewall rank 61: 22526 of 41638 
stonewall rank 355: 29062 of 41638 
stonewall rank 687: 21154 of 41638 
stonewall rank 464: 41246 of 41638 
stonewall rank 189: 36829 of 41638 
stonewall rank 571: 31247 of 41638 
stonewall rank 691: 31141 of 41638 
stonewall rank 484: 39807 of 41638 
stonewall rank 413: 31910 of 41638 
stonewall rank 118: 39118 of 41638 
stonewall rank 698: 31621 of 41638 
stonewall rank 448: 40740 of 41638 
stonewall rank 669: 32598 of 41638 
stonewall rank 138: 39135 of 41638 
stonewall rank 706: 31794 of 41638 
stonewall rank 765: 31586 of 41638 
stonewall rank 142: 38650 of 41638 
stonewall rank 714: 31087 of 41638 
stonewall rank 637: 32219 of 41638 
stonewall rank 362: 29590 of 41638 
stonewall rank 674: 21147 of 41638 
stonewall rank 67: 38537 of 41638 
stonewall rank 570: 31063 of 41638 
stonewall rank 684: 21511 of 41638 
stonewall rank 99: 22694 of 41638 
stonewall rank 114: 38857 of 41638 
stonewall rank 686: 21452 of 41638 
stonewall rank 59: 23007 of 41638 
stonewall rank 810: 38363 of 41638 
stonewall rank 690: 21054 of 41638 
stonewall rank 111: 22682 of 41638 
stonewall rank 578: 31378 of 41638 
stonewall rank 694: 33374 of 41638 
stonewall rank 79: 22686 of 41638 
stonewall rank 151: 25515 of 41638 
stonewall rank 680: 31405 of 41638 
stonewall rank 87: 21676 of 41638 
stonewall rank 159: 38716 of 41638 
stonewall rank 703: 20600 of 41638 
stonewall rank 171: 40500 of 41638 
stonewall rank 354: 28783 of 41638 
stonewall rank 679: 21117 of 41638 
stonewall rank 298: 32181 of 41638 
stonewall rank 610: 30298 of 41638 
stonewall rank 688: 21248 of 41638 
stonewall rank 71: 22741 of 41638 
stonewall rank 122: 25078 of 41638 
stonewall rank 676: 32366 of 41638 
stonewall rank 58: 35163 of 41638 
stonewall rank 367: 28253 of 41638 
stonewall rank 672: 31548 of 41638 
stonewall rank 207: 24111 of 41638 
stonewall rank 575: 31139 of 41638 
stonewall rank 331: 32480 of 41638 
stonewall rank 134: 36707 of 41638 
stonewall rank 559: 36165 of 41638 
stonewall rank 794: 40694 of 41638 
stonewall rank 68: 21963 of 41638 
stonewall rank 162: 38978 of 41638 
stonewall rank 70: 37669 of 41638 
stonewall rank 167: 38880 of 41638 
stonewall rank 78: 37188 of 41638 
stonewall rank 132: 25673 of 41638 
stonewall rank 110: 36391 of 41638 
stonewall rank 136: 39652 of 41638 
stonewall rank 187: 23967 of 41638 
stonewall rank 802: 38678 of 41638 
stonewall rank 315: 32888 of 41638 
stonewall rank 614: 30199 of 41638 
stonewall rank 76: 23201 of 41638 
stonewall rank 152: 25241 of 41638 
stonewall rank 108: 35960 of 41638 
stonewall rank 164: 37556 of 41638 
stonewall rank 290: 32323 of 41638 
stonewall rank 116: 38474 of 41638 
stonewall rank 524: 36073 of 41638 
stonewall rank 391: 29315 of 41638 
stonewall rank 94: 23203 of 41638 
stonewall rank 582: 30540 of 41638 
stonewall rank 95: 37205 of 41638 
stonewall rank 127: 25372 of 41638 
stonewall rank 63: 22812 of 41638 
stonewall rank 791: 36476 of 41638 
stonewall rank 195: 23553 of 41638 
stonewall rank 607: 31214 of 41638 
stonewall rank 306: 32676 of 41638 
stonewall rank 126: 39019 of 41638 
stonewall rank 546: 34779 of 41638 
stonewall rank 148: 25347 of 41638 
stonewall rank 60: 36902 of 41638 
stonewall rank 375: 29780 of 41638 
stonewall rank 80: 38396 of 41638 
stonewall rank 568: 31352 of 41638 
stonewall rank 88: 36213 of 41638 
stonewall rank 144: 24727 of 41638 
stonewall rank 103: 23217 of 41638 
stonewall rank 388: 29532 of 41638 
stonewall rank 56: 38844 of 41638 
stonewall rank 600: 31574 of 41638 
stonewall rank 211: 24306 of 41638 
stonewall rank 124: 38974 of 41638 
stonewall rank 324: 32273 of 41638 
stonewall rank 838: 38522 of 41638 
stonewall rank 506: 35755 of 41638 
stonewall rank 592: 31068 of 41638 
stonewall rank 215: 23800 of 41638 
stonewall rank 374: 29052 of 41638 
stonewall rank 334: 32598 of 41638 
stonewall rank 823: 37978 of 41638 
stonewall rank 550: 35525 of 41638 
stonewall rank 836: 38389 of 41638 
stonewall rank 179: 40527 of 41638 
stonewall rank 814: 38743 of 41638 
stonewall rank 300: 32014 of 41638 
stonewall rank 336: 28912 of 41638 
stonewall rank 548: 36325 of 41638 
stonewall rank 820: 39882 of 41638 
stonewall rank 186: 36958 of 41638 
stonewall rank 799: 26473 of 41638 
stonewall rank 311: 32574 of 41638 
stonewall rank 831: 39812 of 41638 
stonewall rank 202: 39762 of 41638 
stonewall rank 828: 38551 of 41638 
stonewall rank 332: 31524 of 41638 
stonewall rank 792: 26923 of 41638 
stonewall rank 206: 24363 of 41638 
stonewall rank 383: 29102 of 41638 
stonewall rank 310: 32245 of 41638 
stonewall rank 608: 31585 of 41638 
stonewall rank 655: 32784 of 41638 
stonewall rank 804: 38643 of 41638 
stonewall rank 738: 31348 of 41638 
stonewall rank 832: 27022 of 41638 
stonewall rank 199: 23990 of 41638 
stonewall rank 826: 27188 of 41638 
stonewall rank 772: 31142 of 41638 
stonewall rank 824: 38997 of 41638 
stonewall rank 511: 36046 of 41638 
stonewall rank 751: 31724 of 41638 
stonewall rank 204: 24000 of 41638 
stonewall rank 762: 30909 of 41638 
stonewall rank 212: 24110 of 41638 
stonewall rank 319: 32082 of 41638 
stonewall rank 214: 38870 of 41638 
stonewall rank 759: 31823 of 41638 
stonewall rank 172: 23567 of 41638 
stonewall rank 767: 31646 of 41638 
stonewall rank 180: 37855 of 41638 
stonewall rank 182: 38073 of 41638 
stonewall rank 183: 24358 of 41638 
stonewall rank 764: 31697 of 41638 
stonewall rank 847: 33881 of 41638 
stonewall rank 760: 31186 of 41638 
stonewall rank 190: 37702 of 41638 
stonewall rank 191: 24251 of 41638 
stonewall rank 198: 36720 of 41638 
stonewall rank 859: 33658 of 41638 
stonewall rank 620: 33140 of 41638 
stonewall rank 176: 23915 of 41638 
stonewall rank 188: 23574 of 41638 
stonewall rank 419: 31605 of 41638 
stonewall rank 879: 34149 of 41638 
stonewall rank 626: 32615 of 41638 
stonewall rank 439: 30961 of 41638 
stonewall rank 770: 30949 of 41638 
stonewall rank 893: 33579 of 41638 
stonewall rank 843: 33405 of 41638 
stonewall rank 877: 34067 of 41638 
stonewall rank 644: 32168 of 41638 
stonewall rank 407: 31597 of 41638 
stonewall rank 428: 31809 of 41638 
stonewall rank 894: 33647 of 41638 
stonewall rank 447: 31237 of 41638 
stonewall rank 881: 33469 of 41638 
stonewall rank 642: 32316 of 41638 
stonewall rank 414: 31300 of 41638 
stonewall rank 889: 33608 of 41638 
stonewall rank 424: 31595 of 41638 
stonewall rank 630: 32452 of 41638 
stonewall rank 438: 31807 of 41638 
stonewall rank 876: 34130 of 41638 
stonewall rank 444: 31386 of 41638 
stonewall rank 878: 33411 of 41638 
stonewall rank 639: 32722 of 41638 
stonewall rank 860: 33140 of 41638 
stonewall rank 412: 31951 of 41638 
stonewall rank 440: 31137 of 41638 
stonewall rank 885: 33895 of 41638 
stonewall rank 400: 31845 of 41638 
stonewall rank 624: 32607 of 41638 
stonewall rank 192: 38130 of 41638 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :      89616.212      89616.212      89616.212          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          2.021          2.021          2.021          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/13/2018 20:15:33 --
mdtest_hard_delete
-- started at 11/13/2018 20:44:49 --

mdtest-1.9.3 was launched with 950 total task(s) on 16 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-r" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_hard" "-n" "20000" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_hard-stonewall"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39
FS: 461.0 TiB   Used FS: 73.4%   Inodes: 5802.5 Mi   Used Inodes: 9.9%

950 tasks, 19000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :       3978.582       3978.582       3978.582          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          1.786          1.786          1.786          0.000

-- finished at 11/13/2018 20:50:02 --
mdtest_hard_read
-- started at 11/13/2018 20:44:45 --

mdtest-1.9.3 was launched with 950 total task(s) on 16 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-E" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_hard" "-n" "20000" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_hard-stonewall"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39
FS: 461.0 TiB   Used FS: 73.4%   Inodes: 5802.5 Mi   Used Inodes: 9.9%

950 tasks, 19000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :     943329.141     943329.141     943329.141          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/13/2018 20:44:47 --
mdtest_hard_stat
-- started at 11/13/2018 20:29:53 --

mdtest-1.9.3 was launched with 950 total task(s) on 16 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-T" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_hard" "-n" "20000" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_hard-stonewall"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39
FS: 461.0 TiB   Used FS: 73.4%   Inodes: 5802.5 Mi   Used Inodes: 10.6%

950 tasks, 19000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :    3504020.261    3504020.261    3504020.261          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/13/2018 20:29:53 --
mdtest_hard_write
-- started at 11/13/2018 20:21:37 --

mdtest-1.9.3 was launched with 950 total task(s) on 16 node(s)
Command line used: /mnt/weka/io-500-dev/bin/mdtest "-C" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_hard" "-n" "20000" "-x" "/mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39/mdt_hard-stonewall" "-W" "300"
Path: /mnt/weka/io-500-dev/datafiles/io500.2018.11.13-20.02.39
FS: 461.0 TiB   Used FS: 73.4%   Inodes: 5802.5 Mi   Used Inodes: 10.6%

950 tasks, 19000000 files
stonewall rank 943: 1296 of 1307 
stonewall rank 243: 1278 of 1307 
stonewall rank 947: 1297 of 1307 
stonewall rank 115: 1275 of 1307 
stonewall rank 251: 1277 of 1307 
stonewall rank 499: 1259 of 1307 
stonewall rank 49: 1271 of 1307 
stonewall rank 50: 1277 of 1307 
stonewall rank 51: 1274 of 1307 
stonewall rank 53: 1274 of 1307 
stonewall rank 2: 1276 of 1307 
stonewall rank 3: 1273 of 1307 
stonewall rank 5: 1275 of 1307 
stonewall rank 10: 1276 of 1307 
stonewall rank 11: 1276 of 1307 
stonewall rank 13: 1278 of 1307 
stonewall rank 18: 1276 of 1307 
stonewall rank 19: 1275 of 1307 
stonewall rank 25: 1273 of 1307 
stonewall rank 26: 1267 of 1307 
stonewall rank 27: 1271 of 1307 
stonewall rank 29: 1276 of 1307 
stonewall rank 33: 1272 of 1307 
stonewall rank 34: 1274 of 1307 
stonewall rank 35: 1274 of 1307 
stonewall rank 42: 1275 of 1307 
stonewall rank 43: 1274 of 1307 
stonewall rank 46: 1273 of 1307 
stonewall rank 47: 1274 of 1307 
stonewall rank 507: 1260 of 1307 
stonewall rank 927: 1296 of 1307 
stonewall rank 122: 1276 of 1307 
stonewall rank 242: 1273 of 1307 
stonewall rank 498: 1257 of 1307 
stonewall rank 1: 1276 of 1307 
Continue stonewall hit min: 1246 max: 1307 avg: 1270.9 
stonewall rank 0: 1273 of 1307 
stonewall rank 4: 1273 of 1307 
stonewall rank 6: 1278 of 1307 
stonewall rank 7: 1273 of 1307 
stonewall rank 8: 1272 of 1307 
stonewall rank 9: 1273 of 1307 
stonewall rank 12: 1272 of 1307 
stonewall rank 14: 1274 of 1307 
stonewall rank 15: 1276 of 1307 
stonewall rank 16: 1273 of 1307 
stonewall rank 17: 1272 of 1307 
stonewall rank 20: 1273 of 1307 
stonewall rank 21: 1269 of 1307 
stonewall rank 22: 1271 of 1307 
stonewall rank 23: 1277 of 1307 
stonewall rank 24: 1275 of 1307 
stonewall rank 28: 1272 of 1307 
stonewall rank 30: 1271 of 1307 
stonewall rank 31: 1273 of 1307 
stonewall rank 32: 1271 of 1307 
stonewall rank 36: 1273 of 1307 
stonewall rank 37: 1277 of 1307 
stonewall rank 38: 1269 of 1307 
stonewall rank 39: 1276 of 1307 
stonewall rank 40: 1271 of 1307 
stonewall rank 41: 1280 of 1307 
stonewall rank 44: 1275 of 1307 
stonewall rank 45: 1275 of 1307 
stonewall rank 48: 1269 of 1307 
stonewall rank 52: 1276 of 1307 
stonewall rank 54: 1273 of 1307 
stonewall rank 55: 1274 of 1307 
stonewall rank 83: 1278 of 1307 
stonewall rank 895: 1299 of 1307 
stonewall rank 123: 1275 of 1307 
stonewall rank 250: 1271 of 1307 
stonewall rank 467: 1261 of 1307 
stonewall rank 211: 1278 of 1307 
stonewall rank 931: 1298 of 1307 
stonewall rank 147: 1270 of 1307 
stonewall rank 722: 1250 of 1307 
stonewall rank 466: 1259 of 1307 
stonewall rank 755: 1261 of 1307 
stonewall rank 899: 1295 of 1307 
stonewall rank 370: 1278 of 1307 
stonewall rank 723: 1248 of 1307 
stonewall rank 475: 1261 of 1307 
stonewall rank 627: 1258 of 1307 
stonewall rank 935: 1302 of 1307 
stonewall rank 146: 1271 of 1307 
stonewall rank 227: 1276 of 1307 
stonewall rank 474: 1261 of 1307 
stonewall rank 763: 1261 of 1307 
stonewall rank 919: 1299 of 1307 
stonewall rank 371: 1273 of 1307 
stonewall rank 226: 1273 of 1307 
stonewall rank 483: 1258 of 1307 
stonewall rank 210: 1269 of 1307 
stonewall rank 915: 1295 of 1307 
stonewall rank 594: 1259 of 1307 
stonewall rank 707: 1250 of 1307 
stonewall rank 451: 1257 of 1307 
stonewall rank 91: 1272 of 1307 
stonewall rank 903: 1295 of 1307 
stonewall rank 378: 1275 of 1307 
stonewall rank 235: 1276 of 1307 
stonewall rank 482: 1260 of 1307 
stonewall rank 635: 1258 of 1307 
stonewall rank 939: 1293 of 1307 
stonewall rank 595: 1260 of 1307 
stonewall rank 675: 1249 of 1307 
stonewall rank 491: 1261 of 1307 
stonewall rank 659: 1260 of 1307 
stonewall rank 923: 1298 of 1307 
stonewall rank 379: 1277 of 1307 
stonewall rank 234: 1277 of 1307 
stonewall rank 501: 1262 of 1307 
stonewall rank 731: 1261 of 1307 
stonewall rank 911: 1302 of 1307 
stonewall rank 155: 1277 of 1307 
stonewall rank 699: 1251 of 1307 
stonewall rank 459: 1262 of 1307 
stonewall rank 90: 1276 of 1307 
stonewall rank 941: 1295 of 1307 
stonewall rank 154: 1271 of 1307 
stonewall rank 706: 1247 of 1307 
stonewall rank 458: 1260 of 1307 
stonewall rank 626: 1258 of 1307 
stonewall rank 949: 1294 of 1307 
stonewall rank 339: 1273 of 1307 
stonewall rank 674: 1249 of 1307 
stonewall rank 490: 1257 of 1307 
stonewall rank 99: 1272 of 1307 
stonewall rank 901: 1297 of 1307 
stonewall rank 603: 1256 of 1307 
stonewall rank 229: 1274 of 1307 
stonewall rank 493: 1258 of 1307 
stonewall rank 67: 1274 of 1307 
stonewall rank 930: 1297 of 1307 
stonewall rank 347: 1275 of 1307 
stonewall rank 245: 1275 of 1307 
stonewall rank 485: 1260 of 1307 
stonewall rank 739: 1262 of 1307 
stonewall rank 946: 1296 of 1307 
stonewall rank 602: 1259 of 1307 
stonewall rank 715: 1249 of 1307 
stonewall rank 473: 1257 of 1307 
stonewall rank 66: 1273 of 1307 
stonewall rank 948: 1297 of 1307 
stonewall rank 163: 1274 of 1307 
stonewall rank 275: 1271 of 1307 
stonewall rank 461: 1261 of 1307 
stonewall rank 98: 1272 of 1307 
stonewall rank 940: 1291 of 1307 
stonewall rank 338: 1276 of 1307 
stonewall rank 237: 1275 of 1307 
stonewall rank 477: 1260 of 1307 
stonewall rank 658: 1253 of 1307 
stonewall rank 942: 1297 of 1307 
stonewall rank 611: 1261 of 1307 
stonewall rank 255: 1273 of 1307 
stonewall rank 497: 1259 of 1307 
stonewall rank 738: 1266 of 1307 
stonewall rank 944: 1297 of 1307 
stonewall rank 346: 1278 of 1307 
stonewall rank 714: 1248 of 1307 
stonewall rank 465: 1256 of 1307 
stonewall rank 531: 1260 of 1307 
stonewall rank 945: 1300 of 1307 
stonewall rank 579: 1261 of 1307 
stonewall rank 236: 1268 of 1307 
stonewall rank 495: 1264 of 1307 
stonewall rank 879: 1300 of 1307 
stonewall rank 917: 1292 of 1307 
stonewall rank 162: 1281 of 1307 
stonewall rank 238: 1275 of 1307 
stonewall rank 494: 1259 of 1307 
stonewall rank 323: 1276 of 1307 
stonewall rank 921: 1301 of 1307 
stonewall rank 355: 1280 of 1307 
stonewall rank 239: 1273 of 1307 
stonewall rank 450: 1256 of 1307 
stonewall rank 547: 1263 of 1307 
stonewall rank 926: 1295 of 1307 
stonewall rank 610: 1257 of 1307 
stonewall rank 247: 1276 of 1307 
stonewall rank 453: 1260 of 1307 
stonewall rank 218: 1275 of 1307 
stonewall rank 907: 1297 of 1307 
stonewall rank 354: 1274 of 1307 
stonewall rank 249: 1267 of 1307 
stonewall rank 481: 1258 of 1307 
stonewall rank 322: 1274 of 1307 
stonewall rank 909: 1298 of 1307 
stonewall rank 578: 1260 of 1307 
stonewall rank 254: 1279 of 1307 
stonewall rank 455: 1258 of 1307 
stonewall rank 546: 1262 of 1307 
stonewall rank 910: 1298 of 1307 
stonewall rank 131: 1277 of 1307 
stonewall rank 258: 1276 of 1307 
stonewall rank 503: 1263 of 1307 
stonewall rank 58: 1273 of 1307 
stonewall rank 934: 1290 of 1307 
stonewall rank 835: 1262 of 1307 
stonewall rank 259: 1278 of 1307 
stonewall rank 454: 1263 of 1307 
stonewall rank 59: 1278 of 1307 
stonewall rank 898: 1295 of 1307 
stonewall rank 571: 1258 of 1307 
stonewall rank 274: 1273 of 1307 
stonewall rank 492: 1260 of 1307 
stonewall rank 107: 1274 of 1307 
stonewall rank 900: 1300 of 1307 
stonewall rank 138: 1274 of 1307 
stonewall rank 691: 1250 of 1307 
stonewall rank 489: 1262 of 1307 
stonewall rank 219: 1277 of 1307 
stonewall rank 902: 1296 of 1307 
stonewall rank 139: 1274 of 1307 
stonewall rank 225: 1277 of 1307 
stonewall rank 502: 1259 of 1307 
stonewall rank 747: 1261 of 1307 
stonewall rank 938: 1292 of 1307 
stonewall rank 114: 1274 of 1307 
stonewall rank 228: 1268 of 1307 
stonewall rank 471: 1256 of 1307 
stonewall rank 515: 1259 of 1307 
stonewall rank 905: 1297 of 1307 
stonewall rank 803: 1258 of 1307 
stonewall rank 230: 1275 of 1307 
stonewall rank 479: 1263 of 1307 
stonewall rank 61: 1275 of 1307 
stonewall rank 906: 1300 of 1307 
stonewall rank 562: 1257 of 1307 
stonewall rank 231: 1272 of 1307 
stonewall rank 488: 1258 of 1307 
stonewall rank 93: 1279 of 1307 
stonewall rank 913: 1295 of 1307 
stonewall rank 158: 1269 of 1307 
stonewall rank 241: 1271 of 1307 
stonewall rank 500: 1258 of 1307 
stonewall rank 101: 1272 of 1307 
stonewall rank 914: 1299 of 1307 
stonewall rank 159: 1271 of 1307 
stonewall rank 244: 1270 of 1307 
stonewall rank 452: 1259 of 1307 
stonewall rank 106: 1273 of 1307 
stonewall rank 925: 1304 of 1307 
stonewall rank 117: 1278 of 1307 
stonewall rank 246: 1273 of 1307 
stonewall rank 457: 1258 of 1307 
stonewall rank 883: 1300 of 1307 
stonewall rank 932: 1298 of 1307 
stonewall rank 127: 1276 of 1307 
stonewall rank 248: 1272 of 1307 
stonewall rank 469: 1258 of 1307 
stonewall rank 291: 1279 of 1307 
stonewall rank 933: 1291 of 1307 
stonewall rank 130: 1276 of 1307 
stonewall rank 252: 1275 of 1307 
stonewall rank 462: 1259 of 1307 
stonewall rank 523: 1258 of 1307 
stonewall rank 897: 1299 of 1307 
stonewall rank 834: 1261 of 1307 
stonewall rank 253: 1275 of 1307 
stonewall rank 463: 1257 of 1307 
stonewall rank 62: 1277 of 1307 
stonewall rank 922: 1294 of 1307 
stonewall rank 563: 1261 of 1307 
stonewall rank 269: 1275 of 1307 
stonewall rank 470: 1260 of 1307 
stonewall rank 63: 1274 of 1307 
stonewall rank 924: 1294 of 1307 
stonewall rank 145: 1277 of 1307 
stonewall rank 693: 1247 of 1307 
stonewall rank 478: 1260 of 1307 
stonewall rank 71: 1275 of 1307 
stonewall rank 929: 1296 of 1307 
stonewall rank 156: 1272 of 1307 
stonewall rank 240: 1275 of 1307 
stonewall rank 449: 1259 of 1307 
stonewall rank 77: 1278 of 1307 
stonewall rank 918: 1297 of 1307 
stonewall rank 157: 1274 of 1307 
stonewall rank 266: 1280 of 1307 
stonewall rank 487: 1259 of 1307 
stonewall rank 109: 1275 of 1307 
stonewall rank 920: 1297 of 1307 
stonewall rank 113: 1277 of 1307 
stonewall rank 267: 1275 of 1307 
stonewall rank 476: 1257 of 1307 
stonewall rank 634: 1255 of 1307 
stonewall rank 936: 1296 of 1307 
stonewall rank 121: 1273 of 1307 
stonewall rank 224: 1274 of 1307 
stonewall rank 486: 1260 of 1307 
stonewall rank 290: 1277 of 1307 
stonewall rank 937: 1292 of 1307 
stonewall rank 125: 1273 of 1307 
stonewall rank 232: 1277 of 1307 
stonewall rank 460: 1256 of 1307 
stonewall rank 514: 1260 of 1307 
stonewall rank 904: 1297 of 1307 
stonewall rank 363: 1269 of 1307 
stonewall rank 233: 1276 of 1307 
stonewall rank 484: 1259 of 1307 
stonewall rank 60: 1272 of 1307 
stonewall rank 908: 1296 of 1307 
stonewall rank 605: 1259 of 1307 
stonewall rank 701: 1252 of 1307 
stonewall rank 468: 1258 of 1307 
stonewall rank 69: 1279 of 1307 
stonewall rank 916: 1292 of 1307 
stonewall rank 161: 1272 of 1307 
stonewall rank 261: 1277 of 1307 
stonewall rank 472: 1261 of 1307 
stonewall rank 70: 1272 of 1307 
stonewall rank 912: 1300 of 1307 
stonewall rank 118: 1279 of 1307 
stonewall rank 271: 1269 of 1307 
stonewall rank 464: 1258 of 1307 
stonewall rank 81: 1270 of 1307 
stonewall rank 928: 1293 of 1307 
stonewall rank 119: 1279 of 1307 
stonewall rank 277: 1276 of 1307 
stonewall rank 456: 1259 of 1307 
stonewall rank 97: 1275 of 1307 
stonewall rank 896: 1296 of 1307 
stonewall rank 133: 1277 of 1307 
stonewall rank 703: 1249 of 1307 
stonewall rank 496: 1260 of 1307 
stonewall rank 108: 1277 of 1307 
stonewall rank 135: 1274 of 1307 
stonewall rank 263: 1270 of 1307 
stonewall rank 480: 1256 of 1307 
stonewall rank 110: 1276 of 1307 
stonewall rank 141: 1276 of 1307 
stonewall rank 268: 1273 of 1307 
stonewall rank 448: 1259 of 1307 
stonewall rank 111: 1275 of 1307 
stonewall rank 153: 1275 of 1307 
stonewall rank 270: 1276 of 1307 
stonewall rank 402: 1276 of 1307 
stonewall rank 802: 1262 of 1307 
stonewall rank 273: 1276 of 1307 
stonewall rank 741: 1262 of 1307 
stonewall rank 613: 1260 of 1307 
stonewall rank 279: 1274 of 1307 
stonewall rank 506: 1261 of 1307 
stonewall rank 116: 1273 of 1307 
stonewall rank 717: 1249 of 1307 
stonewall rank 89: 1273 of 1307 
stonewall rank 132: 1274 of 1307 
stonewall rank 257: 1271 of 1307 
stonewall rank 103: 1278 of 1307 
stonewall rank 134: 1275 of 1307 
stonewall rank 260: 1272 of 1307 
stonewall rank 105: 1272 of 1307 
stonewall rank 152: 1274 of 1307 
stonewall rank 262: 1275 of 1307 
stonewall rank 68: 1275 of 1307 
stonewall rank 164: 1275 of 1307 
stonewall rank 264: 1267 of 1307 
stonewall rank 74: 1268 of 1307 
stonewall rank 165: 1274 of 1307 
stonewall rank 265: 1275 of 1307 
stonewall rank 75: 1276 of 1307 
stonewall rank 166: 1276 of 1307 
stonewall rank 276: 1273 of 1307 
stonewall rank 403: 1274 of 1307 
stonewall rank 167: 1276 of 1307 
stonewall rank 278: 1280 of 1307 
stonewall rank 309: 1276 of 1307 
stonewall rank 349: 1277 of 1307 
stonewall rank 272: 1273 of 1307 
stonewall rank 559: 1258 of 1307 
stonewall rank 565: 1261 of 1307 
stonewall rank 711: 1248 of 1307 
stonewall rank 56: 1276 of 1307 
stonewall rank 137: 1281 of 1307 
stonewall rank 721: 1249 of 1307 
stonewall rank 57: 1277 of 1307 
stonewall rank 140: 1273 of 1307 
stonewall rank 710: 1249 of 1307 
stonewall rank 78: 1276 of 1307 
stonewall rank 142: 1278 of 1307 
stonewall rank 687: 1251 of 1307 
stonewall rank 79: 1277 of 1307 
stonewall rank 143: 1276 of 1307 
stonewall rank 689: 1248 of 1307 
stonewall rank 85: 1274 of 1307 
stonewall rank 148: 1275 of 1307 
stonewall rank 698: 1249 of 1307 
stonewall rank 86: 1275 of 1307 
stonewall rank 149: 1281 of 1307 
stonewall rank 709: 1249 of 1307 
stonewall rank 87: 1275 of 1307 
stonewall rank 150: 1275 of 1307 
stonewall rank 673: 1249 of 1307 
stonewall rank 95: 1277 of 1307 
stonewall rank 151: 1277 of 1307 
stonewall rank 683: 1251 of 1307 
stonewall rank 100: 1277 of 1307 
stonewall rank 357: 1279 of 1307 
stonewall rank 686: 1251 of 1307 
stonewall rank 102: 1274 of 1307 
stonewall rank 573: 1261 of 1307 
stonewall rank 682: 1246 of 1307 
stonewall rank 104: 1273 of 1307 
stonewall rank 124: 1278 of 1307 
stonewall rank 725: 1254 of 1307 
stonewall rank 851: 1301 of 1307 
stonewall rank 126: 1272 of 1307 
stonewall rank 679: 1248 of 1307 
stonewall rank 757: 1262 of 1307 
stonewall rank 129: 1275 of 1307 
stonewall rank 681: 1252 of 1307 
stonewall rank 525: 1258 of 1307 
stonewall rank 136: 1276 of 1307 
stonewall rank 678: 1250 of 1307 
stonewall rank 64: 1275 of 1307 
stonewall rank 144: 1277 of 1307 
stonewall rank 677: 1250 of 1307 
stonewall rank 65: 1272 of 1307 
stonewall rank 362: 1273 of 1307 
stonewall rank 690: 1251 of 1307 
stonewall rank 72: 1270 of 1307 
stonewall rank 589: 1265 of 1307 
stonewall rank 695: 1247 of 1307 
stonewall rank 73: 1272 of 1307 
stonewall rank 112: 1276 of 1307 
stonewall rank 700: 1248 of 1307 
stonewall rank 76: 1277 of 1307 
stonewall rank 120: 1273 of 1307 
stonewall rank 702: 1248 of 1307 
stonewall rank 80: 1274 of 1307 
stonewall rank 386: 1270 of 1307 
stonewall rank 692: 1249 of 1307 
stonewall rank 82: 1274 of 1307 
stonewall rank 160: 1281 of 1307 
stonewall rank 694: 1248 of 1307 
stonewall rank 84: 1274 of 1307 
stonewall rank 128: 1273 of 1307 
stonewall rank 697: 1250 of 1307 
stonewall rank 88: 1277 of 1307 
stonewall rank 575: 1258 of 1307 
stonewall rank 718: 1250 of 1307 
stonewall rank 92: 1276 of 1307 
stonewall rank 583: 1260 of 1307 
stonewall rank 719: 1249 of 1307 
stonewall rank 94: 1275 of 1307 
stonewall rank 387: 1278 of 1307 
stonewall rank 727: 1250 of 1307 
stonewall rank 96: 1276 of 1307 
stonewall rank 582: 1260 of 1307 
stonewall rank 676: 1247 of 1307 
stonewall rank 333: 1277 of 1307 
stonewall rank 821: 1261 of 1307 
stonewall rank 720: 1251 of 1307 
stonewall rank 555: 1261 of 1307 
stonewall rank 593: 1260 of 1307 
stonewall rank 724: 1250 of 1307 
stonewall rank 733: 1259 of 1307 
stonewall rank 373: 1275 of 1307 
stonewall rank 726: 1252 of 1307 
stonewall rank 537: 1260 of 1307 
stonewall rank 561: 1259 of 1307 
stonewall rank 685: 1250 of 1307 
stonewall rank 545: 1258 of 1307 
stonewall rank 377: 1280 of 1307 
stonewall rank 696: 1252 of 1307 
stonewall rank 667: 1259 of 1307 
stonewall rank 574: 1258 of 1307 
stonewall rank 705: 1251 of 1307 
stonewall rank 517: 1262 of 1307 
stonewall rank 382: 1268 of 1307 
stonewall rank 708: 1250 of 1307 
stonewall rank 850: 1297 of 1307 
stonewall rank 572: 1259 of 1307 
stonewall rank 712: 1248 of 1307 
stonewall rank 315: 1280 of 1307 
stonewall rank 383: 1273 of 1307 
stonewall rank 713: 1247 of 1307 
stonewall rank 186: 1277 of 1307 
stonewall rank 581: 1259 of 1307 
stonewall rank 716: 1249 of 1307 
stonewall rank 749: 1262 of 1307 
stonewall rank 365: 1279 of 1307 
stonewall rank 672: 1253 of 1307 
stonewall rank 187: 1280 of 1307 
stonewall rank 609: 1261 of 1307 
stonewall rank 680: 1248 of 1307 
stonewall rank 314: 1276 of 1307 
stonewall rank 369: 1274 of 1307 
stonewall rank 684: 1249 of 1307 
stonewall rank 194: 1273 of 1307 
stonewall rank 587: 1258 of 1307 
stonewall rank 688: 1251 of 1307 
stonewall rank 327: 1278 of 1307 
stonewall rank 376: 1274 of 1307 
stonewall rank 704: 1251 of 1307 
stonewall rank 195: 1276 of 1307 
stonewall rank 601: 1264 of 1307 
stonewall rank 256: 1275 of 1307 
stonewall rank 765: 1260 of 1307 
stonewall rank 380: 1278 of 1307 
stonewall rank 539: 1258 of 1307 
stonewall rank 586: 1256 of 1307 
stonewall rank 859: 1301 of 1307 
stonewall rank 381: 1274 of 1307 
stonewall rank 318: 1277 of 1307 
stonewall rank 580: 1258 of 1307 
stonewall rank 554: 1259 of 1307 
stonewall rank 353: 1279 of 1307 
stonewall rank 203: 1278 of 1307 
stonewall rank 615: 1260 of 1307 
stonewall rank 319: 1279 of 1307 
stonewall rank 366: 1275 of 1307 
stonewall rank 538: 1259 of 1307 
stonewall rank 607: 1258 of 1307 
stonewall rank 411: 1276 of 1307 
stonewall rank 367: 1279 of 1307 
stonewall rank 326: 1275 of 1307 
stonewall rank 614: 1256 of 1307 
stonewall rank 368: 1272 of 1307 
stonewall rank 541: 1257 of 1307 
stonewall rank 666: 1258 of 1307 
stonewall rank 612: 1260 of 1307 
stonewall rank 735: 1262 of 1307 
stonewall rank 372: 1273 of 1307 
stonewall rank 549: 1260 of 1307 
stonewall rank 567: 1260 of 1307 
stonewall rank 179: 1277 of 1307 
stonewall rank 374: 1270 of 1307 
stonewall rank 324: 1273 of 1307 
stonewall rank 597: 1263 of 1307 
stonewall rank 511: 1259 of 1307 
stonewall rank 375: 1274 of 1307 
stonewall rank 202: 1280 of 1307 
stonewall rank 591: 1260 of 1307 
stonewall rank 325: 1275 of 1307 
stonewall rank 839: 1262 of 1307 
stonewall rank 509: 1262 of 1307 
stonewall rank 604: 1259 of 1307 
stonewall rank 858: 1304 of 1307 
stonewall rank 345: 1276 of 1307 
stonewall rank 330: 1272 of 1307 
stonewall rank 606: 1256 of 1307 
stonewall rank 533: 1259 of 1307 
stonewall rank 351: 1279 of 1307 
stonewall rank 643: 1257 of 1307 
stonewall rank 564: 1259 of 1307 
stonewall rank 331: 1276 of 1307 
stonewall rank 358: 1276 of 1307 
stonewall rank 527: 1259 of 1307 
stonewall rank 566: 1259 of 1307 
stonewall rank 298: 1278 of 1307 
stonewall rank 359: 1276 of 1307 
stonewall rank 529: 1257 of 1307 
stonewall rank 569: 1262 of 1307 
stonewall rank 178: 1275 of 1307 
stonewall rank 360: 1278 of 1307 
stonewall rank 299: 1275 of 1307 
stonewall rank 599: 1257 of 1307 
stonewall rank 505: 1259 of 1307 
stonewall rank 361: 1281 of 1307 
stonewall rank 410: 1275 of 1307 
stonewall rank 570: 1259 of 1307 
stonewall rank 307: 1278 of 1307 
stonewall rank 364: 1278 of 1307 
stonewall rank 522: 1260 of 1307 
stonewall rank 598: 1258 of 1307 
stonewall rank 619: 1255 of 1307 
stonewall rank 388: 1276 of 1307 
stonewall rank 751: 1264 of 1307 
stonewall rank 600: 1260 of 1307 
stonewall rank 530: 1264 of 1307 
stonewall rank 389: 1272 of 1307 
stonewall rank 181: 1278 of 1307 
stonewall rank 608: 1258 of 1307 
stonewall rank 282: 1276 of 1307 
stonewall rank 390: 1282 of 1307 
stonewall rank 535: 1260 of 1307 
stonewall rank 585: 1260 of 1307 
stonewall rank 221: 1270 of 1307 
stonewall rank 391: 1281 of 1307 
stonewall rank 283: 1279 of 1307 
stonewall rank 590: 1261 of 1307 
stonewall rank 543: 1259 of 1307 
stonewall rank 826: 1262 of 1307 
stonewall rank 223: 1277 of 1307 
stonewall rank 568: 1257 of 1307 
stonewall rank 285: 1281 of 1307 
stonewall rank 337: 1277 of 1307 
stonewall rank 557: 1257 of 1307 
stonewall rank 584: 1261 of 1307 
stonewall rank 419: 1275 of 1307 
stonewall rank 340: 1273 of 1307 
stonewall rank 289: 1279 of 1307 
stonewall rank 588: 1257 of 1307 
stonewall rank 518: 1262 of 1307 
stonewall rank 341: 1280 of 1307 
stonewall rank 642: 1259 of 1307 
stonewall rank 592: 1259 of 1307 
stonewall rank 293: 1273 of 1307 
stonewall rank 342: 1277 of 1307 
stonewall rank 519: 1261 of 1307 
stonewall rank 596: 1262 of 1307 
stonewall rank 217: 1273 of 1307 
stonewall rank 343: 1272 of 1307 
stonewall rank 306: 1274 of 1307 
stonewall rank 560: 1256 of 1307 
stonewall rank 521: 1261 of 1307 
stonewall rank 352: 1277 of 1307 
stonewall rank 220: 1275 of 1307 
stonewall rank 576: 1259 of 1307 
stonewall rank 767: 1260 of 1307 
stonewall rank 356: 1273 of 1307 
stonewall rank 558: 1259 of 1307 
stonewall rank 577: 1261 of 1307 
stonewall rank 222: 1278 of 1307 
stonewall rank 827: 1264 of 1307 
stonewall rank 334: 1274 of 1307 
stonewall rank 385: 1277 of 1307 
stonewall rank 516: 1258 of 1307 
stonewall rank 829: 1259 of 1307 
stonewall rank 189: 1276 of 1307 
stonewall rank 831: 1263 of 1307 
stonewall rank 335: 1275 of 1307 
stonewall rank 837: 1259 of 1307 
stonewall rank 513: 1257 of 1307 
stonewall rank 830: 1260 of 1307 
stonewall rank 191: 1279 of 1307 
stonewall rank 787: 1261 of 1307 
stonewall rank 281: 1274 of 1307 
stonewall rank 795: 1260 of 1307 
stonewall rank 551: 1260 of 1307 
stonewall rank 810: 1261 of 1307 
stonewall rank 205: 1277 of 1307 
stonewall rank 811: 1259 of 1307 
stonewall rank 310: 1276 of 1307 
stonewall rank 384: 1271 of 1307 
stonewall rank 526: 1258 of 1307 
stonewall rank 794: 1264 of 1307 
stonewall rank 443: 1274 of 1307 
stonewall rank 819: 1260 of 1307 
stonewall rank 311: 1276 of 1307 
stonewall rank 786: 1259 of 1307 
stonewall rank 553: 1260 of 1307 
stonewall rank 797: 1261 of 1307 
stonewall rank 618: 1257 of 1307 
stonewall rank 801: 1259 of 1307 
stonewall rank 524: 1263 of 1307 
stonewall rank 828: 1262 of 1307 
stonewall rank 188: 1277 of 1307 
stonewall rank 805: 1264 of 1307 
stonewall rank 508: 1259 of 1307 
stonewall rank 793: 1265 of 1307 
stonewall rank 190: 1278 of 1307 
stonewall rank 823: 1262 of 1307 
stonewall rank 510: 1261 of 1307 
stonewall rank 789: 1261 of 1307 
stonewall rank 198: 1279 of 1307 
stonewall rank 822: 1261 of 1307 
stonewall rank 313: 1280 of 1307 
stonewall rank 820: 1260 of 1307 
stonewall rank 542: 1260 of 1307 
stonewall rank 825: 1263 of 1307 
stonewall rank 199: 1277 of 1307 
stonewall rank 785: 1265 of 1307 
stonewall rank 771: 1264 of 1307 
stonewall rank 791: 1262 of 1307 
stonewall rank 534: 1261 of 1307 
stonewall rank 799: 1263 of 1307 
stonewall rank 209: 1278 of 1307 
stonewall rank 833: 1261 of 1307 
stonewall rank 286: 1277 of 1307 
stonewall rank 788: 1266 of 1307 
stonewall rank 540: 1261 of 1307 
stonewall rank 790: 1261 of 1307 
stonewall rank 216: 1278 of 1307 
stonewall rank 798: 1264 of 1307 
stonewall rank 287: 1269 of 1307 
stonewall rank 809: 1260 of 1307 
stonewall rank 556: 1263 of 1307 
stonewall rank 815: 1262 of 1307 
stonewall rank 174: 1276 of 1307 
stonewall rank 796: 1265 of 1307 
stonewall rank 308: 1279 of 1307 
stonewall rank 813: 1265 of 1307 
stonewall rank 550: 1258 of 1307 
stonewall rank 817: 1261 of 1307 
stonewall rank 175: 1273 of 1307 
stonewall rank 824: 1262 of 1307 
stonewall rank 321: 1276 of 1307 
stonewall rank 784: 1259 of 1307 
stonewall rank 552: 1259 of 1307 
stonewall rank 792: 1266 of 1307 
stonewall rank 177: 1280 of 1307 
stonewall rank 838: 1263 of 1307 
stonewall rank 328: 1280 of 1307 
stonewall rank 807: 1262 of 1307 
stonewall rank 504: 1260 of 1307 
stonewall rank 344: 1275 of 1307 
stonewall rank 442: 1277 of 1307 
stonewall rank 348: 1278 of 1307 
stonewall rank 329: 1273 of 1307 
stonewall rank 350: 1272 of 1307 
stonewall rank 544: 1260 of 1307 
stonewall rank 806: 1261 of 1307 
stonewall rank 651: 1258 of 1307 
stonewall rank 336: 1276 of 1307 
stonewall rank 332: 1279 of 1307 
stonewall rank 818: 1261 of 1307 
stonewall rank 548: 1264 of 1307 
stonewall rank 832: 1260 of 1307 
stonewall rank 213: 1277 of 1307 
stonewall rank 836: 1261 of 1307 
stonewall rank 753: 1262 of 1307 
stonewall rank 804: 1261 of 1307 
stonewall rank 520: 1261 of 1307 
stonewall rank 814: 1259 of 1307 
stonewall rank 169: 1277 of 1307 
stonewall rank 816: 1262 of 1307 
stonewall rank 280: 1276 of 1307 
stonewall rank 808: 1258 of 1307 
stonewall rank 532: 1263 of 1307 
stonewall rank 812: 1261 of 1307 
stonewall rank 284: 1278 of 1307 
stonewall rank 800: 1258 of 1307 
stonewall rank 536: 1258 of 1307 
stonewall rank 170: 1280 of 1307 
stonewall rank 295: 1274 of 1307 
stonewall rank 512: 1259 of 1307 
stonewall rank 171: 1277 of 1307 
stonewall rank 296: 1279 of 1307 
stonewall rank 528: 1261 of 1307 
stonewall rank 196: 1278 of 1307 
stonewall rank 297: 1277 of 1307 
stonewall rank 197: 1274 of 1307 
stonewall rank 300: 1279 of 1307 
stonewall rank 418: 1270 of 1307 
stonewall rank 301: 1276 of 1307 
stonewall rank 671: 1256 of 1307 
stonewall rank 302: 1277 of 1307 
stonewall rank 182: 1281 of 1307 
stonewall rank 303: 1281 of 1307 
stonewall rank 183: 1275 of 1307 
stonewall rank 305: 1277 of 1307 
stonewall rank 185: 1279 of 1307 
stonewall rank 761: 1262 of 1307 
stonewall rank 207: 1280 of 1307 
stonewall rank 292: 1277 of 1307 
stonewall rank 867: 1302 of 1307 
stonewall rank 294: 1277 of 1307 
stonewall rank 639: 1256 of 1307 
stonewall rank 762: 1261 of 1307 
stonewall rank 201: 1277 of 1307 
stonewall rank 770: 1263 of 1307 
stonewall rank 204: 1280 of 1307 
stonewall rank 759: 1259 of 1307 
stonewall rank 206: 1275 of 1307 
stonewall rank 729: 1261 of 1307 
stonewall rank 212: 1274 of 1307 
stonewall rank 730: 1260 of 1307 
stonewall rank 214: 1276 of 1307 
stonewall rank 742: 1257 of 1307 
stonewall rank 215: 1275 of 1307 
stonewall rank 743: 1259 of 1307 
stonewall rank 176: 1275 of 1307 
stonewall rank 746: 1263 of 1307 
stonewall rank 180: 1275 of 1307 
stonewall rank 754: 1260 of 1307 
stonewall rank 184: 1271 of 1307 
stonewall rank 740: 1262 of 1307 
stonewall rank 866: 1298 of 1307 
stonewall rank 781: 1264 of 1307 
stonewall rank 621: 1258 of 1307 
stonewall rank 737: 1262 of 1307 
stonewall rank 168: 1277 of 1307 
stonewall rank 745: 1262 of 1307 
stonewall rank 172: 1276 of 1307 
stonewall rank 750: 1265 of 1307 
stonewall rank 173: 1282 of 1307 
stonewall rank 779: 1263 of 1307 
stonewall rank 192: 1272 of 1307 
stonewall rank 744: 1259 of 1307 
stonewall rank 193: 1277 of 1307 
stonewall rank 748: 1259 of 1307 
stonewall rank 778: 1258 of 1307 
stonewall rank 732: 1262 of 1307 
stonewall rank 734: 1259 of 1307 
stonewall rank 766: 1264 of 1307 
stonewall rank 200: 1275 of 1307 
stonewall rank 756: 1262 of 1307 
stonewall rank 208: 1277 of 1307 
stonewall rank 758: 1261 of 1307 
stonewall rank 435: 1277 of 1307 
stonewall rank 760: 1265 of 1307 
stonewall rank 629: 1259 of 1307 
stonewall rank 764: 1262 of 1307 
stonewall rank 669: 1258 of 1307 
stonewall rank 728: 1258 of 1307 
stonewall rank 875: 1298 of 1307 
stonewall rank 773: 1261 of 1307 
stonewall rank 395: 1279 of 1307 
stonewall rank 736: 1262 of 1307 
stonewall rank 434: 1277 of 1307 
stonewall rank 752: 1262 of 1307 
stonewall rank 437: 1279 of 1307 
stonewall rank 783: 1261 of 1307 
stonewall rank 633: 1257 of 1307 
stonewall rank 780: 1260 of 1307 
stonewall rank 891: 1300 of 1307 
stonewall rank 320: 1279 of 1307 
stonewall rank 637: 1257 of 1307 
stonewall rank 782: 1262 of 1307 
stonewall rank 445: 1276 of 1307 
stonewall rank 775: 1260 of 1307 
stonewall rank 657: 1254 of 1307 
stonewall rank 774: 1263 of 1307 
stonewall rank 394: 1279 of 1307 
stonewall rank 777: 1263 of 1307 
stonewall rank 625: 1259 of 1307 
stonewall rank 772: 1260 of 1307 
stonewall rank 414: 1274 of 1307 
stonewall rank 316: 1274 of 1307 
stonewall rank 623: 1259 of 1307 
stonewall rank 415: 1275 of 1307 
stonewall rank 653: 1257 of 1307 
stonewall rank 861: 1294 of 1307 
stonewall rank 317: 1276 of 1307 
stonewall rank 650: 1259 of 1307 
stonewall rank 769: 1260 of 1307 
stonewall rank 412: 1278 of 1307 
stonewall rank 288: 1278 of 1307 
stonewall rank 631: 1259 of 1307 
stonewall rank 304: 1271 of 1307 
stonewall rank 413: 1280 of 1307 
stonewall rank 312: 1276 of 1307 
stonewall rank 665: 1261 of 1307 
stonewall rank 776: 1263 of 1307 
stonewall rank 444: 1277 of 1307 
stonewall rank 768: 1258 of 1307 
stonewall rank 622: 1255 of 1307 
stonewall rank 446: 1282 of 1307 
stonewall rank 647: 1258 of 1307 
stonewall rank 447: 1274 of 1307 
stonewall rank 620: 1258 of 1307 
stonewall rank 869: 1300 of 1307 
stonewall rank 638: 1255 of 1307 
stonewall rank 397: 1275 of 1307 
stonewall rank 645: 1260 of 1307 
stonewall rank 401: 1270 of 1307 
stonewall rank 630: 1262 of 1307 
stonewall rank 430: 1277 of 1307 
stonewall rank 668: 1257 of 1307 
stonewall rank 431: 1274 of 1307 
stonewall rank 670: 1255 of 1307 
stonewall rank 433: 1273 of 1307 
stonewall rank 617: 1259 of 1307 
stonewall rank 874: 1299 of 1307 
stonewall rank 646: 1258 of 1307 
stonewall rank 417: 1271 of 1307 
stonewall rank 636: 1257 of 1307 
stonewall rank 422: 1278 of 1307 
stonewall rank 644: 1255 of 1307 
stonewall rank 423: 1274 of 1307 
stonewall rank 632: 1253 of 1307 
stonewall rank 427: 1277 of 1307 
stonewall rank 661: 1258 of 1307 
stonewall rank 885: 1297 of 1307 
stonewall rank 664: 1256 of 1307 
stonewall rank 420: 1270 of 1307 
stonewall rank 655: 1259 of 1307 
stonewall rank 421: 1276 of 1307 
stonewall rank 628: 1258 of 1307 
stonewall rank 425: 1278 of 1307 
stonewall rank 652: 1257 of 1307 
stonewall rank 845: 1300 of 1307 
stonewall rank 654: 1257 of 1307 
stonewall rank 405: 1275 of 1307 
stonewall rank 624: 1259 of 1307 
stonewall rank 432: 1278 of 1307 
stonewall rank 663: 1257 of 1307 
stonewall rank 436: 1272 of 1307 
stonewall rank 616: 1258 of 1307 
stonewall rank 438: 1277 of 1307 
stonewall rank 649: 1258 of 1307 
stonewall rank 439: 1278 of 1307 
stonewall rank 648: 1255 of 1307 
stonewall rank 440: 1276 of 1307 
stonewall rank 662: 1257 of 1307 
stonewall rank 441: 1277 of 1307 
stonewall rank 641: 1262 of 1307 
stonewall rank 882: 1302 of 1307 
stonewall rank 660: 1257 of 1307 
stonewall rank 404: 1272 of 1307 
stonewall rank 656: 1256 of 1307 
stonewall rank 406: 1277 of 1307 
stonewall rank 640: 1258 of 1307 
stonewall rank 407: 1279 of 1307 
stonewall rank 408: 1276 of 1307 
stonewall rank 409: 1273 of 1307 
stonewall rank 424: 1278 of 1307 
stonewall rank 426: 1278 of 1307 
stonewall rank 428: 1277 of 1307 
stonewall rank 429: 1277 of 1307 
stonewall rank 392: 1280 of 1307 
stonewall rank 393: 1278 of 1307 
stonewall rank 396: 1274 of 1307 
stonewall rank 398: 1277 of 1307 
stonewall rank 399: 1273 of 1307 
stonewall rank 878: 1294 of 1307 
stonewall rank 416: 1277 of 1307 
stonewall rank 400: 1271 of 1307 
stonewall rank 894: 1301 of 1307 
stonewall rank 881: 1297 of 1307 
stonewall rank 876: 1303 of 1307 
stonewall rank 877: 1303 of 1307 
stonewall rank 843: 1301 of 1307 
stonewall rank 865: 1302 of 1307 
stonewall rank 889: 1304 of 1307 
stonewall rank 842: 1299 of 1307 
stonewall rank 857: 1304 of 1307 
stonewall rank 873: 1299 of 1307 
stonewall rank 863: 1305 of 1307 
stonewall rank 870: 1302 of 1307 
stonewall rank 871: 1302 of 1307 
stonewall rank 892: 1294 of 1307 
stonewall rank 893: 1302 of 1307 
stonewall rank 847: 1298 of 1307 
stonewall rank 849: 1302 of 1307 
stonewall rank 853: 1300 of 1307 
stonewall rank 856: 1304 of 1307 
stonewall rank 860: 1300 of 1307 
stonewall rank 862: 1302 of 1307 
stonewall rank 868: 1299 of 1307 
stonewall rank 872: 1300 of 1307 
stonewall rank 841: 1297 of 1307 
stonewall rank 844: 1301 of 1307 
stonewall rank 846: 1298 of 1307 
stonewall rank 848: 1303 of 1307 
stonewall rank 852: 1296 of 1307 
stonewall rank 854: 1300 of 1307 
stonewall rank 855: 1304 of 1307 
stonewall rank 888: 1304 of 1307 
stonewall rank 890: 1303 of 1307 
stonewall rank 840: 1299 of 1307 
stonewall rank 864: 1293 of 1307 
stonewall rank 880: 1300 of 1307 
stonewall rank 884: 1298 of 1307 
stonewall rank 886: 1299 of 1307 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :      61165.218      61165.218      61165.218          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :        268.384        268.384        268.384          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/13/2018 20:26:48 --
result_summary
[RESULT] BW   phase 1            ior_easy_write               37.693 GB/s : time 325.82 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               89.616 kiops : time 444.58 seconds
[RESULT] BW   phase 2            ior_hard_write                5.312 GB/s : time 359.55 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               61.165 kiops : time 312.96 seconds
[RESULT] IOPS phase 3                      find              675.290 kiops : time  60.42 seconds
[RESULT] BW   phase 3             ior_easy_read              180.930 GB/s : time  67.88 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             3738.010 kiops : time  13.21 seconds
[RESULT] BW   phase 4             ior_hard_read               53.973 GB/s : time  35.38 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat             3504.020 kiops : time   2.78 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               44.631 kiops : time 889.73 seconds
[RESULT] IOPS phase 7          mdtest_hard_read              943.329 kiops : time   3.88 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                3.979 kiops : time 315.12 seconds
[SCORE] Bandwidth 37.3932 GB/s : IOPS 231.048 kiops : TOTAL 92.9496