Exascaler on GCP

Institution Google
Client Procs Per Node
Client Operating System Exascaler
Client Operating System Version 5.0 PRE - Centos 7x
Client Kernel Version 3.10.0-862.14.4.el7_lustre.ddn0.x86_64

DATA SERVER

Storage Type NVMe
Volatile Memory 60GB
Storage Interface NVMe
Network Ethernet 16Gbps
Software Version Exascaler 5.0 PRE - Lustre 2.11-56
OS Version Centos 7.5

INFORMATION

Client Nodes 120
Client Total Procs 960
Metadata Nodes 4
Metadata Storage Devices 4
Data Nodes 120
Data Storage Devices 4

METADATA

Easy Write 49.18 kIOP/s
Easy Stat 325.97 kIOP/s
Easy Delete 37.47 kIOP/s
Hard Write 46.70 kIOP/s
Hard Read 68.72 kIOP/s
Hard Stat 178.79 kIOP/s
Hard Delete 25.25 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_stonewall_timer=300 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=/lustre/gcp/io500/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=/lustre/gcp/io500/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
  lfs setdirstripe -c 16 $io500_workdir/mdt_easy
  lfs setdirstripe -c 16 $io500_workdir/mdt_hard
  lfs setdirstripe -c 16 -D $io500_workdir/mdt_easy
  lfs setdirstripe -c 16 -D $io500_workdir/mdt_hard
  mkdir -p $io500_workdir/ior_hard
  lfs setstripe -c -1 -S 47104k $io500_workdir/ior_hard
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 960 -ppn 8 -f /lustre/gcp/cfg/machines.txt"
}

function setup_ior_easy {
  # io500_ior_easy_size is the amount of data written per rank in MiB units,
  # but it can be any number as long as it is somehow used to scale the IOR
  # runtime as part of io500_ior_easy_params
  #io500_ior_easy_size=2000
  io500_ior_easy_size=60000
  # 2M writes, 2 GB per proc, file per proc
  io500_ior_easy_params="-t 2048k -b ${io500_ior_easy_size}m -F"
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
 # io500_mdtest_easy_files_per_proc=25000
  io500_mdtest_easy_files_per_proc=125000
}

function setup_ior_hard {
 # io500_ior_hard_writes_per_proc=10000
  io500_ior_hard_writes_per_proc=42000
  io500_ior_hard_other_options="" #e.g., -E to keep precreated files using lfs setstripe, or -a MPIIO
}

function setup_mdt_hard {
 # io500_mdtest_hard_files_per_proc=5000
  io500_mdtest_hard_files_per_proc=65000
  io500_mdtest_hard_other_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind 
  io500_find_cmd_args="-s $io500_stonewall_timer -r $io500_result_dir/pfind_results"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  io500_mdreal_params="-P=5000 -I=1000"
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Add key/value pairs defining your system
# Feel free to add extra ones if you'd like
function extra_description {
  # top level info
  io500_info_system_name='Lustre on Google Cloud Platform'      # e.g. Oakforest-PACS
  io500_info_institute_name='Google'   # e.g. JCAHPC
  io500_info_storage_age_in_months='0' # not install date but age since last refresh
  io500_info_storage_install_date='11/18'  # MM/YY
  io500_info_filesystem='Lustre'     # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
  io500_info_filesystem_version='Exascaler 5.0 PRE - Lustre 2.11-56 (Master)'
  io500_info_filesystem_vendor='DDN'
  # client side info
  io500_info_num_client_nodes='120'
  io500_info_procs_per_node='8'
  # server side info
  io500_info_num_metadata_server_nodes='4'
  io500_info_num_data_server_nodes='120'
  io500_info_num_data_storage_devices='480'  # if you have 5 data servers, and each has 5 drives, then this number is 25
  io500_info_num_metadata_storage_devices='16'  # if you have 2 metadata servers, and each has 5 drives, then this number is 10
  io500_info_data_storage_type='NVMe' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_metadata_storage_type='NVMe' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_storage_network='16Gbps Ethernet' # infiniband, omnipath, ethernet, etc
  io500_info_storage_interface='NVMe' # SAS, SATA, NVMe, etc
  # miscellaneous
  io500_info_whatever='Medium Size configuration part of development between Google and DDN'
}

main
ior_easy_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 11 16:57:20 2018
Command line        : /scratch/io-500-dev-master/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 2048k -b 60000m -F -o /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_easy/ior_file_easy -O stoneWallingStatusFile=/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_easy/stonewall
Machine             : Linux lgcp-client0
TestID              : 0
StartTime           : Sun Nov 11 16:57:20 2018
Path                : /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_easy
FS                  : 144.4 TiB   Used FS: 39.3%   Inodes: 2400.0 Mi   Used Inodes: 1.6%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 960
clients per node    : 8
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 58.59 GiB
aggregate filesize  : 54.93 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      39181      61440000   2048.00    0.032451   1470.06    0.016843   1470.11    0   
Max Read:  39180.80 MiB/sec (41084.05 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        39180.80   39180.80   39180.80       0.00   19590.40   19590.40   19590.40       0.00 1470.10763     0    960   8    1   1     1        1         0    0      1 62914560000  2097152 57600000.0 POSIX      0
Finished            : Sun Nov 11 17:21:50 2018
ior_easy_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 11 16:28:40 2018
Command line        : /scratch/io-500-dev-master/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 2048k -b 60000m -F -o /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_easy/ior_file_easy -O stoneWallingStatusFile=/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux lgcp-client0
TestID              : 0
StartTime           : Sun Nov 11 16:28:40 2018
Path                : /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_easy
FS                  : 144.4 TiB   Used FS: 0.0%   Inodes: 2400.0 Mi   Used Inodes: 0.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 960
clients per node    : 8
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 58.59 GiB
aggregate filesize  : 54.93 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 11099 max: 30000 -- min data: 21.7 GiB mean data: 41.8 GiB time: 301.5s
write     109800     61440000   2048.00    0.035792   524.53     0.026526   524.59     0   
Max Write: 109800.32 MiB/sec (115133.98 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      109800.32  109800.32  109800.32       0.00   54900.16   54900.16   54900.16       0.00  524.58864     0    960   8    1   1     1        1         0    0      1 62914560000  2097152 57600000.0 POSIX      0
Finished            : Sun Nov 11 16:37:24 2018
ior_hard_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 11 17:23:01 2018
Command line        : /scratch/io-500-dev-master/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 42000 -o /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_hard/IOR_file -O stoneWallingStatusFile=/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_hard/stonewall
Machine             : Linux lgcp-client0
TestID              : 0
StartTime           : Sun Nov 11 17:23:01 2018
Path                : /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_hard
FS                  : 144.4 TiB   Used FS: 39.3%   Inodes: 2400.0 Mi   Used Inodes: 1.6%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 42000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 960
clients per node    : 8
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 1.72 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      12478      45.91      45.91      0.045464   144.77     0.043879   144.86     0   
Max Read:  12478.39 MiB/sec (13084.54 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        12478.39   12478.39   12478.39       0.00  278347.09  278347.09  278347.09       0.00  144.85512     0    960   8    1   0     1        1         0    0  42000    47008    47008 1807558.6 POSIX      0
Finished            : Sun Nov 11 17:25:26 2018
ior_hard_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 11 16:45:08 2018
Command line        : /scratch/io-500-dev-master/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 42000 -o /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_hard/IOR_file -O stoneWallingStatusFile=/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux lgcp-client0
TestID              : 0
StartTime           : Sun Nov 11 16:45:08 2018
Path                : /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_hard
FS                  : 144.4 TiB   Used FS: 38.1%   Inodes: 2400.0 Mi   Used Inodes: 0.9%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 42000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 960
clients per node    : 8
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 1.72 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 38296 max: 42000 -- min data: 1.7 GiB mean data: 1.8 GiB time: 300.0s
write     5793       45.91      45.91      0.022770   312.00     0.010409   312.03     0   
Max Write: 5792.97 MiB/sec (6074.37 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        5792.97    5792.97    5792.97       0.00  129219.96  129219.96  129219.96       0.00  312.02609     0    960   8    1   0     1        1         0    0  42000    47008    47008 1807558.6 POSIX      0
Finished            : Sun Nov 11 16:50:20 2018
mdtest_easy_delete
-- started at 11/11/2018 17:27:05 --

mdtest-1.9.3 was launched with 960 total task(s) on 120 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-r" "-F" "-d" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_easy" "-n" "125000" "-u" "-L" "-x" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_easy-stonewall"
Path: /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26
FS: 144.4 TiB   Used FS: 39.3%   Inodes: 2400.0 Mi   Used Inodes: 1.6%

960 tasks, 120000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :      37465.081      37465.081      37465.081          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.356          0.356          0.356          0.000

-- finished at 11/11/2018 17:37:15 --
mdtest_easy_stat
-- started at 11/11/2018 17:21:51 --

mdtest-1.9.3 was launched with 960 total task(s) on 120 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-T" "-F" "-d" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_easy" "-n" "125000" "-u" "-L" "-x" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_easy-stonewall"
Path: /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26
FS: 144.4 TiB   Used FS: 39.3%   Inodes: 2400.0 Mi   Used Inodes: 1.6%

960 tasks, 120000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :     325971.524     325971.524     325971.524          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/11/2018 17:23:01 --
mdtest_easy_write
-- started at 11/11/2018 16:37:25 --

mdtest-1.9.3 was launched with 960 total task(s) on 120 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-C" "-F" "-d" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_easy" "-n" "125000" "-u" "-L" "-x" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_easy-stonewall" "-W" "300"
Path: /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26
FS: 144.4 TiB   Used FS: 38.1%   Inodes: 2400.0 Mi   Used Inodes: 0.0%

960 tasks, 120000000 files
stonewall rank 30: 16312 of 23669 
stonewall rank 145: 16041 of 23669 
stonewall rank 214: 15717 of 23669 
stonewall rank 394: 16014 of 23669 
stonewall rank 401: 16355 of 23669 
stonewall rank 529: 18479 of 23669 
stonewall rank 658: 15398 of 23669 
stonewall rank 689: 16016 of 23669 
stonewall rank 10: 15304 of 23669 
stonewall rank 49: 15174 of 23669 
stonewall rank 17: 15558 of 23669 
stonewall rank 31: 15818 of 23669 
stonewall rank 74: 15418 of 23669 
stonewall rank 81: 16235 of 23669 
stonewall rank 90: 16347 of 23669 
stonewall rank 137: 16673 of 23669 
stonewall rank 150: 16162 of 23669 
stonewall rank 177: 15218 of 23669 
stonewall rank 218: 21844 of 23669 
stonewall rank 215: 15607 of 23669 
stonewall rank 273: 23046 of 23669 
stonewall rank 282: 15035 of 23669 
stonewall rank 305: 16736 of 23669 
stonewall rank 266: 16464 of 23669 
stonewall rank 337: 16399 of 23669 
stonewall rank 345: 15622 of 23669 
stonewall rank 395: 16128 of 23669 
stonewall rank 410: 16442 of 23669 
stonewall rank 402: 16566 of 23669 
stonewall rank 473: 16391 of 23669 
stonewall rank 531: 18285 of 23669 
stonewall rank 537: 16212 of 23669 
stonewall rank 561: 20155 of 23669 
stonewall rank 593: 15903 of 23669 
stonewall rank 659: 15157 of 23669 
stonewall rank 693: 15754 of 23669 
stonewall rank 789: 16131 of 23669 
stonewall rank 905: 15768 of 23669 
stonewall rank 923: 15876 of 23669 
stonewall rank 11: 14794 of 23669 
stonewall rank 51: 15272 of 23669 
stonewall rank 18: 16114 of 23669 
stonewall rank 42: 16015 of 23669 
stonewall rank 26: 16227 of 23669 
stonewall rank 75: 15190 of 23669 
stonewall rank 84: 15827 of 23669 
stonewall rank 91: 16003 of 23669 
stonewall rank 139: 16317 of 23669 
stonewall rank 156: 15087 of 23669 
stonewall rank 151: 16159 of 23669 
stonewall rank 182: 14977 of 23669 
stonewall rank 219: 21144 of 23669 
stonewall rank 210: 16024 of 23669 
stonewall rank 241: 21329 of 23669 
stonewall rank 274: 22673 of 23669 
stonewall rank 283: 15172 of 23669 
stonewall rank 306: 16216 of 23669 
stonewall rank 267: 16239 of 23669 
stonewall rank 330: 16506 of 23669 
stonewall rank 342: 15842 of 23669 
stonewall rank 346: 15432 of 23669 
stonewall rank 397: 16933 of 23669 
stonewall rank 411: 16682 of 23669 
stonewall rank 403: 16532 of 23669 
stonewall rank 462: 16022 of 23669 
stonewall rank 474: 16680 of 23669 
stonewall rank 521: 15625 of 23669 
stonewall rank 533: 17931 of 23669 
stonewall rank 540: 16542 of 23669 
stonewall rank 564: 20089 of 23669 
stonewall rank 589: 16431 of 23669 
stonewall rank 601: 16334 of 23669 
stonewall rank 594: 16472 of 23669 
stonewall rank 649: 19798 of 23669 
stonewall rank 661: 15371 of 23669 
stonewall rank 695: 15928 of 23669 
stonewall rank 721: 15448 of 23669 
stonewall rank 729: 16032 of 23669 
stonewall rank 755: 15942 of 23669 
stonewall rank 766: 21939 of 23669 
stonewall rank 785: 16090 of 23669 
stonewall rank 820: 23168 of 23669 
stonewall rank 844: 16044 of 23669 
stonewall rank 906: 16728 of 23669 
stonewall rank 921: 16352 of 23669 
stonewall rank 9: 15106 of 23669 
stonewall rank 53: 15449 of 23669 
stonewall rank 19: 15836 of 23669 
stonewall rank 45: 16304 of 23669 
stonewall rank 27: 16167 of 23669 
stonewall rank 77: 14850 of 23669 
stonewall rank 85: 15834 of 23669 
stonewall rank 94: 16109 of 23669 
stonewall rank 138: 16201 of 23669 
stonewall rank 157: 15422 of 23669 
stonewall rank 146: 16268 of 23669 
stonewall rank 178: 14967 of 23669 
stonewall rank 221: 21496 of 23669 
stonewall rank 211: 15850 of 23669 
stonewall rank 242: 20928 of 23669 
stonewall rank 253: 16131 of 23669 
stonewall rank 275: 22975 of 23669 
stonewall rank 286: 15313 of 23669 
stonewall rank 307: 16524 of 23669 
stonewall rank 269: 16443 of 23669 
stonewall rank 331: 17027 of 23669 
stonewall rank 343: 16023 of 23669 
stonewall rank 347: 15615 of 23669 
stonewall rank 365: 15941 of 23669 
stonewall rank 393: 16150 of 23669 
stonewall rank 409: 16480 of 23669 
stonewall rank 406: 15951 of 23669 
stonewall rank 433: 16733 of 23669 
stonewall rank 468: 16537 of 23669 
stonewall rank 463: 16171 of 23669 
stonewall rank 475: 16466 of 23669 
stonewall rank 510: 16032 of 23669 
stonewall rank 522: 15949 of 23669 
stonewall rank 535: 18431 of 23669 
stonewall rank 541: 16190 of 23669 
stonewall rank 553: 17532 of 23669 
stonewall rank 565: 20323 of 23669 
stonewall rank 585: 15510 of 23669 
stonewall rank 606: 16541 of 23669 
stonewall rank 595: 16102 of 23669 
stonewall rank 650: 19796 of 23669 
stonewall rank 657: 15156 of 23669 
stonewall rank 692: 16189 of 23669 
stonewall rank 723: 15396 of 23669 
stonewall rank 730: 16378 of 23669 
stonewall rank 757: 16145 of 23669 
stonewall rank 767: 22660 of 23669 
stonewall rank 777: 16548 of 23669 
stonewall rank 786: 16525 of 23669 
stonewall rank 821: 22969 of 23669 
stonewall rank 845: 16259 of 23669 
stonewall rank 849: 16713 of 23669 
stonewall rank 907: 15905 of 23669 
stonewall rank 922: 16179 of 23669 
stonewall rank 958: 15360 of 23669 
stonewall rank 13: 15799 of 23669 
stonewall rank 54: 15394 of 23669 
stonewall rank 20: 15552 of 23669 
stonewall rank 41: 16061 of 23669 
stonewall rank 29: 15724 of 23669 
stonewall rank 76: 15621 of 23669 
stonewall rank 86: 16232 of 23669 
stonewall rank 95: 16470 of 23669 
stonewall rank 125: 16094 of 23669 
stonewall rank 104: 20015 of 23669 
stonewall rank 141: 16334 of 23669 
stonewall rank 158: 15050 of 23669 
stonewall rank 147: 16458 of 23669 
stonewall rank 179: 15543 of 23669 
stonewall rank 202: 16355 of 23669 
stonewall rank 222: 21819 of 23669 
stonewall rank 212: 16062 of 23669 
stonewall rank 243: 20906 of 23669 
stonewall rank 252: 16103 of 23669 
stonewall rank 276: 22940 of 23669 
stonewall rank 287: 15514 of 23669 
stonewall rank 310: 16664 of 23669 
stonewall rank 264: 16100 of 23669 
stonewall rank 332: 16427 of 23669 
stonewall rank 340: 16444 of 23669 
stonewall rank 350: 15386 of 23669 
stonewall rank 361: 15976 of 23669 
stonewall rank 369: 16213 of 23669 
stonewall rank 392: 16267 of 23669 
stonewall rank 380: 15768 of 23669 
stonewall rank 408: 16624 of 23669 
stonewall rank 407: 16090 of 23669 
stonewall rank 426: 16575 of 23669 
stonewall rank 436: 16822 of 23669 
stonewall rank 469: 16173 of 23669 
stonewall rank 458: 16014 of 23669 
stonewall rank 479: 16253 of 23669 
stonewall rank 489: 15399 of 23669 
stonewall rank 511: 15947 of 23669 
stonewall rank 497: 16685 of 23669 
stonewall rank 523: 15824 of 23669 
stonewall rank 530: 18239 of 23669 
stonewall rank 542: 16334 of 23669 
stonewall rank 554: 17427 of 23669 
stonewall rank 566: 20128 of 23669 
stonewall rank 586: 15968 of 23669 
stonewall rank 607: 16202 of 23669 
stonewall rank 596: 16062 of 23669 
stonewall rank 651: 20008 of 23669 
stonewall rank 660: 15047 of 23669 
stonewall rank 665: 16460 of 23669 
stonewall rank 694: 16078 of 23669 
stonewall rank 725: 15627 of 23669 
stonewall rank 731: 15844 of 23669 
stonewall rank 759: 16142 of 23669 
stonewall rank 764: 22211 of 23669 
stonewall rank 781: 16598 of 23669 
stonewall rank 787: 16228 of 23669 
stonewall rank 822: 23615 of 23669 
stonewall rank 846: 16377 of 23669 
stonewall rank 850: 16595 of 23669 
stonewall rank 857: 15771 of 23669 
stonewall rank 908: 15993 of 23669 
stonewall rank 941: 16826 of 23669 
stonewall rank 959: 15359 of 23669 
stonewall rank 12: 15026 of 23669 
stonewall rank 55: 15039 of 23669 
stonewall rank 21: 15903 of 23669 
stonewall rank 43: 16138 of 23669 
stonewall rank 24: 16002 of 23669 
stonewall rank 58: 18116 of 23669 
stonewall rank 78: 15617 of 23669 
stonewall rank 87: 15824 of 23669 
stonewall rank 88: 16266 of 23669 
stonewall rank 113: 15129 of 23669 
stonewall rank 105: 20239 of 23669 
stonewall rank 136: 16477 of 23669 
stonewall rank 159: 15471 of 23669 
stonewall rank 149: 16169 of 23669 
stonewall rank 180: 15015 of 23669 
stonewall rank 170: 20416 of 23669 
stonewall rank 187: 16124 of 23669 
stonewall rank 203: 16312 of 23669 
stonewall rank 223: 21246 of 23669 
stonewall rank 213: 16027 of 23669 
stonewall rank 232: 16002 of 23669 
stonewall rank 244: 20501 of 23669 
stonewall rank 254: 16294 of 23669 
stonewall rank 277: 23125 of 23669 
stonewall rank 280: 15512 of 23669 
stonewall rank 311: 16635 of 23669 
stonewall rank 265: 16575 of 23669 
stonewall rank 333: 16641 of 23669 
stonewall rank 341: 16110 of 23669 
stonewall rank 351: 15744 of 23669 
stonewall rank 362: 16343 of 23669 
stonewall rank 373: 16947 of 23669 
stonewall rank 396: 16555 of 23669 
stonewall rank 381: 16003 of 23669 
stonewall rank 412: 16697 of 23669 
stonewall rank 404: 16110 of 23669 
stonewall rank 427: 16825 of 23669 
stonewall rank 437: 16719 of 23669 
stonewall rank 464: 16165 of 23669 
stonewall rank 459: 15922 of 23669 
stonewall rank 476: 16673 of 23669 
stonewall rank 490: 15264 of 23669 
stonewall rank 508: 16132 of 23669 
stonewall rank 498: 16557 of 23669 
stonewall rank 525: 15650 of 23669 
stonewall rank 532: 17994 of 23669 
stonewall rank 543: 16350 of 23669 
stonewall rank 555: 17473 of 23669 
stonewall rank 567: 20288 of 23669 
stonewall rank 587: 16432 of 23669 
stonewall rank 600: 16916 of 23669 
stonewall rank 597: 16311 of 23669 
stonewall rank 620: 21096 of 23669 
stonewall rank 652: 20020 of 23669 
stonewall rank 662: 15354 of 23669 
stonewall rank 669: 16719 of 23669 
stonewall rank 690: 15853 of 23669 
stonewall rank 727: 15545 of 23669 
stonewall rank 735: 15665 of 23669 
stonewall rank 753: 16120 of 23669 
stonewall rank 765: 22094 of 23669 
stonewall rank 778: 16195 of 23669 
stonewall rank 788: 16223 of 23669 
stonewall rank 793: 17101 of 23669 
stonewall rank 823: 23455 of 23669 
stonewall rank 847: 16448 of 23669 
stonewall rank 851: 16499 of 23669 
stonewall rank 858: 16159 of 23669 
stonewall rank 900: 15860 of 23669 
stonewall rank 893: 14861 of 23669 
stonewall rank 916: 16581 of 23669 
stonewall rank 909: 15957 of 23669 
stonewall rank 933: 15328 of 23669 
stonewall rank 940: 16156 of 23669 
stonewall rank 953: 15448 of 23669 
stonewall rank 14: 15240 of 23669 
stonewall rank 50: 15625 of 23669 
stonewall rank 22: 16345 of 23669 
stonewall rank 47: 16240 of 23669 
stonewall rank 25: 15858 of 23669 
stonewall rank 59: 17728 of 23669 
stonewall rank 79: 15369 of 23669 
stonewall rank 83: 16067 of 23669 
stonewall rank 89: 16574 of 23669 
stonewall rank 115: 15305 of 23669 
stonewall rank 106: 20452 of 23669 
stonewall rank 140: 16358 of 23669 
stonewall rank 153: 15047 of 23669 
stonewall rank 148: 16169 of 23669 
stonewall rank 183: 15470 of 23669 
stonewall rank 171: 20781 of 23669 
stonewall rank 186: 15794 of 23669 
stonewall rank 207: 16914 of 23669 
stonewall rank 216: 21202 of 23669 
stonewall rank 209: 15915 of 23669 
stonewall rank 233: 16123 of 23669 
stonewall rank 245: 21145 of 23669 
stonewall rank 255: 15762 of 23669 
stonewall rank 278: 22636 of 23669 
stonewall rank 281: 15095 of 23669 
stonewall rank 309: 16425 of 23669 
stonewall rank 268: 16638 of 23669 
stonewall rank 334: 16392 of 23669 
stonewall rank 338: 16004 of 23669 
stonewall rank 344: 15745 of 23669 
stonewall rank 363: 16003 of 23669 
stonewall rank 375: 16051 of 23669 
stonewall rank 398: 16423 of 23669 
stonewall rank 382: 16223 of 23669 
stonewall rank 413: 16452 of 23669 
stonewall rank 405: 16204 of 23669 
stonewall rank 425: 16461 of 23669 
stonewall rank 438: 16821 of 23669 
stonewall rank 446: 16567 of 23669 
stonewall rank 465: 15901 of 23669 
stonewall rank 457: 15408 of 23669 
stonewall rank 477: 16590 of 23669 
stonewall rank 491: 15426 of 23669 
stonewall rank 509: 15890 of 23669 
stonewall rank 499: 16432 of 23669 
stonewall rank 527: 16172 of 23669 
stonewall rank 534: 18011 of 23669 
stonewall rank 536: 16457 of 23669 
stonewall rank 556: 17029 of 23669 
stonewall rank 562: 20361 of 23669 
stonewall rank 588: 16461 of 23669 
stonewall rank 602: 16636 of 23669 
stonewall rank 598: 16268 of 23669 
stonewall rank 621: 21790 of 23669 
stonewall rank 625: 17591 of 23669 
stonewall rank 638: 15851 of 23669 
stonewall rank 653: 20163 of 23669 
stonewall rank 663: 15074 of 23669 
stonewall rank 671: 16912 of 23669 
stonewall rank 681: 15004 of 23669 
stonewall rank 691: 15845 of 23669 
stonewall rank 722: 15029 of 23669 
stonewall rank 714: 16330 of 23669 
stonewall rank 728: 16188 of 23669 
stonewall rank 745: 16328 of 23669 
stonewall rank 754: 16128 of 23669 
stonewall rank 779: 16114 of 23669 
stonewall rank 790: 16259 of 23669 
stonewall rank 794: 16631 of 23669 
stonewall rank 817: 23144 of 23669 
stonewall rank 842: 16722 of 23669 
stonewall rank 852: 16503 of 23669 
stonewall rank 859: 15901 of 23669 
stonewall rank 901: 16297 of 23669 
stonewall rank 894: 15120 of 23669 
stonewall rank 917: 16531 of 23669 
stonewall rank 910: 15620 of 23669 
stonewall rank 927: 16033 of 23669 
stonewall rank 934: 14942 of 23669 
stonewall rank 942: 17006 of 23669 
stonewall rank 954: 15379 of 23669 
stonewall rank 15: 15298 of 23669 
stonewall rank 52: 15217 of 23669 
stonewall rank 23: 15739 of 23669 
stonewall rank 46: 15808 of 23669 
stonewall rank 28: 15915 of 23669 
stonewall rank 62: 18099 of 23669 
stonewall rank 73: 14998 of 23669 
stonewall rank 82: 15737 of 23669 
stonewall rank 92: 16317 of 23669 
stonewall rank 121: 15816 of 23669 
stonewall rank 118: 15720 of 23669 
stonewall rank 107: 20196 of 23669 
stonewall rank 142: 16728 of 23669 
stonewall rank 154: 15515 of 23669 
stonewall rank 144: 15818 of 23669 
stonewall rank 181: 15476 of 23669 
stonewall rank 174: 20883 of 23669 
stonewall rank 185: 15633 of 23669 
stonewall rank 200: 16764 of 23669 
stonewall rank 217: 21207 of 23669 
stonewall rank 208: 16040 of 23669 
stonewall rank 234: 16172 of 23669 
stonewall rank 246: 21288 of 23669 
stonewall rank 249: 15716 of 23669 
stonewall rank 279: 22727 of 23669 
stonewall rank 284: 15265 of 23669 
stonewall rank 308: 16330 of 23669 
stonewall rank 270: 16616 of 23669 
stonewall rank 297: 20657 of 23669 
stonewall rank 335: 16270 of 23669 
stonewall rank 313: 15900 of 23669 
stonewall rank 339: 16019 of 23669 
stonewall rank 348: 15729 of 23669 
stonewall rank 364: 16396 of 23669 
stonewall rank 371: 16093 of 23669 
stonewall rank 399: 16638 of 23669 
stonewall rank 383: 15702 of 23669 
stonewall rank 414: 16735 of 23669 
stonewall rank 400: 16151 of 23669 
stonewall rank 430: 16760 of 23669 
stonewall rank 439: 16663 of 23669 
stonewall rank 447: 16283 of 23669 
stonewall rank 466: 16481 of 23669 
stonewall rank 460: 15713 of 23669 
stonewall rank 478: 16322 of 23669 
stonewall rank 493: 15289 of 23669 
stonewall rank 505: 15739 of 23669 
stonewall rank 500: 16741 of 23669 
stonewall rank 520: 15922 of 23669 
stonewall rank 528: 17987 of 23669 
stonewall rank 538: 16321 of 23669 
stonewall rank 557: 17251 of 23669 
stonewall rank 563: 19853 of 23669 
stonewall rank 590: 16106 of 23669 
stonewall rank 603: 16337 of 23669 
stonewall rank 599: 16103 of 23669 
stonewall rank 622: 21738 of 23669 
stonewall rank 626: 17147 of 23669 
stonewall rank 639: 15913 of 23669 
stonewall rank 654: 19980 of 23669 
stonewall rank 656: 14937 of 23669 
stonewall rank 668: 16483 of 23669 
stonewall rank 682: 15656 of 23669 
stonewall rank 724: 15052 of 23669 
stonewall rank 715: 16260 of 23669 
stonewall rank 732: 15868 of 23669 
stonewall rank 746: 16678 of 23669 
stonewall rank 756: 16061 of 23669 
stonewall rank 761: 22114 of 23669 
stonewall rank 780: 16214 of 23669 
stonewall rank 791: 16218 of 23669 
stonewall rank 795: 17028 of 23669 
stonewall rank 819: 23238 of 23669 
stonewall rank 843: 15937 of 23669 
stonewall rank 853: 16095 of 23669 
stonewall rank 862: 16234 of 23669 
stonewall rank 902: 15992 of 23669 
stonewall rank 895: 15595 of 23669 
stonewall rank 918: 16407 of 23669 
stonewall rank 911: 16011 of 23669 
stonewall rank 926: 16140 of 23669 
stonewall rank 935: 15462 of 23669 
stonewall rank 943: 16564 of 23669 
stonewall rank 955: 15377 of 23669 
stonewall rank 8: 15192 of 23669 
stonewall rank 48: 15388 of 23669 
stonewall rank 16: 15821 of 23669 
stonewall rank 40: 16040 of 23669 
stonewall rank 63: 18097 of 23669 
stonewall rank 72: 15730 of 23669 
stonewall rank 80: 15992 of 23669 
stonewall rank 93: 16264 of 23669 
stonewall rank 123: 16115 of 23669 
stonewall rank 119: 15206 of 23669 
stonewall rank 108: 19973 of 23669 
stonewall rank 143: 16340 of 23669 
stonewall rank 155: 14936 of 23669 
stonewall rank 176: 15722 of 23669 
stonewall rank 175: 20417 of 23669 
stonewall rank 189: 15681 of 23669 
stonewall rank 201: 16608 of 23669 
stonewall rank 220: 21099 of 23669 
stonewall rank 235: 16009 of 23669 
stonewall rank 247: 21174 of 23669 
stonewall rank 250: 15848 of 23669 
stonewall rank 272: 22572 of 23669 
stonewall rank 285: 15221 of 23669 
stonewall rank 304: 16405 of 23669 
stonewall rank 271: 16813 of 23669 
stonewall rank 298: 20973 of 23669 
stonewall rank 329: 16614 of 23669 
stonewall rank 318: 16445 of 23669 
stonewall rank 336: 15615 of 23669 
stonewall rank 349: 15964 of 23669 
stonewall rank 366: 15857 of 23669 
stonewall rank 372: 16541 of 23669 
stonewall rank 377: 15911 of 23669 
stonewall rank 415: 16392 of 23669 
stonewall rank 431: 16711 of 23669 
stonewall rank 434: 16327 of 23669 
stonewall rank 444: 16384 of 23669 
stonewall rank 467: 16222 of 23669 
stonewall rank 461: 15889 of 23669 
stonewall rank 472: 16484 of 23669 
stonewall rank 495: 15246 of 23669 
stonewall rank 506: 16096 of 23669 
stonewall rank 501: 16407 of 23669 
stonewall rank 524: 16009 of 23669 
stonewall rank 539: 16735 of 23669 
stonewall rank 558: 17286 of 23669 
stonewall rank 570: 16987 of 23669 
stonewall rank 591: 15867 of 23669 
stonewall rank 604: 16260 of 23669 
stonewall rank 592: 16098 of 23669 
stonewall rank 623: 21900 of 23669 
stonewall rank 627: 17500 of 23669 
stonewall rank 636: 16283 of 23669 
stonewall rank 655: 20332 of 23669 
stonewall rank 670: 16390 of 23669 
stonewall rank 683: 15379 of 23669 
stonewall rank 701: 15268 of 23669 
stonewall rank 726: 15446 of 23669 
stonewall rank 718: 15894 of 23669 
stonewall rank 733: 15980 of 23669 
stonewall rank 747: 16328 of 23669 
stonewall rank 758: 16125 of 23669 
stonewall rank 762: 22678 of 23669 
stonewall rank 782: 16405 of 23669 
stonewall rank 784: 16491 of 23669 
stonewall rank 798: 16878 of 23669 
stonewall rank 840: 16040 of 23669 
stonewall rank 854: 16598 of 23669 
stonewall rank 863: 16088 of 23669 
stonewall rank 903: 16157 of 23669 
stonewall rank 892: 15085 of 23669 
stonewall rank 919: 16506 of 23669 
stonewall rank 904: 16250 of 23669 
stonewall rank 932: 15242 of 23669 
stonewall rank 939: 16536 of 23669 
stonewall rank 949: 16181 of 23669 
stonewall rank 957: 15045 of 23669 
stonewall rank 44: 16344 of 23669 
stonewall rank 57: 18221 of 23669 
stonewall rank 122: 15801 of 23669 
stonewall rank 114: 15211 of 23669 
stonewall rank 109: 20207 of 23669 
stonewall rank 152: 15358 of 23669 
stonewall rank 172: 21008 of 23669 
stonewall rank 184: 15791 of 23669 
stonewall rank 204: 16128 of 23669 
stonewall rank 236: 16184 of 23669 
stonewall rank 240: 21257 of 23669 
stonewall rank 251: 16032 of 23669 
stonewall rank 299: 20831 of 23669 
stonewall rank 328: 16816 of 23669 
stonewall rank 319: 16286 of 23669 
stonewall rank 367: 16115 of 23669 
stonewall rank 374: 16143 of 23669 
stonewall rank 378: 16037 of 23669 
stonewall rank 428: 16431 of 23669 
stonewall rank 435: 16261 of 23669 
stonewall rank 445: 16406 of 23669 
stonewall rank 470: 16356 of 23669 
stonewall rank 456: 15751 of 23669 
stonewall rank 488: 15117 of 23669 
stonewall rank 507: 15903 of 23669 
stonewall rank 502: 16639 of 23669 
stonewall rank 526: 15465 of 23669 
stonewall rank 559: 16913 of 23669 
stonewall rank 560: 20361 of 23669 
stonewall rank 571: 16609 of 23669 
stonewall rank 584: 16185 of 23669 
stonewall rank 605: 16978 of 23669 
stonewall rank 616: 22047 of 23669 
stonewall rank 629: 17356 of 23669 
stonewall rank 637: 15807 of 23669 
stonewall rank 648: 20279 of 23669 
stonewall rank 666: 16707 of 23669 
stonewall rank 684: 15198 of 23669 
stonewall rank 697: 15012 of 23669 
stonewall rank 688: 15930 of 23669 
stonewall rank 720: 15055 of 23669 
stonewall rank 719: 16239 of 23669 
stonewall rank 734: 15974 of 23669 
stonewall rank 744: 16675 of 23669 
stonewall rank 752: 15968 of 23669 
stonewall rank 763: 22268 of 23669 
stonewall rank 783: 16185 of 23669 
stonewall rank 799: 17082 of 23669 
stonewall rank 825: 15158 of 23669 
stonewall rank 816: 22812 of 23669 
stonewall rank 841: 16737 of 23669 
stonewall rank 855: 16310 of 23669 
stonewall rank 861: 15872 of 23669 
stonewall rank 883: 16271 of 23669 
stonewall rank 877: 16772 of 23669 
stonewall rank 888: 15451 of 23669 
stonewall rank 925: 15925 of 23669 
stonewall rank 937: 16412 of 23669 
stonewall rank 951: 15947 of 23669 
stonewall rank 956: 15235 of 23669 
stonewall rank 56: 18060 of 23669 
stonewall rank 124: 16051 of 23669 
stonewall rank 116: 14955 of 23669 
stonewall rank 110: 19896 of 23669 
stonewall rank 173: 20960 of 23669 
stonewall rank 188: 15708 of 23669 
stonewall rank 205: 15995 of 23669 
stonewall rank 237: 15842 of 23669 
stonewall rank 248: 15944 of 23669 
stonewall rank 302: 20931 of 23669 
stonewall rank 314: 15826 of 23669 
stonewall rank 360: 16198 of 23669 
stonewall rank 370: 16702 of 23669 
stonewall rank 379: 15740 of 23669 
stonewall rank 429: 16599 of 23669 
stonewall rank 432: 16492 of 23669 
stonewall rank 442: 16229 of 23669 
stonewall rank 471: 15911 of 23669 
stonewall rank 492: 14819 of 23669 
stonewall rank 504: 15993 of 23669 
stonewall rank 503: 16572 of 23669 
stonewall rank 552: 17118 of 23669 
stonewall rank 572: 16990 of 23669 
stonewall rank 617: 21654 of 23669 
stonewall rank 631: 17674 of 23669 
stonewall rank 632: 16173 of 23669 
stonewall rank 667: 16704 of 23669 
stonewall rank 685: 15459 of 23669 
stonewall rank 698: 15220 of 23669 
stonewall rank 712: 16035 of 23669 
stonewall rank 748: 16937 of 23669 
stonewall rank 760: 22677 of 23669 
stonewall rank 776: 16013 of 23669 
stonewall rank 792: 17284 of 23669 
stonewall rank 827: 15098 of 23669 
stonewall rank 848: 16217 of 23669 
stonewall rank 856: 16089 of 23669 
stonewall rank 885: 16486 of 23669 
stonewall rank 878: 16530 of 23669 
stonewall rank 889: 15120 of 23669 
stonewall rank 920: 16043 of 23669 
stonewall rank 938: 16808 of 23669 
stonewall rank 952: 15400 of 23669 
stonewall rank 60: 17727 of 23669 
stonewall rank 126: 15843 of 23669 
stonewall rank 117: 15325 of 23669 
stonewall rank 111: 19348 of 23669 
stonewall rank 169: 20792 of 23669 
stonewall rank 190: 15593 of 23669 
stonewall rank 206: 16087 of 23669 
stonewall rank 238: 16503 of 23669 
stonewall rank 303: 20774 of 23669 
stonewall rank 315: 16112 of 23669 
stonewall rank 368: 16296 of 23669 
stonewall rank 376: 15968 of 23669 
stonewall rank 424: 17558 of 23669 
stonewall rank 443: 16217 of 23669 
stonewall rank 494: 15188 of 23669 
stonewall rank 496: 16209 of 23669 
stonewall rank 573: 16966 of 23669 
stonewall rank 618: 21300 of 23669 
stonewall rank 633: 15838 of 23669 
stonewall rank 664: 16647 of 23669 
stonewall rank 686: 15083 of 23669 
stonewall rank 699: 15136 of 23669 
stonewall rank 713: 15762 of 23669 
stonewall rank 749: 16477 of 23669 
stonewall rank 796: 16749 of 23669 
stonewall rank 828: 14999 of 23669 
stonewall rank 860: 15911 of 23669 
stonewall rank 881: 16319 of 23669 
stonewall rank 879: 16231 of 23669 
stonewall rank 890: 15376 of 23669 
stonewall rank 924: 16112 of 23669 
stonewall rank 936: 16565 of 23669 
stonewall rank 948: 16210 of 23669 
stonewall rank 61: 17988 of 23669 
stonewall rank 127: 15832 of 23669 
stonewall rank 112: 15113 of 23669 
stonewall rank 168: 19761 of 23669 
stonewall rank 191: 15938 of 23669 
stonewall rank 239: 15768 of 23669 
stonewall rank 296: 21090 of 23669 
stonewall rank 312: 16036 of 23669 
stonewall rank 440: 16212 of 23669 
stonewall rank 574: 16984 of 23669 
stonewall rank 619: 21006 of 23669 
stonewall rank 628: 17209 of 23669 
stonewall rank 634: 15846 of 23669 
stonewall rank 687: 15487 of 23669 
stonewall rank 700: 15268 of 23669 
stonewall rank 716: 15914 of 23669 
stonewall rank 750: 16401 of 23669 
stonewall rank 797: 17615 of 23669 
stonewall rank 813: 18245 of 23669 
stonewall rank 829: 15452 of 23669 
stonewall rank 872: 16416 of 23669 
stonewall rank 891: 15213 of 23669 
stonewall rank 950: 15847 of 23669 
stonewall rank 120: 15964 of 23669 
stonewall rank 300: 20715 of 23669 
stonewall rank 317: 16219 of 23669 
stonewall rank 441: 15977 of 23669 
stonewall rank 575: 16926 of 23669 
stonewall rank 630: 18065 of 23669 
stonewall rank 635: 16325 of 23669 
stonewall rank 680: 15435 of 23669 
stonewall rank 702: 15205 of 23669 
stonewall rank 717: 16264 of 23669 
stonewall rank 751: 16404 of 23669 
stonewall rank 830: 15473 of 23669 
stonewall rank 873: 16489 of 23669 
stonewall rank 301: 20558 of 23669 
stonewall rank 316: 15794 of 23669 
stonewall rank 569: 16829 of 23669 
stonewall rank 624: 17509 of 23669 
stonewall rank 703: 14969 of 23669 
stonewall rank 815: 18029 of 23669 
stonewall rank 831: 15343 of 23669 
stonewall rank 882: 16713 of 23669 
stonewall rank 874: 16905 of 23669 
stonewall rank 568: 16904 of 23669 
stonewall rank 696: 15263 of 23669 
stonewall rank 810: 17803 of 23669 
stonewall rank 824: 15847 of 23669 
stonewall rank 884: 16310 of 23669 
stonewall rank 875: 16236 of 23669 
stonewall rank 811: 18384 of 23669 
stonewall rank 826: 14788 of 23669 
stonewall rank 886: 16255 of 23669 
stonewall rank 876: 16598 of 23669 
stonewall rank 812: 17646 of 23669 
stonewall rank 887: 16028 of 23669 
stonewall rank 814: 17582 of 23669 
stonewall rank 880: 15950 of 23669 
stonewall rank 808: 18025 of 23669 
stonewall rank 809: 17779 of 23669 
stonewall rank 1: 22706 of 23669 
stonewall rank 65: 16283 of 23669 
stonewall rank 129: 15964 of 23669 
stonewall rank 193: 16910 of 23669 
stonewall rank 5: 22588 of 23669 
stonewall rank 69: 15853 of 23669 
stonewall rank 133: 15768 of 23669 
stonewall rank 196: 17430 of 23669 
stonewall rank 257: 16068 of 23669 
stonewall rank 6: 22606 of 23669 
stonewall rank 67: 16246 of 23669 
stonewall rank 135: 15860 of 23669 
stonewall rank 197: 17655 of 23669 
stonewall rank 260: 16227 of 23669 
stonewall rank 514: 15069 of 23669 
stonewall rank 7: 22663 of 23669 
stonewall rank 68: 15945 of 23669 
stonewall rank 132: 16028 of 23669 
stonewall rank 198: 16854 of 23669 
stonewall rank 261: 15761 of 23669 
stonewall rank 449: 15142 of 23669 
stonewall rank 515: 15434 of 23669 
stonewall rank 705: 20108 of 23669 
stonewall rank 4: 22210 of 23669 
stonewall rank 70: 16421 of 23669 
stonewall rank 134: 15613 of 23669 
stonewall rank 199: 17184 of 23669 
stonewall rank 262: 15754 of 23669 
stonewall rank 450: 15181 of 23669 
stonewall rank 516: 15048 of 23669 
stonewall rank 711: 19878 of 23669 
stonewall rank 929: 15334 of 23669 
stonewall rank 3: 22503 of 23669 
stonewall rank 263: 16287 of 23669 
stonewall rank 451: 15141 of 23669 
stonewall rank 517: 14918 of 23669 
stonewall rank 706: 19786 of 23669 
stonewall rank 930: 15242 of 23669 
stonewall rank 2: 22389 of 23669 
stonewall rank 71: 16102 of 23669 
stonewall rank 130: 15515 of 23669 
stonewall rank 194: 17289 of 23669 
stonewall rank 226: 16267 of 23669 
stonewall rank 452: 15101 of 23669 
stonewall rank 518: 15272 of 23669 
stonewall rank 707: 20340 of 23669 
stonewall rank 641: 16664 of 23669 
stonewall rank 931: 15690 of 23669 
stonewall rank 66: 16568 of 23669 
stonewall rank 131: 15809 of 23669 
stonewall rank 195: 17179 of 23669 
stonewall rank 227: 16283 of 23669 
stonewall rank 258: 15949 of 23669 
stonewall rank 321: 15294 of 23669 
stonewall rank 385: 15431 of 23669 
stonewall rank 453: 15099 of 23669 
stonewall rank 519: 15580 of 23669 
stonewall rank 577: 16646 of 23669 
stonewall rank 708: 20345 of 23669 
stonewall rank 769: 16675 of 23669 
stonewall rank 642: 16417 of 23669 
stonewall rank 897: 15774 of 23669 
stonewall rank 98: 16510 of 23669 
stonewall rank 259: 15667 of 23669 
stonewall rank 322: 15308 of 23669 
stonewall rank 386: 15714 of 23669 
stonewall rank 513: 14629 of 23669 
stonewall rank 578: 16242 of 23669 
stonewall rank 709: 20298 of 23669 
stonewall rank 770: 17065 of 23669 
stonewall rank 643: 16317 of 23669 
stonewall rank 899: 15806 of 23669 
stonewall rank 99: 16279 of 23669 
stonewall rank 225: 16244 of 23669 
stonewall rank 323: 15027 of 23669 
stonewall rank 387: 14874 of 23669 
stonewall rank 454: 15176 of 23669 
stonewall rank 579: 17051 of 23669 
stonewall rank 710: 19456 of 23669 
stonewall rank 771: 16421 of 23669 
stonewall rank 644: 16700 of 23669 
stonewall rank 898: 16266 of 23669 
stonewall rank 97: 16470 of 23669 
stonewall rank 192: 17088 of 23669 
stonewall rank 230: 16064 of 23669 
stonewall rank 324: 15305 of 23669 
stonewall rank 388: 15303 of 23669 
stonewall rank 455: 15234 of 23669 
stonewall rank 580: 16731 of 23669 
stonewall rank 774: 16858 of 23669 
stonewall rank 645: 16550 of 23669 
stonewall rank 833: 16769 of 23669 
stonewall rank 33: 16673 of 23669 
stonewall rank 64: 16209 of 23669 
stonewall rank 103: 16518 of 23669 
stonewall rank 161: 17027 of 23669 
stonewall rank 231: 16506 of 23669 
stonewall rank 325: 15106 of 23669 
stonewall rank 389: 15659 of 23669 
stonewall rank 448: 15322 of 23669 
stonewall rank 481: 16148 of 23669 
stonewall rank 581: 16542 of 23669 
stonewall rank 775: 16182 of 23669 
stonewall rank 646: 16626 of 23669 
stonewall rank 834: 16543 of 23669 
stonewall rank 944: 16004 of 23669 
stonewall rank 34: 16106 of 23669 
stonewall rank 100: 16319 of 23669 
stonewall rank 162: 16742 of 23669 
stonewall rank 224: 15869 of 23669 
stonewall rank 482: 16569 of 23669 
stonewall rank 582: 16713 of 23669 
stonewall rank 772: 16719 of 23669 
stonewall rank 647: 16640 of 23669 
stonewall rank 835: 16622 of 23669 
stonewall rank 928: 15413 of 23669 
stonewall rank 945: 15954 of 23669 
stonewall rank 35: 16149 of 23669 
stonewall rank 128: 15987 of 23669 
stonewall rank 163: 16051 of 23669 
stonewall rank 228: 16182 of 23669 
stonewall rank 326: 15238 of 23669 
stonewall rank 353: 20070 of 23669 
stonewall rank 390: 15103 of 23669 
stonewall rank 483: 16125 of 23669 
stonewall rank 583: 16649 of 23669 
stonewall rank 773: 16551 of 23669 
stonewall rank 837: 16570 of 23669 
stonewall rank 946: 15934 of 23669 
stonewall rank 37: 16291 of 23669 
stonewall rank 101: 16305 of 23669 
stonewall rank 165: 16261 of 23669 
stonewall rank 327: 15407 of 23669 
stonewall rank 354: 19996 of 23669 
stonewall rank 391: 14844 of 23669 
stonewall rank 485: 16046 of 23669 
stonewall rank 739: 16206 of 23669 
stonewall rank 839: 16182 of 23669 
stonewall rank 947: 16182 of 23669 
stonewall rank 32: 16662 of 23669 
stonewall rank 102: 16603 of 23669 
stonewall rank 164: 16812 of 23669 
stonewall rank 320: 15416 of 23669 
stonewall rank 355: 20010 of 23669 
stonewall rank 480: 16278 of 23669 
stonewall rank 740: 16197 of 23669 
stonewall rank 836: 16672 of 23669 
stonewall rank 96: 16564 of 23669 
stonewall rank 229: 15874 of 23669 
stonewall rank 359: 19983 of 23669 
stonewall rank 610: 16195 of 23669 
stonewall rank 704: 20389 of 23669 
stonewall rank 741: 16396 of 23669 
stonewall rank 838: 16226 of 23669 
stonewall rank 913: 16607 of 23669 
stonewall rank 36: 16267 of 23669 
stonewall rank 166: 15717 of 23669 
stonewall rank 352: 20721 of 23669 
stonewall rank 417: 16267 of 23669 
stonewall rank 484: 16228 of 23669 
stonewall rank 611: 16333 of 23669 
stonewall rank 673: 15681 of 23669 
stonewall rank 742: 15834 of 23669 
stonewall rank 912: 16661 of 23669 
stonewall rank 38: 16690 of 23669 
stonewall rank 167: 16488 of 23669 
stonewall rank 384: 15334 of 23669 
stonewall rank 419: 16295 of 23669 
stonewall rank 486: 15880 of 23669 
stonewall rank 609: 16391 of 23669 
stonewall rank 672: 16157 of 23669 
stonewall rank 743: 15973 of 23669 
stonewall rank 640: 16161 of 23669 
stonewall rank 896: 16184 of 23669 
stonewall rank 914: 16628 of 23669 
stonewall rank 39: 15837 of 23669 
stonewall rank 160: 16459 of 23669 
stonewall rank 289: 15959 of 23669 
stonewall rank 356: 20078 of 23669 
stonewall rank 418: 16240 of 23669 
stonewall rank 487: 16038 of 23669 
stonewall rank 613: 15968 of 23669 
stonewall rank 674: 16116 of 23669 
stonewall rank 736: 16133 of 23669 
stonewall rank 832: 16170 of 23669 
stonewall rank 915: 16753 of 23669 
stonewall rank 290: 15971 of 23669 
stonewall rank 357: 19888 of 23669 
stonewall rank 421: 16414 of 23669 
stonewall rank 546: 15937 of 23669 
stonewall rank 675: 15721 of 23669 
stonewall rank 737: 16355 of 23669 
stonewall rank 864: 15037 of 23669 
stonewall rank 291: 15850 of 23669 
stonewall rank 358: 20648 of 23669 
stonewall rank 422: 16155 of 23669 
stonewall rank 547: 16142 of 23669 
stonewall rank 676: 16081 of 23669 
stonewall rank 738: 16272 of 23669 
stonewall rank 865: 15297 of 23669 
stonewall rank 292: 16183 of 23669 
stonewall rank 550: 16034 of 23669 
stonewall rank 677: 15816 of 23669 
stonewall rank 866: 15003 of 23669 
stonewall rank 293: 16009 of 23669 
stonewall rank 551: 16237 of 23669 
stonewall rank 678: 15763 of 23669 
stonewall rank 867: 15255 of 23669 
stonewall rank 294: 15995 of 23669 
stonewall rank 423: 15881 of 23669 
stonewall rank 548: 16029 of 23669 
stonewall rank 679: 15845 of 23669 
stonewall rank 868: 15190 of 23669 
stonewall rank 295: 15552 of 23669 
stonewall rank 416: 15792 of 23669 
stonewall rank 549: 15961 of 23669 
stonewall rank 869: 15044 of 23669 
stonewall rank 288: 16444 of 23669 
stonewall rank 420: 16032 of 23669 
stonewall rank 545: 15913 of 23669 
stonewall rank 802: 16199 of 23669 
stonewall rank 870: 15452 of 23669 
stonewall rank 768: 16443 of 23669 
stonewall rank 803: 15941 of 23669 
stonewall rank 871: 15247 of 23669 
stonewall rank 256: 15922 of 23669 
stonewall rank 800: 15716 of 23669 
stonewall rank 801: 16358 of 23669 
stonewall rank 804: 16220 of 23669 
stonewall rank 805: 16285 of 23669 
stonewall rank 544: 16220 of 23669 
stonewall rank 806: 16433 of 23669 
stonewall rank 807: 15895 of 23669 
stonewall rank 512: 14779 of 23669 
stonewall rank 576: 16353 of 23669 
stonewall rank 608: 16504 of 23669 
stonewall rank 612: 16266 of 23669 
stonewall rank 614: 16148 of 23669 
stonewall rank 615: 16453 of 23669 
Continue stonewall hit min: 14629 max: 23669 avg: 16717.6 
stonewall rank 0: 23169 of 23669 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :     260025.007     260025.007     260025.007          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          1.716          1.716          1.716          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/11/2018 16:45:07 --
mdtest_hard_delete
-- started at 11/11/2018 17:41:29 --

mdtest-1.9.3 was launched with 960 total task(s) on 120 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-r" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_hard" "-n" "65000" "-x" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_hard-stonewall"
Path: /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26
FS: 144.4 TiB   Used FS: 39.3%   Inodes: 2400.0 Mi   Used Inodes: 0.7%

960 tasks, 62400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :      25244.621      25244.621      25244.621          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          2.345          2.345          2.345          0.000

-- finished at 11/11/2018 17:53:00 --
mdtest_hard_read
-- started at 11/11/2018 17:37:15 --

mdtest-1.9.3 was launched with 960 total task(s) on 120 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-E" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_hard" "-n" "65000" "-x" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_hard-stonewall"
Path: /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26
FS: 144.4 TiB   Used FS: 39.3%   Inodes: 2400.0 Mi   Used Inodes: 0.7%

960 tasks, 62400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :      68715.973      68715.973      68715.973          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/11/2018 17:41:29 --
mdtest_hard_stat
-- started at 11/11/2018 17:25:27 --

mdtest-1.9.3 was launched with 960 total task(s) on 120 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-T" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_hard" "-n" "65000" "-x" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_hard-stonewall"
Path: /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26
FS: 144.4 TiB   Used FS: 39.3%   Inodes: 2400.0 Mi   Used Inodes: 1.6%

960 tasks, 62400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :     178785.571     178785.571     178785.571          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/11/2018 17:27:05 --
mdtest_hard_write
-- started at 11/11/2018 16:50:21 --

mdtest-1.9.3 was launched with 960 total task(s) on 120 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-C" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_hard" "-n" "65000" "-x" "/lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26/mdt_hard-stonewall" "-W" "300"
Path: /lustre/gcp/io500/datafiles/io500.2018.11.11-16.28.26
FS: 144.4 TiB   Used FS: 39.3%   Inodes: 2400.0 Mi   Used Inodes: 0.9%

960 tasks, 62400000 files
stonewall rank 77: 15094 of 18147 
stonewall rank 262: 15556 of 18147 
stonewall rank 289: 15205 of 18147 
stonewall rank 321: 15303 of 18147 
stonewall rank 582: 16090 of 18147 
stonewall rank 589: 15903 of 18147 
stonewall rank 594: 15676 of 18147 
stonewall rank 642: 16024 of 18147 
stonewall rank 650: 16714 of 18147 
stonewall rank 774: 15929 of 18147 
stonewall rank 785: 16247 of 18147 
stonewall rank 833: 15913 of 18147 
stonewall rank 849: 15532 of 18147 
stonewall rank 2: 16590 of 18147 
stonewall rank 17: 15155 of 18147 
stonewall rank 26: 15318 of 18147 
stonewall rank 33: 14871 of 18147 
stonewall rank 49: 15400 of 18147 
stonewall rank 65: 15187 of 18147 
stonewall rank 58: 15479 of 18147 
stonewall rank 81: 14938 of 18147 
stonewall rank 90: 16152 of 18147 
stonewall rank 98: 15858 of 18147 
stonewall rank 122: 15709 of 18147 
stonewall rank 113: 14744 of 18147 
stonewall rank 138: 15579 of 18147 
stonewall rank 145: 15829 of 18147 
stonewall rank 154: 14155 of 18147 
stonewall rank 76: 15114 of 18147 
stonewall rank 166: 15279 of 18147 
stonewall rank 130: 14646 of 18147 
stonewall rank 177: 15426 of 18147 
stonewall rank 186: 14829 of 18147 
stonewall rank 193: 15950 of 18147 
stonewall rank 205: 15006 of 18147 
stonewall rank 218: 15787 of 18147 
stonewall rank 209: 15095 of 18147 
stonewall rank 225: 14675 of 18147 
stonewall rank 250: 15550 of 18147 
stonewall rank 241: 15548 of 18147 
stonewall rank 263: 15451 of 18147 
stonewall rank 273: 16948 of 18147 
stonewall rank 265: 15183 of 18147 
stonewall rank 294: 15257 of 18147 
stonewall rank 282: 14733 of 18147 
stonewall rank 326: 15380 of 18147 
stonewall rank 314: 15204 of 18147 
stonewall rank 305: 15272 of 18147 
stonewall rank 333: 15954 of 18147 
stonewall rank 337: 15688 of 18147 
stonewall rank 346: 15719 of 18147 
stonewall rank 358: 16831 of 18147 
stonewall rank 369: 15594 of 18147 
stonewall rank 378: 15190 of 18147 
stonewall rank 386: 14326 of 18147 
stonewall rank 394: 15926 of 18147 
stonewall rank 401: 16196 of 18147 
stonewall rank 410: 15573 of 18147 
stonewall rank 417: 15788 of 18147 
stonewall rank 433: 15935 of 18147 
stonewall rank 442: 15922 of 18147 
stonewall rank 461: 15360 of 18147 
stonewall rank 449: 15247 of 18147 
stonewall rank 465: 15862 of 18147 
stonewall rank 481: 15570 of 18147 
stonewall rank 506: 16111 of 18147 
stonewall rank 498: 15312 of 18147 
stonewall rank 514: 14649 of 18147 
stonewall rank 529: 16716 of 18147 
stonewall rank 538: 15820 of 18147 
stonewall rank 545: 15308 of 18147 
stonewall rank 561: 16570 of 18147 
stonewall rank 570: 16264 of 18147 
stonewall rank 583: 16077 of 18147 
stonewall rank 585: 15846 of 18147 
stonewall rank 595: 15656 of 18147 
stonewall rank 602: 15837 of 18147 
stonewall rank 629: 15776 of 18147 
stonewall rank 634: 15803 of 18147 
stonewall rank 643: 16108 of 18147 
stonewall rank 651: 16706 of 18147 
stonewall rank 666: 16201 of 18147 
stonewall rank 657: 14636 of 18147 
stonewall rank 698: 15154 of 18147 
stonewall rank 673: 14842 of 18147 
stonewall rank 705: 18116 of 18147 
stonewall rank 689: 15256 of 18147 
stonewall rank 610: 15815 of 18147 
stonewall rank 721: 14974 of 18147 
stonewall rank 717: 15368 of 18147 
stonewall rank 737: 16035 of 18147 
stonewall rank 730: 15885 of 18147 
stonewall rank 753: 15807 of 18147 
stonewall rank 775: 15974 of 18147 
stonewall rank 777: 15880 of 18147 
stonewall rank 789: 16280 of 18147 
stonewall rank 762: 15838 of 18147 
stonewall rank 794: 15719 of 18147 
stonewall rank 801: 15555 of 18147 
stonewall rank 817: 16732 of 18147 
stonewall rank 838: 16027 of 18147 
stonewall rank 850: 15507 of 18147 
stonewall rank 865: 14699 of 18147 
stonewall rank 890: 15375 of 18147 
stonewall rank 897: 15508 of 18147 
stonewall rank 913: 16057 of 18147 
stonewall rank 921: 15704 of 18147 
stonewall rank 937: 16071 of 18147 
stonewall rank 3: 16488 of 18147 
stonewall rank 21: 15187 of 18147 
stonewall rank 27: 15331 of 18147 
stonewall rank 42: 16015 of 18147 
stonewall rank 35: 14899 of 18147 
stonewall rank 50: 15365 of 18147 
stonewall rank 69: 15162 of 18147 
stonewall rank 59: 15621 of 18147 
stonewall rank 82: 14961 of 18147 
stonewall rank 91: 16124 of 18147 
stonewall rank 99: 15825 of 18147 
stonewall rank 123: 15636 of 18147 
stonewall rank 114: 14698 of 18147 
stonewall rank 139: 15564 of 18147 
stonewall rank 148: 15814 of 18147 
stonewall rank 155: 14187 of 18147 
stonewall rank 78: 15016 of 18147 
stonewall rank 167: 15334 of 18147 
stonewall rank 131: 14646 of 18147 
stonewall rank 178: 15504 of 18147 
stonewall rank 187: 14838 of 18147 
stonewall rank 198: 15970 of 18147 
stonewall rank 201: 15079 of 18147 
stonewall rank 219: 15832 of 18147 
stonewall rank 210: 15068 of 18147 
stonewall rank 226: 14698 of 18147 
stonewall rank 251: 15446 of 18147 
stonewall rank 243: 15576 of 18147 
stonewall rank 260: 15539 of 18147 
stonewall rank 277: 16880 of 18147 
stonewall rank 270: 15247 of 18147 
stonewall rank 295: 15193 of 18147 
stonewall rank 283: 14677 of 18147 
stonewall rank 301: 15736 of 18147 
stonewall rank 327: 15297 of 18147 
stonewall rank 315: 15210 of 18147 
stonewall rank 306: 15245 of 18147 
stonewall rank 329: 16029 of 18147 
stonewall rank 338: 15660 of 18147 
stonewall rank 347: 15642 of 18147 
stonewall rank 359: 16791 of 18147 
stonewall rank 370: 15594 of 18147 
stonewall rank 379: 15139 of 18147 
stonewall rank 387: 14455 of 18147 
stonewall rank 395: 15881 of 18147 
stonewall rank 404: 16092 of 18147 
stonewall rank 411: 15574 of 18147 
stonewall rank 418: 15780 of 18147 
stonewall rank 434: 16018 of 18147 
stonewall rank 443: 15965 of 18147 
stonewall rank 457: 15340 of 18147 
stonewall rank 455: 15222 of 18147 
stonewall rank 466: 15831 of 18147 
stonewall rank 474: 15584 of 18147 
stonewall rank 482: 15601 of 18147 
stonewall rank 507: 16073 of 18147 
stonewall rank 499: 15316 of 18147 
stonewall rank 515: 14597 of 18147 
stonewall rank 533: 16780 of 18147 
stonewall rank 539: 15831 of 18147 
stonewall rank 555: 16559 of 18147 
stonewall rank 546: 15096 of 18147 
stonewall rank 562: 16521 of 18147 
stonewall rank 571: 16208 of 18147 
stonewall rank 577: 16086 of 18147 
stonewall rank 588: 15914 of 18147 
stonewall rank 593: 15633 of 18147 
stonewall rank 603: 15918 of 18147 
stonewall rank 630: 15830 of 18147 
stonewall rank 635: 15789 of 18147 
stonewall rank 645: 15985 of 18147 
stonewall rank 653: 16634 of 18147 
stonewall rank 667: 16129 of 18147 
stonewall rank 661: 14600 of 18147 
stonewall rank 699: 15124 of 18147 
stonewall rank 674: 14786 of 18147 
stonewall rank 710: 18039 of 18147 
stonewall rank 690: 15238 of 18147 
stonewall rank 611: 15963 of 18147 
stonewall rank 722: 14942 of 18147 
stonewall rank 713: 15372 of 18147 
stonewall rank 741: 16075 of 18147 
stonewall rank 731: 15927 of 18147 
stonewall rank 754: 15831 of 18147 
stonewall rank 772: 15994 of 18147 
stonewall rank 791: 16181 of 18147 
stonewall rank 763: 15900 of 18147 
stonewall rank 795: 15810 of 18147 
stonewall rank 806: 15608 of 18147 
stonewall rank 818: 16714 of 18147 
stonewall rank 811: 15996 of 18147 
stonewall rank 839: 15983 of 18147 
stonewall rank 851: 15562 of 18147 
stonewall rank 866: 14726 of 18147 
stonewall rank 891: 15387 of 18147 
stonewall rank 901: 15519 of 18147 
stonewall rank 915: 16065 of 18147 
stonewall rank 931: 15398 of 18147 
stonewall rank 938: 16046 of 18147 
stonewall rank 1: 16575 of 18147 
stonewall rank 13: 15270 of 18147 
stonewall rank 23: 15203 of 18147 
stonewall rank 29: 15394 of 18147 
stonewall rank 43: 16036 of 18147 
stonewall rank 38: 14897 of 18147 
stonewall rank 51: 15418 of 18147 
stonewall rank 68: 15143 of 18147 
stonewall rank 61: 15559 of 18147 
stonewall rank 83: 14923 of 18147 
stonewall rank 93: 16123 of 18147 
stonewall rank 97: 15861 of 18147 
stonewall rank 106: 16118 of 18147 
stonewall rank 125: 15733 of 18147 
stonewall rank 115: 14730 of 18147 
stonewall rank 141: 15549 of 18147 
stonewall rank 149: 15867 of 18147 
stonewall rank 157: 14165 of 18147 
stonewall rank 79: 14992 of 18147 
stonewall rank 162: 15335 of 18147 
stonewall rank 133: 14586 of 18147 
stonewall rank 179: 15486 of 18147 
stonewall rank 170: 15818 of 18147 
stonewall rank 189: 14758 of 18147 
stonewall rank 199: 15899 of 18147 
stonewall rank 204: 15055 of 18147 
stonewall rank 222: 15805 of 18147 
stonewall rank 211: 15059 of 18147 
stonewall rank 234: 15360 of 18147 
stonewall rank 227: 14661 of 18147 
stonewall rank 253: 15550 of 18147 
stonewall rank 245: 15564 of 18147 
stonewall rank 261: 15566 of 18147 
stonewall rank 278: 16984 of 18147 
stonewall rank 271: 15234 of 18147 
stonewall rank 290: 15230 of 18147 
stonewall rank 285: 14663 of 18147 
stonewall rank 298: 15674 of 18147 
stonewall rank 324: 15400 of 18147 
stonewall rank 317: 15163 of 18147 
stonewall rank 307: 15244 of 18147 
stonewall rank 335: 16031 of 18147 
stonewall rank 339: 15769 of 18147 
stonewall rank 349: 15570 of 18147 
stonewall rank 353: 16815 of 18147 
stonewall rank 362: 15638 of 18147 
stonewall rank 371: 15624 of 18147 
stonewall rank 381: 15077 of 18147 
stonewall rank 389: 14320 of 18147 
stonewall rank 393: 15925 of 18147 
stonewall rank 405: 16146 of 18147 
stonewall rank 413: 15532 of 18147 
stonewall rank 419: 15810 of 18147 
stonewall rank 435: 15992 of 18147 
stonewall rank 427: 15599 of 18147 
stonewall rank 445: 15996 of 18147 
stonewall rank 463: 15410 of 18147 
stonewall rank 452: 15196 of 18147 
stonewall rank 467: 15913 of 18147 
stonewall rank 475: 15524 of 18147 
stonewall rank 483: 15644 of 18147 
stonewall rank 490: 14806 of 18147 
stonewall rank 509: 16009 of 18147 
stonewall rank 500: 15249 of 18147 
stonewall rank 513: 14646 of 18147 
stonewall rank 535: 16728 of 18147 
stonewall rank 525: 15148 of 18147 
stonewall rank 541: 15801 of 18147 
stonewall rank 557: 16513 of 18147 
stonewall rank 547: 15132 of 18147 
stonewall rank 563: 16611 of 18147 
stonewall rank 573: 16171 of 18147 
stonewall rank 580: 16052 of 18147 
stonewall rank 590: 15847 of 18147 
stonewall rank 597: 15630 of 18147 
stonewall rank 605: 15778 of 18147 
stonewall rank 631: 15699 of 18147 
stonewall rank 637: 15749 of 18147 
stonewall rank 641: 16124 of 18147 
stonewall rank 649: 16631 of 18147 
stonewall rank 669: 16164 of 18147 
stonewall rank 663: 14654 of 18147 
stonewall rank 685: 14668 of 18147 
stonewall rank 701: 15137 of 18147 
stonewall rank 678: 14824 of 18147 
stonewall rank 711: 18051 of 18147 
stonewall rank 691: 15322 of 18147 
stonewall rank 609: 15890 of 18147 
stonewall rank 723: 14976 of 18147 
stonewall rank 718: 15360 of 18147 
stonewall rank 742: 16014 of 18147 
stonewall rank 734: 15835 of 18147 
stonewall rank 746: 16171 of 18147 
stonewall rank 755: 15838 of 18147 
stonewall rank 618: 16428 of 18147 
stonewall rank 773: 15943 of 18147 
stonewall rank 788: 16258 of 18147 
stonewall rank 765: 15849 of 18147 
stonewall rank 797: 15823 of 18147 
stonewall rank 807: 15548 of 18147 
stonewall rank 819: 16766 of 18147 
stonewall rank 813: 15925 of 18147 
stonewall rank 826: 15152 of 18147 
stonewall rank 836: 15997 of 18147 
stonewall rank 853: 15583 of 18147 
stonewall rank 867: 14653 of 18147 
stonewall rank 881: 16282 of 18147 
stonewall rank 893: 15428 of 18147 
stonewall rank 898: 15498 of 18147 
stonewall rank 914: 16039 of 18147 
stonewall rank 922: 15794 of 18147 
stonewall rank 930: 15289 of 18147 
stonewall rank 939: 16100 of 18147 
stonewall rank 7: 16547 of 18147 
stonewall rank 20: 15158 of 18147 
stonewall rank 28: 15416 of 18147 
stonewall rank 45: 16047 of 18147 
stonewall rank 39: 14904 of 18147 
stonewall rank 53: 15339 of 18147 
stonewall rank 70: 15246 of 18147 
stonewall rank 60: 15533 of 18147 
stonewall rank 85: 14927 of 18147 
stonewall rank 92: 16134 of 18147 
stonewall rank 102: 15862 of 18147 
stonewall rank 107: 16109 of 18147 
stonewall rank 124: 15678 of 18147 
stonewall rank 116: 14745 of 18147 
stonewall rank 137: 15588 of 18147 
stonewall rank 150: 15833 of 18147 
stonewall rank 158: 14231 of 18147 
stonewall rank 73: 15051 of 18147 
stonewall rank 163: 15315 of 18147 
stonewall rank 129: 14601 of 18147 
stonewall rank 181: 15457 of 18147 
stonewall rank 171: 15898 of 18147 
stonewall rank 188: 14841 of 18147 
stonewall rank 196: 15955 of 18147 
stonewall rank 206: 15073 of 18147 
stonewall rank 223: 15782 of 18147 
stonewall rank 213: 15075 of 18147 
stonewall rank 235: 15351 of 18147 
stonewall rank 228: 14733 of 18147 
stonewall rank 252: 15510 of 18147 
stonewall rank 247: 15590 of 18147 
stonewall rank 257: 15510 of 18147 
stonewall rank 279: 16967 of 18147 
stonewall rank 291: 15227 of 18147 
stonewall rank 287: 14591 of 18147 
stonewall rank 299: 15696 of 18147 
stonewall rank 325: 15344 of 18147 
stonewall rank 316: 15200 of 18147 
stonewall rank 310: 15323 of 18147 
stonewall rank 334: 16010 of 18147 
stonewall rank 341: 15640 of 18147 
stonewall rank 348: 15643 of 18147 
stonewall rank 354: 16865 of 18147 
stonewall rank 363: 15616 of 18147 
stonewall rank 373: 15519 of 18147 
stonewall rank 380: 15043 of 18147 
stonewall rank 385: 14377 of 18147 
stonewall rank 397: 15885 of 18147 
stonewall rank 406: 16147 of 18147 
stonewall rank 412: 15499 of 18147 
stonewall rank 422: 15791 of 18147 
stonewall rank 437: 16046 of 18147 
stonewall rank 429: 15600 of 18147 
stonewall rank 444: 15944 of 18147 
stonewall rank 460: 15420 of 18147 
stonewall rank 453: 15223 of 18147 
stonewall rank 470: 15882 of 18147 
stonewall rank 477: 15539 of 18147 
stonewall rank 486: 15641 of 18147 
stonewall rank 491: 14861 of 18147 
stonewall rank 508: 16119 of 18147 
stonewall rank 501: 15260 of 18147 
stonewall rank 518: 14663 of 18147 
stonewall rank 532: 16794 of 18147 
stonewall rank 542: 15797 of 18147 
stonewall rank 554: 16465 of 18147 
stonewall rank 550: 15170 of 18147 
stonewall rank 565: 16533 of 18147 
stonewall rank 572: 16222 of 18147 
stonewall rank 581: 16023 of 18147 
stonewall rank 591: 15912 of 18147 
stonewall rank 599: 15647 of 18147 
stonewall rank 604: 15885 of 18147 
stonewall rank 625: 15744 of 18147 
stonewall rank 636: 15832 of 18147 
stonewall rank 644: 16082 of 18147 
stonewall rank 648: 16677 of 18147 
stonewall rank 668: 16211 of 18147 
stonewall rank 660: 14615 of 18147 
stonewall rank 682: 14622 of 18147 
stonewall rank 702: 15114 of 18147 
stonewall rank 679: 14841 of 18147 
stonewall rank 709: 18037 of 18147 
stonewall rank 693: 15272 of 18147 
stonewall rank 615: 15829 of 18147 
stonewall rank 725: 14916 of 18147 
stonewall rank 719: 15319 of 18147 
stonewall rank 743: 16064 of 18147 
stonewall rank 735: 15919 of 18147 
stonewall rank 747: 16169 of 18147 
stonewall rank 756: 15774 of 18147 
stonewall rank 619: 16384 of 18147 
stonewall rank 769: 16032 of 18147 
stonewall rank 790: 16243 of 18147 
stonewall rank 764: 15848 of 18147 
stonewall rank 796: 15716 of 18147 
stonewall rank 803: 15598 of 18147 
stonewall rank 821: 16846 of 18147 
stonewall rank 810: 16006 of 18147 
stonewall rank 827: 15140 of 18147 
stonewall rank 837: 15939 of 18147 
stonewall rank 854: 15540 of 18147 
stonewall rank 870: 14681 of 18147 
stonewall rank 882: 16306 of 18147 
stonewall rank 892: 15397 of 18147 
stonewall rank 899: 15543 of 18147 
stonewall rank 917: 15962 of 18147 
stonewall rank 923: 15777 of 18147 
stonewall rank 941: 16048 of 18147 
stonewall rank 947: 16044 of 18147 
stonewall rank 6: 16560 of 18147 
stonewall rank 22: 15183 of 18147 
stonewall rank 30: 15464 of 18147 
stonewall rank 36: 14930 of 18147 
stonewall rank 55: 15313 of 18147 
stonewall rank 71: 15195 of 18147 
stonewall rank 62: 15535 of 18147 
stonewall rank 84: 14921 of 18147 
stonewall rank 94: 16096 of 18147 
stonewall rank 103: 15841 of 18147 
stonewall rank 109: 16138 of 18147 
stonewall rank 126: 15663 of 18147 
stonewall rank 117: 14742 of 18147 
stonewall rank 140: 15667 of 18147 
stonewall rank 151: 15827 of 18147 
stonewall rank 159: 14112 of 18147 
stonewall rank 161: 15313 of 18147 
stonewall rank 132: 14600 of 18147 
stonewall rank 180: 15457 of 18147 
stonewall rank 173: 15938 of 18147 
stonewall rank 190: 14763 of 18147 
stonewall rank 197: 15983 of 18147 
stonewall rank 207: 15176 of 18147 
stonewall rank 220: 15848 of 18147 
stonewall rank 215: 15150 of 18147 
stonewall rank 237: 15311 of 18147 
stonewall rank 229: 14686 of 18147 
stonewall rank 254: 15490 of 18147 
stonewall rank 242: 15585 of 18147 
stonewall rank 258: 15557 of 18147 
stonewall rank 274: 16905 of 18147 
stonewall rank 292: 15197 of 18147 
stonewall rank 284: 14697 of 18147 
stonewall rank 322: 15355 of 18147 
stonewall rank 318: 15204 of 18147 
stonewall rank 311: 15289 of 18147 
stonewall rank 332: 15910 of 18147 
stonewall rank 343: 15748 of 18147 
stonewall rank 350: 15644 of 18147 
stonewall rank 355: 16757 of 18147 
stonewall rank 365: 15651 of 18147 
stonewall rank 374: 15663 of 18147 
stonewall rank 382: 15076 of 18147 
stonewall rank 391: 14344 of 18147 
stonewall rank 399: 15952 of 18147 
stonewall rank 407: 16178 of 18147 
stonewall rank 414: 15668 of 18147 
stonewall rank 423: 15750 of 18147 
stonewall rank 438: 15985 of 18147 
stonewall rank 426: 15512 of 18147 
stonewall rank 446: 15965 of 18147 
stonewall rank 462: 15388 of 18147 
stonewall rank 454: 15227 of 18147 
stonewall rank 471: 15832 of 18147 
stonewall rank 476: 15540 of 18147 
stonewall rank 487: 15583 of 18147 
stonewall rank 493: 14850 of 18147 
stonewall rank 510: 16054 of 18147 
stonewall rank 502: 15368 of 18147 
stonewall rank 519: 14579 of 18147 
stonewall rank 534: 16787 of 18147 
stonewall rank 543: 15763 of 18147 
stonewall rank 551: 15165 of 18147 
stonewall rank 567: 16577 of 18147 
stonewall rank 574: 16178 of 18147 
stonewall rank 579: 16047 of 18147 
stonewall rank 586: 15909 of 18147 
stonewall rank 596: 15711 of 18147 
stonewall rank 606: 15806 of 18147 
stonewall rank 626: 15668 of 18147 
stonewall rank 638: 15810 of 18147 
stonewall rank 646: 16090 of 18147 
stonewall rank 652: 16601 of 18147 
stonewall rank 670: 16109 of 18147 
stonewall rank 662: 14595 of 18147 
stonewall rank 683: 14673 of 18147 
stonewall rank 703: 15149 of 18147 
stonewall rank 675: 14819 of 18147 
stonewall rank 706: 18007 of 18147 
stonewall rank 692: 15327 of 18147 
stonewall rank 614: 15803 of 18147 
stonewall rank 727: 14934 of 18147 
stonewall rank 716: 15354 of 18147 
stonewall rank 739: 16039 of 18147 
stonewall rank 732: 15892 of 18147 
stonewall rank 749: 16181 of 18147 
stonewall rank 757: 15784 of 18147 
stonewall rank 621: 16453 of 18147 
stonewall rank 771: 15933 of 18147 
stonewall rank 786: 16214 of 18147 
stonewall rank 766: 15863 of 18147 
stonewall rank 798: 15736 of 18147 
stonewall rank 802: 15611 of 18147 
stonewall rank 822: 16824 of 18147 
stonewall rank 834: 15962 of 18147 
stonewall rank 858: 15057 of 18147 
stonewall rank 855: 15594 of 18147 
stonewall rank 871: 14684 of 18147 
stonewall rank 883: 16252 of 18147 
stonewall rank 894: 15371 of 18147 
stonewall rank 900: 15558 of 18147 
stonewall rank 925: 15782 of 18147 
stonewall rank 942: 16087 of 18147 
stonewall rank 945: 16068 of 18147 
stonewall rank 4: 16529 of 18147 
stonewall rank 12: 15225 of 18147 
stonewall rank 18: 15163 of 18147 
stonewall rank 31: 15349 of 18147 
stonewall rank 44: 16025 of 18147 
stonewall rank 37: 14940 of 18147 
stonewall rank 52: 15460 of 18147 
stonewall rank 67: 15160 of 18147 
stonewall rank 63: 15484 of 18147 
stonewall rank 86: 14984 of 18147 
stonewall rank 95: 16123 of 18147 
stonewall rank 100: 15896 of 18147 
stonewall rank 108: 16109 of 18147 
stonewall rank 127: 15679 of 18147 
stonewall rank 118: 14775 of 18147 
stonewall rank 142: 15593 of 18147 
stonewall rank 146: 15814 of 18147 
stonewall rank 156: 14197 of 18147 
stonewall rank 72: 15024 of 18147 
stonewall rank 164: 15226 of 18147 
stonewall rank 134: 14575 of 18147 
stonewall rank 182: 15460 of 18147 
stonewall rank 172: 15884 of 18147 
stonewall rank 191: 14786 of 18147 
stonewall rank 194: 15949 of 18147 
stonewall rank 200: 15108 of 18147 
stonewall rank 221: 15873 of 18147 
stonewall rank 212: 15087 of 18147 
stonewall rank 236: 15338 of 18147 
stonewall rank 230: 14637 of 18147 
stonewall rank 255: 15551 of 18147 
stonewall rank 244: 15526 of 18147 
stonewall rank 259: 15507 of 18147 
stonewall rank 275: 17042 of 18147 
stonewall rank 266: 15174 of 18147 
stonewall rank 293: 15168 of 18147 
stonewall rank 286: 14680 of 18147 
stonewall rank 302: 15754 of 18147 
stonewall rank 323: 15348 of 18147 
stonewall rank 319: 15181 of 18147 
stonewall rank 309: 15393 of 18147 
stonewall rank 328: 16015 of 18147 
stonewall rank 340: 15694 of 18147 
stonewall rank 351: 15608 of 18147 
stonewall rank 356: 16810 of 18147 
stonewall rank 364: 15613 of 18147 
stonewall rank 375: 15628 of 18147 
stonewall rank 383: 15101 of 18147 
stonewall rank 388: 14364 of 18147 
stonewall rank 398: 15906 of 18147 
stonewall rank 402: 16178 of 18147 
stonewall rank 415: 15531 of 18147 
stonewall rank 420: 15802 of 18147 
stonewall rank 439: 16000 of 18147 
stonewall rank 428: 15537 of 18147 
stonewall rank 447: 15929 of 18147 
stonewall rank 458: 15348 of 18147 
stonewall rank 450: 15241 of 18147 
stonewall rank 468: 15864 of 18147 
stonewall rank 478: 15612 of 18147 
stonewall rank 484: 15667 of 18147 
stonewall rank 492: 14825 of 18147 
stonewall rank 511: 16021 of 18147 
stonewall rank 503: 15298 of 18147 
stonewall rank 516: 14640 of 18147 
stonewall rank 530: 16826 of 18147 
stonewall rank 524: 15096 of 18147 
stonewall rank 540: 15816 of 18147 
stonewall rank 556: 16556 of 18147 
stonewall rank 548: 15159 of 18147 
stonewall rank 564: 16595 of 18147 
stonewall rank 575: 16180 of 18147 
stonewall rank 578: 16104 of 18147 
stonewall rank 587: 15883 of 18147 
stonewall rank 598: 15658 of 18147 
stonewall rank 607: 15837 of 18147 
stonewall rank 627: 15657 of 18147 
stonewall rank 639: 15795 of 18147 
stonewall rank 647: 16049 of 18147 
stonewall rank 654: 16580 of 18147 
stonewall rank 671: 16090 of 18147 
stonewall rank 658: 14661 of 18147 
stonewall rank 687: 14615 of 18147 
stonewall rank 700: 15146 of 18147 
stonewall rank 676: 14819 of 18147 
stonewall rank 707: 17966 of 18147 
stonewall rank 694: 15262 of 18147 
stonewall rank 612: 15985 of 18147 
stonewall rank 724: 14948 of 18147 
stonewall rank 712: 15393 of 18147 
stonewall rank 740: 16034 of 18147 
stonewall rank 733: 15873 of 18147 
stonewall rank 748: 16213 of 18147 
stonewall rank 758: 15826 of 18147 
stonewall rank 620: 16409 of 18147 
stonewall rank 770: 16029 of 18147 
stonewall rank 778: 15864 of 18147 
stonewall rank 787: 16182 of 18147 
stonewall rank 767: 15798 of 18147 
stonewall rank 799: 15820 of 18147 
stonewall rank 804: 15579 of 18147 
stonewall rank 823: 16754 of 18147 
stonewall rank 812: 16010 of 18147 
stonewall rank 835: 15917 of 18147 
stonewall rank 859: 15126 of 18147 
stonewall rank 852: 15499 of 18147 
stonewall rank 868: 14650 of 18147 
stonewall rank 885: 16226 of 18147 
stonewall rank 895: 15341 of 18147 
stonewall rank 902: 15499 of 18147 
stonewall rank 909: 15876 of 18147 
stonewall rank 918: 16065 of 18147 
stonewall rank 926: 15753 of 18147 
stonewall rank 929: 15317 of 18147 
stonewall rank 943: 16111 of 18147 
stonewall rank 946: 16090 of 18147 
stonewall rank 953: 15320 of 18147 
stonewall rank 5: 16643 of 18147 
stonewall rank 14: 15285 of 18147 
stonewall rank 19: 15156 of 18147 
stonewall rank 24: 15336 of 18147 
stonewall rank 46: 16000 of 18147 
stonewall rank 34: 14897 of 18147 
stonewall rank 54: 15359 of 18147 
stonewall rank 66: 15146 of 18147 
stonewall rank 56: 15533 of 18147 
stonewall rank 87: 14928 of 18147 
stonewall rank 88: 16041 of 18147 
stonewall rank 101: 15834 of 18147 
stonewall rank 110: 16106 of 18147 
stonewall rank 120: 15646 of 18147 
stonewall rank 119: 14738 of 18147 
stonewall rank 143: 15598 of 18147 
stonewall rank 147: 15853 of 18147 
stonewall rank 152: 14212 of 18147 
stonewall rank 74: 15056 of 18147 
stonewall rank 165: 15338 of 18147 
stonewall rank 135: 14664 of 18147 
stonewall rank 183: 15478 of 18147 
stonewall rank 174: 15948 of 18147 
stonewall rank 185: 14839 of 18147 
stonewall rank 195: 15960 of 18147 
stonewall rank 202: 15051 of 18147 
stonewall rank 216: 15909 of 18147 
stonewall rank 214: 15087 of 18147 
stonewall rank 238: 15331 of 18147 
stonewall rank 231: 14660 of 18147 
stonewall rank 248: 15508 of 18147 
stonewall rank 246: 15529 of 18147 
stonewall rank 276: 16914 of 18147 
stonewall rank 267: 15210 of 18147 
stonewall rank 288: 15211 of 18147 
stonewall rank 280: 14711 of 18147 
stonewall rank 303: 15708 of 18147 
stonewall rank 320: 15337 of 18147 
stonewall rank 312: 15200 of 18147 
stonewall rank 308: 15337 of 18147 
stonewall rank 330: 16001 of 18147 
stonewall rank 342: 15670 of 18147 
stonewall rank 344: 15637 of 18147 
stonewall rank 357: 16775 of 18147 
stonewall rank 366: 15635 of 18147 
stonewall rank 372: 15544 of 18147 
stonewall rank 376: 15142 of 18147 
stonewall rank 390: 14384 of 18147 
stonewall rank 392: 15884 of 18147 
stonewall rank 403: 16183 of 18147 
stonewall rank 408: 15604 of 18147 
stonewall rank 421: 15750 of 18147 
stonewall rank 436: 16033 of 18147 
stonewall rank 430: 15542 of 18147 
stonewall rank 440: 15929 of 18147 
stonewall rank 459: 15389 of 18147 
stonewall rank 451: 15194 of 18147 
stonewall rank 469: 15839 of 18147 
stonewall rank 479: 15525 of 18147 
stonewall rank 485: 15603 of 18147 
stonewall rank 494: 14759 of 18147 
stonewall rank 504: 16125 of 18147 
stonewall rank 497: 15271 of 18147 
stonewall rank 517: 14552 of 18147 
stonewall rank 531: 16796 of 18147 
stonewall rank 526: 15068 of 18147 
stonewall rank 536: 15750 of 18147 
stonewall rank 558: 16555 of 18147 
stonewall rank 549: 15089 of 18147 
stonewall rank 566: 16573 of 18147 
stonewall rank 568: 16185 of 18147 
stonewall rank 576: 16085 of 18147 
stonewall rank 584: 15919 of 18147 
stonewall rank 592: 15614 of 18147 
stonewall rank 600: 15840 of 18147 
stonewall rank 628: 15794 of 18147 
stonewall rank 632: 15808 of 18147 
stonewall rank 640: 16102 of 18147 
stonewall rank 655: 16595 of 18147 
stonewall rank 664: 16135 of 18147 
stonewall rank 659: 14653 of 18147 
stonewall rank 684: 14553 of 18147 
stonewall rank 696: 15158 of 18147 
stonewall rank 677: 14913 of 18147 
stonewall rank 704: 18021 of 18147 
stonewall rank 695: 15249 of 18147 
stonewall rank 613: 15963 of 18147 
stonewall rank 726: 14909 of 18147 
stonewall rank 714: 15399 of 18147 
stonewall rank 738: 15974 of 18147 
stonewall rank 728: 15874 of 18147 
stonewall rank 750: 16228 of 18147 
stonewall rank 759: 15844 of 18147 
stonewall rank 622: 16382 of 18147 
stonewall rank 768: 15919 of 18147 
stonewall rank 779: 15949 of 18147 
stonewall rank 784: 16263 of 18147 
stonewall rank 760: 15837 of 18147 
stonewall rank 792: 15768 of 18147 
stonewall rank 805: 15563 of 18147 
stonewall rank 820: 16760 of 18147 
stonewall rank 814: 15963 of 18147 
stonewall rank 829: 15156 of 18147 
stonewall rank 832: 15944 of 18147 
stonewall rank 845: 15960 of 18147 
stonewall rank 848: 15556 of 18147 
stonewall rank 869: 14667 of 18147 
stonewall rank 886: 16228 of 18147 
stonewall rank 889: 15374 of 18147 
stonewall rank 903: 15511 of 18147 
stonewall rank 906: 15858 of 18147 
stonewall rank 919: 16052 of 18147 
stonewall rank 927: 15740 of 18147 
stonewall rank 934: 15341 of 18147 
stonewall rank 936: 16030 of 18147 
stonewall rank 949: 16059 of 18147 
stonewall rank 954: 15295 of 18147 
Continue stonewall hit min: 14112 max: 18147 avg: 15626.0 
stonewall rank 0: 16577 of 18147 
stonewall rank 15: 15267 of 18147 
stonewall rank 16: 15144 of 18147 
stonewall rank 25: 15406 of 18147 
stonewall rank 47: 16055 of 18147 
stonewall rank 32: 14903 of 18147 
stonewall rank 48: 15371 of 18147 
stonewall rank 64: 15286 of 18147 
stonewall rank 57: 15506 of 18147 
stonewall rank 80: 14965 of 18147 
stonewall rank 89: 16109 of 18147 
stonewall rank 96: 15823 of 18147 
stonewall rank 111: 16152 of 18147 
stonewall rank 121: 15695 of 18147 
stonewall rank 112: 14738 of 18147 
stonewall rank 136: 15597 of 18147 
stonewall rank 144: 15822 of 18147 
stonewall rank 153: 14151 of 18147 
stonewall rank 75: 15045 of 18147 
stonewall rank 160: 15326 of 18147 
stonewall rank 128: 14643 of 18147 
stonewall rank 176: 15453 of 18147 
stonewall rank 175: 15940 of 18147 
stonewall rank 184: 14850 of 18147 
stonewall rank 192: 15932 of 18147 
stonewall rank 203: 15116 of 18147 
stonewall rank 217: 15840 of 18147 
stonewall rank 208: 15101 of 18147 
stonewall rank 239: 15409 of 18147 
stonewall rank 224: 14741 of 18147 
stonewall rank 249: 15490 of 18147 
stonewall rank 240: 15596 of 18147 
stonewall rank 256: 15588 of 18147 
stonewall rank 272: 16939 of 18147 
stonewall rank 264: 15239 of 18147 
stonewall rank 281: 14696 of 18147 
stonewall rank 300: 15689 of 18147 
stonewall rank 313: 15192 of 18147 
stonewall rank 304: 15340 of 18147 
stonewall rank 331: 16075 of 18147 
stonewall rank 336: 15672 of 18147 
stonewall rank 345: 15661 of 18147 
stonewall rank 352: 16763 of 18147 
stonewall rank 367: 15642 of 18147 
stonewall rank 368: 15620 of 18147 
stonewall rank 377: 15087 of 18147 
stonewall rank 384: 14388 of 18147 
stonewall rank 396: 15890 of 18147 
stonewall rank 400: 16192 of 18147 
stonewall rank 409: 15577 of 18147 
stonewall rank 416: 15795 of 18147 
stonewall rank 432: 16015 of 18147 
stonewall rank 431: 15594 of 18147 
stonewall rank 441: 15988 of 18147 
stonewall rank 456: 15356 of 18147 
stonewall rank 448: 15208 of 18147 
stonewall rank 464: 15869 of 18147 
stonewall rank 472: 15542 of 18147 
stonewall rank 480: 15702 of 18147 
stonewall rank 495: 14810 of 18147 
stonewall rank 505: 16078 of 18147 
stonewall rank 496: 15321 of 18147 
stonewall rank 512: 14623 of 18147 
stonewall rank 528: 16756 of 18147 
stonewall rank 527: 15077 of 18147 
stonewall rank 537: 15789 of 18147 
stonewall rank 559: 16496 of 18147 
stonewall rank 544: 15135 of 18147 
stonewall rank 560: 16630 of 18147 
stonewall rank 569: 16223 of 18147 
stonewall rank 601: 15834 of 18147 
stonewall rank 624: 15778 of 18147 
stonewall rank 633: 15811 of 18147 
stonewall rank 665: 16090 of 18147 
stonewall rank 656: 14677 of 18147 
stonewall rank 686: 14672 of 18147 
stonewall rank 697: 15177 of 18147 
stonewall rank 672: 14869 of 18147 
stonewall rank 688: 15298 of 18147 
stonewall rank 608: 15855 of 18147 
stonewall rank 720: 14955 of 18147 
stonewall rank 715: 15271 of 18147 
stonewall rank 736: 15978 of 18147 
stonewall rank 729: 15829 of 18147 
stonewall rank 751: 16175 of 18147 
stonewall rank 752: 15742 of 18147 
stonewall rank 623: 16440 of 18147 
stonewall rank 782: 15837 of 18147 
stonewall rank 761: 15991 of 18147 
stonewall rank 793: 15810 of 18147 
stonewall rank 800: 15595 of 18147 
stonewall rank 816: 16808 of 18147 
stonewall rank 815: 15968 of 18147 
stonewall rank 828: 15161 of 18147 
stonewall rank 841: 15950 of 18147 
stonewall rank 861: 15015 of 18147 
stonewall rank 864: 14689 of 18147 
stonewall rank 887: 16290 of 18147 
stonewall rank 888: 15405 of 18147 
stonewall rank 896: 15486 of 18147 
stonewall rank 907: 15916 of 18147 
stonewall rank 916: 16083 of 18147 
stonewall rank 920: 15830 of 18147 
stonewall rank 935: 15352 of 18147 
stonewall rank 940: 16029 of 18147 
stonewall rank 950: 16009 of 18147 
stonewall rank 955: 15244 of 18147 
stonewall rank 10: 15243 of 18147 
stonewall rank 40: 16018 of 18147 
stonewall rank 104: 16097 of 18147 
stonewall rank 168: 15963 of 18147 
stonewall rank 232: 15337 of 18147 
stonewall rank 268: 15204 of 18147 
stonewall rank 296: 15713 of 18147 
stonewall rank 360: 15692 of 18147 
stonewall rank 424: 15535 of 18147 
stonewall rank 473: 15591 of 18147 
stonewall rank 488: 14815 of 18147 
stonewall rank 521: 15036 of 18147 
stonewall rank 552: 16518 of 18147 
stonewall rank 681: 14737 of 18147 
stonewall rank 744: 16228 of 18147 
stonewall rank 616: 16430 of 18147 
stonewall rank 783: 15870 of 18147 
stonewall rank 808: 15950 of 18147 
stonewall rank 830: 15201 of 18147 
stonewall rank 844: 15990 of 18147 
stonewall rank 860: 15034 of 18147 
stonewall rank 874: 15890 of 18147 
stonewall rank 884: 16228 of 18147 
stonewall rank 905: 15818 of 18147 
stonewall rank 912: 16060 of 18147 
stonewall rank 924: 15767 of 18147 
stonewall rank 932: 15319 of 18147 
stonewall rank 951: 16196 of 18147 
stonewall rank 957: 15288 of 18147 
stonewall rank 11: 15299 of 18147 
stonewall rank 41: 16007 of 18147 
stonewall rank 105: 16119 of 18147 
stonewall rank 169: 15991 of 18147 
stonewall rank 233: 15323 of 18147 
stonewall rank 269: 15150 of 18147 
stonewall rank 297: 15636 of 18147 
stonewall rank 361: 15616 of 18147 
stonewall rank 425: 15490 of 18147 
stonewall rank 489: 14925 of 18147 
stonewall rank 522: 15177 of 18147 
stonewall rank 553: 16591 of 18147 
stonewall rank 680: 14678 of 18147 
stonewall rank 745: 16272 of 18147 
stonewall rank 617: 16381 of 18147 
stonewall rank 780: 15888 of 18147 
stonewall rank 809: 16103 of 18147 
stonewall rank 831: 15205 of 18147 
stonewall rank 846: 16022 of 18147 
stonewall rank 862: 15000 of 18147 
stonewall rank 875: 15903 of 18147 
stonewall rank 880: 16325 of 18147 
stonewall rank 910: 15893 of 18147 
stonewall rank 933: 15310 of 18147 
stonewall rank 948: 16057 of 18147 
stonewall rank 958: 15283 of 18147 
stonewall rank 8: 15306 of 18147 
stonewall rank 523: 15069 of 18147 
stonewall rank 781: 15895 of 18147 
stonewall rank 825: 15185 of 18147 
stonewall rank 847: 15976 of 18147 
stonewall rank 863: 15036 of 18147 
stonewall rank 877: 15892 of 18147 
stonewall rank 911: 15796 of 18147 
stonewall rank 928: 15410 of 18147 
stonewall rank 944: 16029 of 18147 
stonewall rank 959: 15251 of 18147 
stonewall rank 9: 15280 of 18147 
stonewall rank 520: 15040 of 18147 
stonewall rank 776: 15898 of 18147 
stonewall rank 824: 15160 of 18147 
stonewall rank 842: 15983 of 18147 
stonewall rank 856: 14995 of 18147 
stonewall rank 879: 15905 of 18147 
stonewall rank 908: 15879 of 18147 
stonewall rank 952: 15300 of 18147 
stonewall rank 843: 15885 of 18147 
stonewall rank 857: 15016 of 18147 
stonewall rank 876: 15957 of 18147 
stonewall rank 904: 15912 of 18147 
stonewall rank 956: 15248 of 18147 
stonewall rank 840: 15889 of 18147 
stonewall rank 878: 15942 of 18147 
stonewall rank 872: 15874 of 18147 
stonewall rank 873: 15979 of 18147 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :     167463.008     167463.008     167463.008          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          5.229          5.229          5.229          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/11/2018 16:56:34 --
result_summary
[RESULT] BW   phase 1            ior_easy_write              107.227 GB/s : time 524.59 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              260.025 kiops : time 462.81 seconds
[RESULT] BW   phase 2            ior_hard_write                5.657 GB/s : time 312.03 seconds
[RESULT] IOPS phase 2         mdtest_hard_write              167.463 kiops : time 373.55 seconds
[RESULT] IOPS phase 3                      find              881.610 kiops : time  45.53 seconds
[RESULT] BW   phase 3             ior_easy_read               38.263 GB/s : time 1470.11 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              325.972 kiops : time  70.44 seconds
[RESULT] BW   phase 4             ior_hard_read               12.185 GB/s : time 144.86 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat              178.786 kiops : time  98.18 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               37.465 kiops : time 610.03 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               68.716 kiops : time 254.21 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               25.245 kiops : time 691.30 seconds
[SCORE] Bandwidth 23.0612 GB/s : IOPS 139.74 kiops : TOTAL 56.7677