- io500
-
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.
set -euo pipefail # better error handling
# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True" # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True" # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False" # this one is optional
io500_cleanup_workdir="False" # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_stonewall_timer=300 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...
# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
setup_directories
setup_paths
setup_ior_easy # required if you want a complete score
setup_ior_hard # required if you want a complete score
setup_mdt_easy # required if you want a complete score
setup_mdt_hard # required if you want a complete score
setup_find # required if you want a complete score
setup_mdreal # optional
run_benchmarks
}
function setup_directories {
# set directories for where the benchmark files are created and where the results will go.
# If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
timestamp=`date +%Y.%m.%d-%H.%M.%S` # create a uniquifier
io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
io500_result_dir=$PWD/results/$timestamp # the directory where the output results will be kept
mkdir -p $io500_workdir $io500_result_dir
}
function setup_paths {
# Set the paths to the binaries. If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
io500_ior_cmd=$PWD/bin/ior
io500_mdtest_cmd=$PWD/bin/mdtest
io500_mdreal_cmd=$PWD/bin/md-real-io
io500_mpirun="jsrun"
io500_mpiargs="-n 504 -a 2 -c 2 -r 1"
}
function setup_ior_easy {
# io500_ior_easy_size is the amount of data written per rank in MiB units,
# but it can be any number as long as it is somehow used to scale the IOR
# runtime as part of io500_ior_easy_params
io500_ior_easy_size=1385536
# 2M writes, 2 GB per proc, file per proc
io500_ior_easy_params="-t 16384k -b ${io500_ior_easy_size}m -F"
}
function setup_mdt_easy {
io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
io500_mdtest_easy_files_per_proc=1000000
}
function setup_ior_hard {
io500_ior_hard_writes_per_proc=6000
io500_ior_hard_other_options="" #e.g., -E to keep precreated files using lfs setstripe, or -a MPIIO
}
function setup_mdt_hard {
io500_mdtest_hard_files_per_proc=7500
io500_mdtest_hard_other_options=""
}
function setup_find {
#
# setup the find command. This is an area where innovation is allowed.
# There are three default options provided. One is a serial find, one is python
# parallel version, one is C parallel version. Current default is to use serial.
# But it is very slow. We recommend to either customize or use the C parallel version.
# For GPFS, we recommend to use the provided mmfind wrapper described below.
# Instructions below.
# If a custom approach is used, please provide enough info so others can reproduce.
# the serial version that should run (SLOWLY) without modification
#io500_find_mpi="False"
#io500_find_cmd=$PWD/bin/sfind.sh
#io500_find_cmd_args=""
# a parallel version in C, the -s adds a stonewall
# for a real run, turn -s (stonewall) off or set it at 300 or more
# to prepare this (assuming you've run ./utilities/prepare.sh already):
# > cd build/pfind
# > ./prepare.sh
# > ./compile.sh
# > cp pfind ../../bin/
# If you use io500_find_mpi="True", then this will run with the same
# number of MPI nodes and ranks as the other phases.
# If you prefer another number, and fewer might be better here,
# Then you can set io500_find_mpi to be "False" and write a wrapper
# script for this which sets up MPI as you would like. Then change
# io500_find_cmd to point to your wrapper script.
io500_find_mpi="True"
io500_find_cmd="$PWD/bin/pfind"
# uses stonewalling, run pfind
io500_find_cmd_args="-s $io500_stonewall_timer -r $io500_result_dir/pfind_results"
# for GPFS systems, you should probably use the provided mmfind wrapper
# if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
#io500_find_mpi="False"
#io500_find_cmd="$PWD/bin/mmfind.sh"
#io500_find_cmd_args=""
}
function setup_mdreal {
io500_mdreal_params="-P=5000 -I=1000"
}
function run_benchmarks {
# Important: source the io500_fixed.sh script. Do not change it. If you discover
# a need to change it, please email the mailing list to discuss
source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}
# Add key/value pairs defining your system
# Feel free to add extra ones if you'd like
function extra_description {
# top level info
io500_info_system_name='xxx' # e.g. Oakforest-PACS
io500_info_institute_name='xxx' # e.g. JCAHPC
io500_info_storage_age_in_months='xxx' # not install date but age since last refresh
io500_info_storage_install_date='xxx' # MM/YY
io500_info_filesystem='xxx' # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
io500_info_filesystem_version='xxx'
io500_info_filesystem_vendor='xxx'
# client side info
io500_info_num_client_nodes='xxx'
io500_info_procs_per_node='xxx'
# server side info
io500_info_num_metadata_server_nodes='xxx'
io500_info_num_data_server_nodes='xxx'
io500_info_num_data_storage_devices='xxx' # if you have 5 data servers, and each has 5 drives, then this number is 25
io500_info_num_metadata_storage_devices='xxx' # if you have 2 metadata servers, and each has 5 drives, then this number is 10
io500_info_data_storage_type='xxx' # HDD, SSD, persistent memory, etc, feel free to put specific models
io500_info_metadata_storage_type='xxx' # HDD, SSD, persistent memory, etc, feel free to put specific models
io500_info_storage_network='xxx' # infiniband, omnipath, ethernet, etc
io500_info_storage_interface='xxx' # SAS, SATA, NVMe, etc
# miscellaneous
io500_info_whatever='WhateverElseYouThinkRelevant'
}
main
- ior_easy_read
-
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Fri Nov 9 03:39:41 2018
Command line : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 16384k -b 1385536m -F -o /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_easy/ior_file_easy -O stoneWallingStatusFile=/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_easy/stonewall
Machine : Linux f25n01
TestID : 0
StartTime : Fri Nov 9 03:39:41 2018
Path : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_easy
FS : 225668.2 TiB Used FS: 5.7% Inodes: 30000.0 Mi Used Inodes: 16.2%
Options:
api : POSIX
apiVersion :
test filename : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 1008
clients per node : 2
repetitions : 1
xfersize : 16 MiB
blocksize : 1.32 TiB
aggregate filesize : 1331.92 TiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
WARNING: Expected aggregate file size = 1464462515109888.
WARNING: Stat() of aggregate file size = 839872533233664.
WARNING: Using actual aggregate bytes moved = 839872533233664.
read 1831236 1418788864 16384 0.539930 436.85 0.000242 437.39 0
Max Read: 1831235.86 MiB/sec (1920189.98 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 1831235.86 1831235.86 1831235.86 0.00 114452.24 114452.24 114452.24 0.00 437.39033 0 1008 2 1 1 1 1 0 0 1 1452839796736 16777216 800964864.0 POSIX 0
Finished : Fri Nov 9 03:46:58 2018
- ior_easy_write
-
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Fri Nov 9 03:13:14 2018
Command line : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 16384k -b 1385536m -F -o /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_easy/ior_file_easy -O stoneWallingStatusFile=/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux f25n01
TestID : 0
StartTime : Fri Nov 9 03:13:14 2018
Path : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_easy
FS : 225668.2 TiB Used FS: 5.4% Inodes: 30000.0 Mi Used Inodes: 13.0%
Options:
api : POSIX
apiVersion :
test filename : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 1008
clients per node : 2
repetitions : 1
xfersize : 16 MiB
blocksize : 1.32 TiB
aggregate filesize : 1331.92 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 37133 max: 49663 -- min data: 580.2 GiB mean data: 678.1 GiB time: 300.4s
WARNING: Expected aggregate file size = 1464462515109888.
WARNING: Stat() of aggregate file size = 839872533233664.
WARNING: Using actual aggregate bytes moved = 839872533233664.
WARNING: maybe caused by deadlineForStonewalling
write 2210511 1418788864 16384 2.93 359.41 0.000584 362.34 0
Max Write: 2210511.33 MiB/sec (2317889.13 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 2210511.33 2210511.33 2210511.33 0.00 138156.96 138156.96 138156.96 0.00 362.34370 0 1008 2 1 1 1 1 0 0 1 1452839796736 16777216 800964864.0 POSIX 0
Finished : Fri Nov 9 03:19:17 2018
- ior_hard_read
-
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Fri Nov 9 03:48:07 2018
Command line : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 6000 -o /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_hard/IOR_file -O stoneWallingStatusFile=/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_hard/stonewall
Machine : Linux f25n01
TestID : 0
StartTime : Fri Nov 9 03:48:07 2018
Path : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_hard
FS : 225668.2 TiB Used FS: 5.7% Inodes: 30000.0 Mi Used Inodes: 16.2%
Options:
api : POSIX
apiVersion :
test filename : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 6000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 1008
clients per node : 2
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 264.78 GiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
read 28061 45.91 45.91 0.300558 9.36 0.000173 9.66 0
Max Read: 28060.81 MiB/sec (29423.90 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 28060.81 28060.81 28060.81 0.00 625933.78 625933.78 625933.78 0.00 9.66236 0 1008 2 1 0 1 1 0 0 6000 47008 47008 271133.8 POSIX 0
Finished : Fri Nov 9 03:48:17 2018
- ior_hard_write
-
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Fri Nov 9 03:25:27 2018
Command line : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 6000 -o /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_hard/IOR_file -O stoneWallingStatusFile=/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux f25n01
TestID : 0
StartTime : Fri Nov 9 03:25:27 2018
Path : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_hard
FS : 225668.2 TiB Used FS: 5.7% Inodes: 30000.0 Mi Used Inodes: 15.9%
Options:
api : POSIX
apiVersion :
test filename : /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 6000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 1008
clients per node : 2
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 264.78 GiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 339 max: 6000 -- min data: 0.0 GiB mean data: 0.1 GiB time: 300.5s
write 585.91 45.91 45.91 0.158839 462.58 0.017306 462.76 0
Max Write: 585.91 MiB/sec (614.37 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 585.91 585.91 585.91 0.00 13069.53 13069.53 13069.53 0.00 462.75568 0 1008 2 1 0 1 1 0 0 6000 47008 47008 271133.8 POSIX 0
Finished : Fri Nov 9 03:33:10 2018
- mdtest_easy_delete
-
-- started at 11/09/2018 03:48:53 --
mdtest-1.9.3 was launched with 1008 total task(s) on 504 node(s)
Command line used: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/mdtest "-r" "-F" "-d" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_easy" "-n" "1000000" "-u" "-L" "-x" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_easy-stonewall"
Path: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50
FS: 225668.2 TiB Used FS: 5.7% Inodes: 30000.0 Mi Used Inodes: 16.2%
1008 tasks, 1008000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 2698997.092 2698997.092 2698997.092 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.125 0.125 0.125 0.000
-- finished at 11/09/2018 03:55:14 --
- mdtest_easy_stat
-
-- started at 11/09/2018 03:47:16 --
mdtest-1.9.3 was launched with 1008 total task(s) on 504 node(s)
Command line used: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/mdtest "-T" "-F" "-d" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_easy" "-n" "1000000" "-u" "-L" "-x" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_easy-stonewall"
Path: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50
FS: 225668.2 TiB Used FS: 5.7% Inodes: 30000.0 Mi Used Inodes: 16.2%
1008 tasks, 1008000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 28769436.935 28769436.935 28769436.935 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2018 03:47:51 --
- mdtest_easy_write
-
-- started at 11/09/2018 03:19:35 --
mdtest-1.9.3 was launched with 1008 total task(s) on 504 node(s)
Command line used: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/mdtest "-C" "-F" "-d" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_easy" "-n" "1000000" "-u" "-L" "-x" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_easy-stonewall" "-W" "300"
Path: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50
FS: 225668.2 TiB Used FS: 5.7% Inodes: 30000.0 Mi Used Inodes: 13.0%
1008 tasks, 1008000000 files
Continue stonewall hit min: 871513 max: 1000000 avg: 978223.4
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 3071261.901 3071261.901 3071261.901 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.147 0.147 0.147 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2018 03:25:10 --
stonewall rank 400: 967110 of 1000000
stonewall rank 325: 950029 of 1000000
stonewall rank 829: 939257 of 1000000
stonewall rank 894: 941005 of 1000000
stonewall rank 954: 932968 of 1000000
stonewall rank 450: 902421 of 1000000
stonewall rank 472: 953881 of 1000000
stonewall rank 976: 958701 of 1000000
stonewall rank 495: 895988 of 1000000
stonewall rank 490: 955341 of 1000000
stonewall rank 834: 947588 of 1000000
stonewall rank 999: 936017 of 1000000
stonewall rank 994: 925846 of 1000000
stonewall rank 862: 937058 of 1000000
stonewall rank 391: 919031 of 1000000
stonewall rank 358: 956663 of 1000000
stonewall rank 895: 975634 of 1000000
stonewall rank 858: 876510 of 1000000
stonewall rank 354: 898608 of 1000000
stonewall rank 347: 927656 of 1000000
stonewall rank 390: 968387 of 1000000
stonewall rank 320: 945813 of 1000000
stonewall rank 916: 905181 of 1000000
stonewall rank 851: 942341 of 1000000
stonewall rank 330: 923123 of 1000000
stonewall rank 387: 962268 of 1000000
stonewall rank 412: 941960 of 1000000
stonewall rank 276: 949306 of 1000000
stonewall rank 953: 914254 of 1000000
stonewall rank 265: 903769 of 1000000
stonewall rank 449: 977452 of 1000000
stonewall rank 440: 921066 of 1000000
stonewall rank 446: 986914 of 1000000
stonewall rank 891: 982122 of 1000000
stonewall rank 984: 991896 of 1000000
stonewall rank 901: 988834 of 1000000
stonewall rank 944: 923638 of 1000000
stonewall rank 759: 947348 of 1000000
stonewall rank 455: 970651 of 1000000
stonewall rank 780: 942468 of 1000000
stonewall rank 439: 932408 of 1000000
stonewall rank 409: 939984 of 1000000
stonewall rank 255: 930953 of 1000000
stonewall rank 331: 965406 of 1000000
stonewall rank 824: 969841 of 1000000
stonewall rank 479: 924922 of 1000000
stonewall rank 712: 986388 of 1000000
stonewall rank 283: 967586 of 1000000
stonewall rank 482: 964068 of 1000000
stonewall rank 396: 943649 of 1000000
stonewall rank 787: 946899 of 1000000
stonewall rank 366: 995407 of 1000000
stonewall rank 808: 994393 of 1000000
stonewall rank 304: 952933 of 1000000
stonewall rank 983: 930603 of 1000000
stonewall rank 267: 962147 of 1000000
stonewall rank 859: 978180 of 1000000
stonewall rank 316: 978673 of 1000000
stonewall rank 355: 926005 of 1000000
stonewall rank 900: 980875 of 1000000
stonewall rank 931: 902518 of 1000000
stonewall rank 870: 954038 of 1000000
stonewall rank 771: 969433 of 1000000
stonewall rank 480: 961957 of 1000000
stonewall rank 404: 944158 of 1000000
stonewall rank 885: 912277 of 1000000
stonewall rank 417: 986972 of 1000000
stonewall rank 943: 932270 of 1000000
stonewall rank 252: 963395 of 1000000
stonewall rank 913: 964264 of 1000000
stonewall rank 419: 918842 of 1000000
stonewall rank 959: 946537 of 1000000
stonewall rank 341: 978907 of 1000000
stonewall rank 845: 946293 of 1000000
stonewall rank 756: 959670 of 1000000
stonewall rank 872: 963533 of 1000000
stonewall rank 309: 947927 of 1000000
stonewall rank 286: 965500 of 1000000
stonewall rank 301: 932351 of 1000000
stonewall rank 332: 994636 of 1000000
stonewall rank 769: 937438 of 1000000
stonewall rank 432: 948579 of 1000000
stonewall rank 371: 970312 of 1000000
stonewall rank 805: 975291 of 1000000
stonewall rank 906: 945094 of 1000000
stonewall rank 784: 998343 of 1000000
stonewall rank 445: 974908 of 1000000
stonewall rank 499: 924294 of 1000000
stonewall rank 368: 952912 of 1000000
stonewall rank 836: 944251 of 1000000
stonewall rank 921: 958754 of 1000000
stonewall rank 986: 940885 of 1000000
stonewall rank 838: 948536 of 1000000
stonewall rank 908: 944125 of 1000000
stonewall rank 856: 990240 of 1000000
stonewall rank 1003: 927682 of 1000000
stonewall rank 977: 961206 of 1000000
stonewall rank 352: 969196 of 1000000
stonewall rank 887: 922844 of 1000000
stonewall rank 418: 953839 of 1000000
stonewall rank 922: 980007 of 1000000
stonewall rank 413: 983822 of 1000000
stonewall rank 1007: 958579 of 1000000
stonewall rank 790: 929671 of 1000000
stonewall rank 503: 934814 of 1000000
stonewall rank 383: 963813 of 1000000
stonewall rank 936: 979055 of 1000000
stonewall rank 757: 960290 of 1000000
stonewall rank 474: 933881 of 1000000
stonewall rank 302: 919767 of 1000000
stonewall rank 269: 951569 of 1000000
stonewall rank 806: 927868 of 1000000
stonewall rank 427: 937724 of 1000000
stonewall rank 349: 983020 of 1000000
stonewall rank 773: 993657 of 1000000
stonewall rank 496: 906219 of 1000000
stonewall rank 963: 949008 of 1000000
stonewall rank 423: 943687 of 1000000
stonewall rank 853: 977202 of 1000000
stonewall rank 835: 987713 of 1000000
stonewall rank 381: 886253 of 1000000
stonewall rank 896: 930105 of 1000000
stonewall rank 778: 995072 of 1000000
stonewall rank 813: 972045 of 1000000
stonewall rank 288: 998680 of 1000000
stonewall rank 981: 952014 of 1000000
stonewall rank 993: 943102 of 1000000
stonewall rank 489: 937749 of 1000000
stonewall rank 274: 987211 of 1000000
stonewall rank 272: 955846 of 1000000
stonewall rank 819: 982264 of 1000000
stonewall rank 477: 947109 of 1000000
stonewall rank 459: 950541 of 1000000
stonewall rank 969: 945055 of 1000000
stonewall rank 823: 919642 of 1000000
stonewall rank 890: 976298 of 1000000
stonewall rank 473: 954944 of 1000000
stonewall rank 917: 956158 of 1000000
stonewall rank 945: 919961 of 1000000
stonewall rank 878: 954174 of 1000000
stonewall rank 923: 941139 of 1000000
stonewall rank 776: 950991 of 1000000
stonewall rank 442: 950224 of 1000000
stonewall rank 907: 922337 of 1000000
stonewall rank 882: 909537 of 1000000
stonewall rank 334: 989701 of 1000000
stonewall rank 1000: 930614 of 1000000
stonewall rank 949: 957729 of 1000000
stonewall rank 476: 985961 of 1000000
stonewall rank 392: 932660 of 1000000
stonewall rank 792: 966957 of 1000000
stonewall rank 816: 974611 of 1000000
stonewall rank 465: 955544 of 1000000
stonewall rank 379: 970915 of 1000000
stonewall rank 441: 939875 of 1000000
stonewall rank 312: 928839 of 1000000
stonewall rank 875: 941447 of 1000000
stonewall rank 876: 988610 of 1000000
stonewall rank 321: 981804 of 1000000
stonewall rank 386: 959913 of 1000000
stonewall rank 500: 970545 of 1000000
stonewall rank 1004: 953390 of 1000000
stonewall rank 955: 973041 of 1000000
stonewall rank 941: 963624 of 1000000
stonewall rank 402: 940021 of 1000000
stonewall rank 974: 918356 of 1000000
stonewall rank 429: 939991 of 1000000
stonewall rank 846: 943988 of 1000000
stonewall rank 933: 932905 of 1000000
stonewall rank 927: 952556 of 1000000
stonewall rank 403: 934939 of 1000000
stonewall rank 467: 994983 of 1000000
stonewall rank 324: 946654 of 1000000
stonewall rank 460: 995959 of 1000000
stonewall rank 883: 962653 of 1000000
stonewall rank 964: 992822 of 1000000
stonewall rank 464: 871513 of 1000000
stonewall rank 433: 970846 of 1000000
stonewall rank 268: 908296 of 1000000
stonewall rank 435: 987493 of 1000000
stonewall rank 978: 976467 of 1000000
stonewall rank 277: 936794 of 1000000
stonewall rank 937: 952158 of 1000000
stonewall rank 315: 955795 of 1000000
stonewall rank 971: 954294 of 1000000
stonewall rank 857: 971809 of 1000000
stonewall rank 451: 952946 of 1000000
stonewall rank 968: 932466 of 1000000
stonewall rank 340: 969967 of 1000000
stonewall rank 170: 997008 of 1000000
stonewall rank 466: 964163 of 1000000
stonewall rank 781: 935397 of 1000000
stonewall rank 996: 923186 of 1000000
stonewall rank 939: 952269 of 1000000
stonewall rank 428: 964173 of 1000000
stonewall rank 772: 911438 of 1000000
stonewall rank 973: 955067 of 1000000
stonewall rank 374: 969669 of 1000000
stonewall rank 980: 949158 of 1000000
stonewall rank 372: 963162 of 1000000
stonewall rank 407: 957735 of 1000000
stonewall rank 828: 948419 of 1000000
stonewall rank 985: 915218 of 1000000
stonewall rank 414: 930036 of 1000000
stonewall rank 319: 905408 of 1000000
stonewall rank 378: 918654 of 1000000
stonewall rank 946: 986721 of 1000000
stonewall rank 260: 946735 of 1000000
stonewall rank 307: 951494 of 1000000
stonewall rank 470: 928601 of 1000000
stonewall rank 303: 896213 of 1000000
stonewall rank 306: 971594 of 1000000
stonewall rank 353: 934465 of 1000000
stonewall rank 837: 938778 of 1000000
stonewall rank 438: 948884 of 1000000
stonewall rank 483: 946640 of 1000000
stonewall rank 342: 938861 of 1000000
stonewall rank 911: 965010 of 1000000
stonewall rank 833: 932651 of 1000000
stonewall rank 947: 950606 of 1000000
stonewall rank 762: 940702 of 1000000
stonewall rank 997: 912748 of 1000000
stonewall rank 932: 945756 of 1000000
stonewall rank 918: 960423 of 1000000
stonewall rank 367: 960378 of 1000000
stonewall rank 292: 991144 of 1000000
stonewall rank 942: 919809 of 1000000
stonewall rank 764: 969090 of 1000000
stonewall rank 831: 978644 of 1000000
stonewall rank 486: 995943 of 1000000
stonewall rank 453: 962590 of 1000000
stonewall rank 254: 923977 of 1000000
stonewall rank 420: 958654 of 1000000
stonewall rank 488: 997841 of 1000000
stonewall rank 952: 980534 of 1000000
stonewall rank 760: 917671 of 1000000
stonewall rank 422: 917925 of 1000000
stonewall rank 854: 963574 of 1000000
stonewall rank 807: 952280 of 1000000
stonewall rank 333: 951236 of 1000000
stonewall rank 437: 963820 of 1000000
stonewall rank 765: 959017 of 1000000
stonewall rank 988: 978693 of 1000000
stonewall rank 448: 948537 of 1000000
stonewall rank 811: 956621 of 1000000
stonewall rank 345: 957037 of 1000000
stonewall rank 375: 939652 of 1000000
stonewall rank 801: 945006 of 1000000
stonewall rank 494: 913210 of 1000000
stonewall rank 318: 948700 of 1000000
stonewall rank 373: 905744 of 1000000
stonewall rank 972: 954821 of 1000000
stonewall rank 327: 936576 of 1000000
stonewall rank 461: 961996 of 1000000
stonewall rank 758: 936440 of 1000000
stonewall rank 285: 931330 of 1000000
stonewall rank 258: 950750 of 1000000
stonewall rank 408: 982716 of 1000000
stonewall rank 350: 966598 of 1000000
stonewall rank 879: 973549 of 1000000
stonewall rank 329: 965225 of 1000000
stonewall rank 298: 976391 of 1000000
stonewall rank 462: 919091 of 1000000
stonewall rank 928: 954330 of 1000000
stonewall rank 791: 932547 of 1000000
stonewall rank 261: 961382 of 1000000
stonewall rank 915: 963313 of 1000000
stonewall rank 388: 991263 of 1000000
stonewall rank 987: 969553 of 1000000
stonewall rank 443: 922673 of 1000000
stonewall rank 310: 970596 of 1000000
stonewall rank 849: 969269 of 1000000
stonewall rank 873: 906961 of 1000000
stonewall rank 468: 953555 of 1000000
stonewall rank 357: 933549 of 1000000
stonewall rank 454: 987747 of 1000000
stonewall rank 737: 980237 of 1000000
stonewall rank 425: 966126 of 1000000
stonewall rank 278: 955694 of 1000000
stonewall rank 766: 956550 of 1000000
stonewall rank 410: 943337 of 1000000
stonewall rank 398: 984935 of 1000000
stonewall rank 487: 926360 of 1000000
stonewall rank 339: 947268 of 1000000
stonewall rank 877: 943998 of 1000000
stonewall rank 863: 939215 of 1000000
stonewall rank 297: 900701 of 1000000
stonewall rank 965: 948875 of 1000000
stonewall rank 926: 911626 of 1000000
stonewall rank 970: 972527 of 1000000
stonewall rank 492: 986743 of 1000000
stonewall rank 501: 980528 of 1000000
stonewall rank 436: 967554 of 1000000
stonewall rank 789: 928922 of 1000000
stonewall rank 336: 993287 of 1000000
stonewall rank 314: 945887 of 1000000
stonewall rank 370: 957890 of 1000000
stonewall rank 362: 926614 of 1000000
stonewall rank 289: 960975 of 1000000
stonewall rank 825: 978058 of 1000000
stonewall rank 393: 997628 of 1000000
stonewall rank 958: 992421 of 1000000
stonewall rank 929: 959829 of 1000000
stonewall rank 975: 955309 of 1000000
stonewall rank 909: 977478 of 1000000
stonewall rank 998: 919468 of 1000000
stonewall rank 281: 984355 of 1000000
stonewall rank 814: 969087 of 1000000
stonewall rank 844: 994101 of 1000000
stonewall rank 365: 945140 of 1000000
stonewall rank 385: 950511 of 1000000
stonewall rank 966: 903377 of 1000000
stonewall rank 770: 986062 of 1000000
stonewall rank 910: 891047 of 1000000
stonewall rank 793: 960619 of 1000000
stonewall rank 892: 953423 of 1000000
stonewall rank 990: 989779 of 1000000
stonewall rank 1005: 960019 of 1000000
stonewall rank 70: 992016 of 1000000
stonewall rank 940: 976655 of 1000000
stonewall rank 273: 964507 of 1000000
stonewall rank 818: 938432 of 1000000
stonewall rank 880: 944871 of 1000000
stonewall rank 359: 954330 of 1000000
stonewall rank 300: 946706 of 1000000
stonewall rank 830: 966152 of 1000000
stonewall rank 262: 941230 of 1000000
stonewall rank 469: 958403 of 1000000
stonewall rank 919: 993973 of 1000000
stonewall rank 498: 948245 of 1000000
stonewall rank 424: 992065 of 1000000
stonewall rank 779: 907257 of 1000000
stonewall rank 376: 936397 of 1000000
stonewall rank 798: 968548 of 1000000
stonewall rank 457: 949979 of 1000000
stonewall rank 934: 920070 of 1000000
stonewall rank 343: 946361 of 1000000
stonewall rank 256: 947534 of 1000000
stonewall rank 406: 908885 of 1000000
stonewall rank 389: 993113 of 1000000
stonewall rank 502: 997411 of 1000000
stonewall rank 785: 951088 of 1000000
stonewall rank 822: 961848 of 1000000
stonewall rank 369: 958843 of 1000000
stonewall rank 871: 949930 of 1000000
stonewall rank 326: 959565 of 1000000
stonewall rank 430: 960256 of 1000000
stonewall rank 266: 952346 of 1000000
stonewall rank 475: 938441 of 1000000
stonewall rank 415: 994589 of 1000000
stonewall rank 897: 979019 of 1000000
stonewall rank 847: 912856 of 1000000
stonewall rank 869: 943228 of 1000000
stonewall rank 293: 985213 of 1000000
stonewall rank 826: 948112 of 1000000
stonewall rank 1002: 968629 of 1000000
stonewall rank 935: 964312 of 1000000
stonewall rank 777: 999075 of 1000000
stonewall rank 259: 952278 of 1000000
stonewall rank 481: 953305 of 1000000
stonewall rank 471: 945322 of 1000000
stonewall rank 889: 982890 of 1000000
stonewall rank 1006: 987171 of 1000000
stonewall rank 763: 958777 of 1000000
stonewall rank 861: 954337 of 1000000
stonewall rank 797: 946358 of 1000000
stonewall rank 967: 961181 of 1000000
stonewall rank 788: 997136 of 1000000
stonewall rank 380: 942673 of 1000000
stonewall rank 979: 928894 of 1000000
stonewall rank 914: 937962 of 1000000
stonewall rank 323: 962818 of 1000000
stonewall rank 493: 911702 of 1000000
stonewall rank 308: 935826 of 1000000
stonewall rank 291: 933403 of 1000000
stonewall rank 961: 952322 of 1000000
stonewall rank 263: 991989 of 1000000
stonewall rank 478: 941418 of 1000000
stonewall rank 905: 969913 of 1000000
stonewall rank 328: 968701 of 1000000
stonewall rank 287: 949076 of 1000000
stonewall rank 812: 901827 of 1000000
stonewall rank 840: 951369 of 1000000
stonewall rank 884: 925005 of 1000000
stonewall rank 395: 923721 of 1000000
stonewall rank 957: 945097 of 1000000
stonewall rank 583: 960750 of 1000000
stonewall rank 431: 938312 of 1000000
stonewall rank 264: 953641 of 1000000
stonewall rank 322: 930696 of 1000000
stonewall rank 874: 955103 of 1000000
stonewall rank 795: 971705 of 1000000
stonewall rank 962: 944506 of 1000000
stonewall rank 768: 969032 of 1000000
stonewall rank 426: 908745 of 1000000
stonewall rank 338: 962963 of 1000000
stonewall rank 865: 931767 of 1000000
stonewall rank 290: 922545 of 1000000
stonewall rank 899: 963626 of 1000000
stonewall rank 991: 935282 of 1000000
stonewall rank 463: 948719 of 1000000
stonewall rank 405: 947643 of 1000000
stonewall rank 832: 946747 of 1000000
stonewall rank 989: 946042 of 1000000
stonewall rank 782: 992630 of 1000000
stonewall rank 982: 924633 of 1000000
stonewall rank 364: 973867 of 1000000
stonewall rank 294: 965830 of 1000000
stonewall rank 903: 996819 of 1000000
stonewall rank 401: 980578 of 1000000
stonewall rank 767: 955492 of 1000000
stonewall rank 485: 972711 of 1000000
stonewall rank 275: 943662 of 1000000
stonewall rank 337: 956789 of 1000000
stonewall rank 866: 943171 of 1000000
stonewall rank 800: 980935 of 1000000
stonewall rank 79: 997879 of 1000000
stonewall rank 827: 974067 of 1000000
stonewall rank 786: 923879 of 1000000
stonewall rank 842: 928602 of 1000000
stonewall rank 361: 951915 of 1000000
stonewall rank 296: 972452 of 1000000
stonewall rank 898: 975291 of 1000000
stonewall rank 458: 934033 of 1000000
stonewall rank 382: 938975 of 1000000
stonewall rank 284: 957103 of 1000000
stonewall rank 815: 929273 of 1000000
stonewall rank 850: 949476 of 1000000
stonewall rank 803: 957209 of 1000000
stonewall rank 868: 978770 of 1000000
stonewall rank 311: 970093 of 1000000
stonewall rank 491: 897551 of 1000000
stonewall rank 346: 969513 of 1000000
stonewall rank 802: 958397 of 1000000
stonewall rank 456: 946926 of 1000000
stonewall rank 434: 974258 of 1000000
stonewall rank 774: 998930 of 1000000
stonewall rank 394: 998632 of 1000000
stonewall rank 995: 983395 of 1000000
stonewall rank 841: 977817 of 1000000
stonewall rank 452: 939331 of 1000000
stonewall rank 447: 979307 of 1000000
stonewall rank 257: 954385 of 1000000
stonewall rank 804: 926616 of 1000000
stonewall rank 956: 961052 of 1000000
stonewall rank 938: 946904 of 1000000
stonewall rank 794: 967538 of 1000000
stonewall rank 855: 942002 of 1000000
stonewall rank 951: 969359 of 1000000
stonewall rank 282: 992096 of 1000000
stonewall rank 344: 966106 of 1000000
stonewall rank 351: 994252 of 1000000
stonewall rank 299: 995192 of 1000000
stonewall rank 363: 963680 of 1000000
stonewall rank 930: 938484 of 1000000
stonewall rank 886: 931034 of 1000000
stonewall rank 867: 960444 of 1000000
stonewall rank 860: 975526 of 1000000
stonewall rank 925: 935764 of 1000000
stonewall rank 960: 944523 of 1000000
stonewall rank 421: 983947 of 1000000
stonewall rank 377: 968410 of 1000000
stonewall rank 356: 901259 of 1000000
stonewall rank 775: 971760 of 1000000
stonewall rank 881: 944609 of 1000000
stonewall rank 271: 923960 of 1000000
stonewall rank 360: 901027 of 1000000
stonewall rank 864: 894268 of 1000000
- mdtest_hard_delete
-
-- started at 11/09/2018 03:55:48 --
mdtest-1.9.3 was launched with 1008 total task(s) on 504 node(s)
Command line used: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/mdtest "-r" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_hard" "-n" "7500" "-x" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_hard-stonewall"
Path: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50
FS: 225668.2 TiB Used FS: 5.7% Inodes: 30000.0 Mi Used Inodes: 13.1%
1008 tasks, 7560000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 26504.478 26504.478 26504.478 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.009 0.009 0.009 0.000
-- finished at 11/09/2018 03:58:33 --
- mdtest_hard_read
-
-- started at 11/09/2018 03:55:31 --
mdtest-1.9.3 was launched with 1008 total task(s) on 504 node(s)
Command line used: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/mdtest "-E" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_hard" "-n" "7500" "-x" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_hard-stonewall"
Path: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50
FS: 225668.2 TiB Used FS: 5.7% Inodes: 30000.0 Mi Used Inodes: 13.1%
1008 tasks, 7560000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 15354716.853 15354716.853 15354716.853 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2018 03:55:32 --
- mdtest_hard_stat
-
-- started at 11/09/2018 03:48:34 --
mdtest-1.9.3 was launched with 1008 total task(s) on 504 node(s)
Command line used: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/mdtest "-T" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_hard" "-n" "7500" "-x" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_hard-stonewall"
Path: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50
FS: 225668.2 TiB Used FS: 5.7% Inodes: 30000.0 Mi Used Inodes: 16.2%
1008 tasks, 7560000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 560885.922 560885.922 560885.922 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2018 03:48:36 --
- mdtest_hard_write
-
-- started at 11/09/2018 03:33:27 --
mdtest-1.9.3 was launched with 1008 total task(s) on 504 node(s)
Command line used: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/bin/mdtest "-C" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_hard" "-n" "7500" "-x" "/gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50/mdt_hard-stonewall" "-W" "300"
Path: /gpfs/alpine/stf007/scratch/gmarkoma/io-500-dev/datafiles/io500.2018.11.09-03.12.50
FS: 225668.2 TiB Used FS: 5.7% Inodes: 30000.0 Mi Used Inodes: 16.2%
1008 tasks, 7560000 files
Continue stonewall hit min: 763 max: 1440 avg: 1058.7
stonewall rank 0: 1341 of 1440
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 24374.460 24374.460 24374.460 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 6754.378 6754.378 6754.378 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2018 03:38:37 --
stonewall rank 13: 1052 of 1440
stonewall rank 517: 1052 of 1440
stonewall rank 325: 1140 of 1440
stonewall rank 829: 1159 of 1440
stonewall rank 708: 1093 of 1440
stonewall rank 908: 1109 of 1440
stonewall rank 824: 1015 of 1440
stonewall rank 320: 1025 of 1440
stonewall rank 637: 892 of 1440
stonewall rank 133: 895 of 1440
stonewall rank 684: 998 of 1440
stonewall rank 488: 1063 of 1440
stonewall rank 276: 994 of 1440
stonewall rank 220: 975 of 1440
stonewall rank 180: 1001 of 1440
stonewall rank 780: 993 of 1440
stonewall rank 496: 1238 of 1440
stonewall rank 204: 1094 of 1440
stonewall rank 528: 1123 of 1440
stonewall rank 404: 1104 of 1440
stonewall rank 986: 1112 of 1440
stonewall rank 1000: 1246 of 1440
stonewall rank 724: 975 of 1440
stonewall rank 103: 946 of 1440
stonewall rank 992: 1068 of 1440
stonewall rank 308: 1226 of 1440
stonewall rank 607: 952 of 1440
stonewall rank 578: 1037 of 1440
stonewall rank 523: 839 of 1440
stonewall rank 74: 1038 of 1440
stonewall rank 22: 1052 of 1440
stonewall rank 738: 988 of 1440
stonewall rank 961: 1048 of 1440
stonewall rank 790: 1048 of 1440
stonewall rank 482: 1113 of 1440
stonewall rank 399: 1101 of 1440
stonewall rank 615: 1130 of 1440
stonewall rank 24: 1125 of 1440
stonewall rank 12: 1133 of 1440
stonewall rank 1004: 955 of 1440
stonewall rank 556: 1059 of 1440
stonewall rank 606: 937 of 1440
stonewall rank 11: 1126 of 1440
stonewall rank 515: 1153 of 1440
stonewall rank 812: 1222 of 1440
stonewall rank 524: 894 of 1440
stonewall rank 20: 895 of 1440
stonewall rank 526: 1053 of 1440
stonewall rank 500: 973 of 1440
stonewall rank 516: 1136 of 1440
stonewall rank 234: 1017 of 1440
stonewall rank 123: 1159 of 1440
stonewall rank 53: 1259 of 1440
stonewall rank 605: 979 of 1440
stonewall rank 655: 1144 of 1440
stonewall rank 664: 1053 of 1440
stonewall rank 160: 1049 of 1440
stonewall rank 52: 1050 of 1440
stonewall rank 19: 871 of 1440
stonewall rank 260: 1002 of 1440
stonewall rank 51: 882 of 1440
stonewall rank 417: 1001 of 1440
stonewall rank 557: 1271 of 1440
stonewall rank 903: 1091 of 1440
stonewall rank 764: 999 of 1440
stonewall rank 485: 894 of 1440
stonewall rank 314: 1167 of 1440
stonewall rank 101: 973 of 1440
stonewall rank 95: 1072 of 1440
stonewall rank 111: 1130 of 1440
stonewall rank 353: 1287 of 1440
stonewall rank 522: 1062 of 1440
stonewall rank 1003: 1066 of 1440
stonewall rank 286: 1049 of 1440
stonewall rank 543: 1037 of 1440
stonewall rank 935: 999 of 1440
stonewall rank 102: 913 of 1440
stonewall rank 599: 1067 of 1440
stonewall rank 457: 1048 of 1440
stonewall rank 857: 1281 of 1440
stonewall rank 64: 1267 of 1440
stonewall rank 449: 954 of 1440
stonewall rank 39: 1037 of 1440
stonewall rank 56: 975 of 1440
stonewall rank 431: 999 of 1440
stonewall rank 499: 1063 of 1440
stonewall rank 818: 1168 of 1440
stonewall rank 686: 1033 of 1440
stonewall rank 650: 995 of 1440
stonewall rank 646: 949 of 1440
stonewall rank 856: 1129 of 1440
stonewall rank 184: 934 of 1440
stonewall rank 560: 975 of 1440
stonewall rank 18: 1048 of 1440
stonewall rank 707: 1027 of 1440
stonewall rank 73: 1071 of 1440
stonewall rank 748: 960 of 1440
stonewall rank 922: 1198 of 1440
stonewall rank 352: 1130 of 1440
stonewall rank 151: 1145 of 1440
stonewall rank 374: 869 of 1440
stonewall rank 480: 1059 of 1440
stonewall rank 678: 972 of 1440
stonewall rank 274: 951 of 1440
stonewall rank 890: 1062 of 1440
stonewall rank 819: 936 of 1440
stonewall rank 554: 1006 of 1440
stonewall rank 300: 1162 of 1440
stonewall rank 182: 1032 of 1440
stonewall rank 977: 1004 of 1440
stonewall rank 688: 930 of 1440
stonewall rank 487: 970 of 1440
stonewall rank 953: 963 of 1440
stonewall rank 142: 945 of 1440
stonewall rank 555: 882 of 1440
stonewall rank 627: 1161 of 1440
stonewall rank 513: 1048 of 1440
stonewall rank 219: 935 of 1440
stonewall rank 878: 860 of 1440
stonewall rank 984: 1048 of 1440
stonewall rank 577: 1089 of 1440
stonewall rank 177: 1281 of 1440
stonewall rank 921: 1008 of 1440
stonewall rank 855: 944 of 1440
stonewall rank 991: 970 of 1440
stonewall rank 651: 1035 of 1440
stonewall rank 55: 1110 of 1440
stonewall rank 128: 1126 of 1440
stonewall rank 478: 1115 of 1440
stonewall rank 85: 1045 of 1440
stonewall rank 383: 894 of 1440
stonewall rank 589: 1048 of 1440
stonewall rank 3: 1113 of 1440
stonewall rank 137: 1002 of 1440
stonewall rank 386: 1064 of 1440
stonewall rank 203: 1045 of 1440
stonewall rank 418: 1196 of 1440
stonewall rank 641: 1002 of 1440
stonewall rank 147: 1031 of 1440
stonewall rank 69: 959 of 1440
stonewall rank 174: 950 of 1440
stonewall rank 176: 984 of 1440
stonewall rank 266: 956 of 1440
stonewall rank 982: 1115 of 1440
stonewall rank 770: 956 of 1440
stonewall rank 568: 1262 of 1440
stonewall rank 735: 1181 of 1440
stonewall rank 989: 893 of 1440
stonewall rank 38: 998 of 1440
stonewall rank 9: 1049 of 1440
stonewall rank 190: 1274 of 1440
stonewall rank 339: 983 of 1440
stonewall rank 843: 983 of 1440
stonewall rank 681: 1261 of 1440
stonewall rank 895: 1123 of 1440
stonewall rank 518: 981 of 1440
stonewall rank 544: 917 of 1440
stonewall rank 728: 933 of 1440
stonewall rank 926: 1161 of 1440
stonewall rank 680: 982 of 1440
stonewall rank 632: 1125 of 1440
stonewall rank 723: 939 of 1440
stonewall rank 573: 956 of 1440
stonewall rank 50: 1007 of 1440
stonewall rank 676: 980 of 1440
stonewall rank 231: 1161 of 1440
stonewall rank 475: 1139 of 1440
stonewall rank 595: 1116 of 1440
stonewall rank 306: 1026 of 1440
stonewall rank 389: 978 of 1440
stonewall rank 146: 1019 of 1440
stonewall rank 761: 1018 of 1440
stonewall rank 1001: 1090 of 1440
stonewall rank 924: 1233 of 1440
stonewall rank 424: 932 of 1440
stonewall rank 428: 999 of 1440
stonewall rank 208: 965 of 1440
stonewall rank 153: 1057 of 1440
stonewall rank 669: 1131 of 1440
stonewall rank 195: 1018 of 1440
stonewall rank 110: 1106 of 1440
stonewall rank 645: 888 of 1440
stonewall rank 887: 885 of 1440
stonewall rank 804: 1169 of 1440
stonewall rank 966: 1141 of 1440
stonewall rank 525: 1092 of 1440
stonewall rank 765: 912 of 1440
stonewall rank 979: 1141 of 1440
stonewall rank 959: 1071 of 1440
stonewall rank 810: 1026 of 1440
stonewall rank 694: 1266 of 1440
stonewall rank 462: 1153 of 1440
stonewall rank 315: 939 of 1440
stonewall rank 778: 957 of 1440
stonewall rank 130: 1157 of 1440
stonewall rank 21: 1092 of 1440
stonewall rank 504: 1341 of 1440
stonewall rank 89: 764 of 1440
stonewall rank 712: 966 of 1440
stonewall rank 699: 1020 of 1440
stonewall rank 811: 1117 of 1440
stonewall rank 928: 1030 of 1440
stonewall rank 670: 1154 of 1440
stonewall rank 35: 951 of 1440
stonewall rank 455: 1067 of 1440
stonewall rank 657: 1056 of 1440
stonewall rank 593: 763 of 1440
stonewall rank 63: 1037 of 1440
stonewall rank 336: 952 of 1440
stonewall rank 814: 1070 of 1440
stonewall rank 243: 1007 of 1440
stonewall rank 5: 1211 of 1440
stonewall rank 783: 935 of 1440
stonewall rank 932: 1002 of 1440
stonewall rank 697: 1082 of 1440
stonewall rank 542: 999 of 1440
stonewall rank 415: 1063 of 1440
stonewall rank 138: 885 of 1440
stonewall rank 57: 995 of 1440
stonewall rank 310: 1071 of 1440
stonewall rank 951: 983 of 1440
stonewall rank 371: 1214 of 1440
stonewall rank 460: 1137 of 1440
stonewall rank 690: 957 of 1440
stonewall rank 537: 1187 of 1440
stonewall rank 671: 990 of 1440
stonewall rank 489: 1085 of 1440
stonewall rank 244: 967 of 1440
stonewall rank 60: 1191 of 1440
stonewall rank 821: 1038 of 1440
stonewall rank 380: 1010 of 1440
stonewall rank 7: 1155 of 1440
stonewall rank 193: 1058 of 1440
stonewall rank 757: 1000 of 1440
stonewall rank 444: 1212 of 1440
stonewall rank 840: 944 of 1440
stonewall rank 297: 1034 of 1440
stonewall rank 893: 978 of 1440
stonewall rank 40: 917 of 1440
stonewall rank 614: 1126 of 1440
stonewall rank 623: 1023 of 1440
stonewall rank 775: 934 of 1440
stonewall rank 758: 1000 of 1440
stonewall rank 564: 1190 of 1440
stonewall rank 803: 948 of 1440
stonewall rank 993: 1083 of 1440
stonewall rank 509: 1212 of 1440
stonewall rank 826: 1236 of 1440
stonewall rank 188: 1099 of 1440
stonewall rank 343: 1101 of 1440
stonewall rank 447: 984 of 1440
stonewall rank 769: 1033 of 1440
stonewall rank 144: 1152 of 1440
stonewall rank 251: 1007 of 1440
stonewall rank 559: 1114 of 1440
stonewall rank 639: 1059 of 1440
stonewall rank 725: 890 of 1440
stonewall rank 473: 1009 of 1440
stonewall rank 401: 926 of 1440
stonewall rank 801: 1051 of 1440
stonewall rank 361: 949 of 1440
stonewall rank 330: 1151 of 1440
stonewall rank 507: 1088 of 1440
stonewall rank 691: 1090 of 1440
stonewall rank 99: 1096 of 1440
stonewall rank 224: 926 of 1440
stonewall rank 175: 1141 of 1440
stonewall rank 945: 1040 of 1440
stonewall rank 420: 1216 of 1440
stonewall rank 273: 900 of 1440
stonewall rank 265: 1038 of 1440
stonewall rank 391: 1131 of 1440
stonewall rank 497: 1116 of 1440
stonewall rank 86: 1387 of 1440
stonewall rank 654: 1209 of 1440
stonewall rank 65: 1019 of 1440
stonewall rank 561: 986 of 1440
stonewall rank 642: 882 of 1440
stonewall rank 465: 1053 of 1440
stonewall rank 919: 1064 of 1440
stonewall rank 960: 982 of 1440
stonewall rank 692: 1098 of 1440
stonewall rank 539: 950 of 1440
stonewall rank 737: 1079 of 1440
stonewall rank 172: 979 of 1440
stonewall rank 433: 989 of 1440
stonewall rank 782: 844 of 1440
stonewall rank 351: 946 of 1440
stonewall rank 267: 1088 of 1440
stonewall rank 884: 1008 of 1440
stonewall rank 94: 922 of 1440
stonewall rank 157: 931 of 1440
stonewall rank 253: 999 of 1440
stonewall rank 569: 1000 of 1440
stonewall rank 141: 901 of 1440
stonewall rank 215: 997 of 1440
stonewall rank 470: 1070 of 1440
stonewall rank 905: 921 of 1440
stonewall rank 858: 1000 of 1440
stonewall rank 322: 1194 of 1440
stonewall rank 14: 981 of 1440
stonewall rank 187: 1090 of 1440
stonewall rank 603: 1093 of 1440
stonewall rank 732: 1083 of 1440
stonewall rank 167: 991 of 1440
stonewall rank 422: 1162 of 1440
stonewall rank 937: 991 of 1440
stonewall rank 278: 855 of 1440
stonewall rank 309: 1003 of 1440
stonewall rank 798: 1421 of 1440
stonewall rank 456: 973 of 1440
stonewall rank 385: 870 of 1440
stonewall rank 648: 1152 of 1440
stonewall rank 755: 1007 of 1440
stonewall rank 567: 1036 of 1440
stonewall rank 218: 1124 of 1440
stonewall rank 303: 1087 of 1440
stonewall rank 834: 1160 of 1440
stonewall rank 519: 948 of 1440
stonewall rank 91: 1126 of 1440
stonewall rank 33: 1188 of 1440
stonewall rank 612: 1158 of 1440
stonewall rank 743: 1089 of 1440
stonewall rank 441: 1039 of 1440
stonewall rank 165: 1110 of 1440
stonewall rank 809: 952 of 1440
stonewall rank 261: 912 of 1440
stonewall rank 472: 1291 of 1440
stonewall rank 590: 1403 of 1440
stonewall rank 148: 986 of 1440
stonewall rank 199: 1040 of 1440
stonewall rank 254: 1008 of 1440
stonewall rank 727: 1068 of 1440
stonewall rank 925: 1016 of 1440
stonewall rank 17: 1296 of 1440
stonewall rank 604: 1233 of 1440
stonewall rank 166: 1147 of 1440
stonewall rank 813: 1017 of 1440
stonewall rank 264: 881 of 1440
stonewall rank 976: 1278 of 1440
stonewall rank 870: 984 of 1440
stonewall rank 294: 1415 of 1440
stonewall rank 900: 1056 of 1440
stonewall rank 495: 1144 of 1440
stonewall rank 964: 1152 of 1440
stonewall rank 584: 1156 of 1440
stonewall rank 701: 1159 of 1440
stonewall rank 759: 1244 of 1440
stonewall rank 634: 1141 of 1440
stonewall rank 630: 871 of 1440
stonewall rank 221: 884 of 1440
stonewall rank 281: 1192 of 1440
stonewall rank 347: 934 of 1440
stonewall rank 414: 1039 of 1440
stonewall rank 30: 892 of 1440
stonewall rank 511: 1157 of 1440
stonewall rank 189: 1060 of 1440
stonewall rank 150: 1202 of 1440
stonewall rank 75: 1088 of 1440
stonewall rank 32: 1027 of 1440
stonewall rank 239: 1089 of 1440
stonewall rank 677: 825 of 1440
stonewall rank 938: 1061 of 1440
stonewall rank 317: 1038 of 1440
stonewall rank 768: 923 of 1440
stonewall rank 875: 1212 of 1440
stonewall rank 868: 1410 of 1440
stonewall rank 388: 937 of 1440
stonewall rank 502: 1040 of 1440
stonewall rank 80: 1157 of 1440
stonewall rank 652: 974 of 1440
stonewall rank 705: 1183 of 1440
stonewall rank 49: 1122 of 1440
stonewall rank 131: 1331 of 1440
stonewall rank 114: 1059 of 1440
stonewall rank 714: 1140 of 1440
stonewall rank 777: 897 of 1440
stonewall rank 342: 1038 of 1440
stonewall rank 802: 1137 of 1440
stonewall rank 999: 1155 of 1440
stonewall rank 534: 888 of 1440
stonewall rank 72: 1128 of 1440
stonewall rank 108: 1170 of 1440
stonewall rank 225: 1064 of 1440
stonewall rank 679: 1144 of 1440
stonewall rank 307: 1105 of 1440
stonewall rank 981: 1138 of 1440
stonewall rank 364: 1406 of 1440
stonewall rank 825: 1103 of 1440
stonewall rank 892: 937 of 1440
stonewall rank 1006: 1038 of 1440
stonewall rank 186: 946 of 1440
stonewall rank 661: 927 of 1440
stonewall rank 201: 1182 of 1440
stonewall rank 255: 1245 of 1440
stonewall rank 120: 971 of 1440
stonewall rank 722: 1127 of 1440
stonewall rank 257: 1019 of 1440
stonewall rank 289: 1064 of 1440
stonewall rank 332: 1041 of 1440
stonewall rank 967: 919 of 1440
stonewall rank 512: 999 of 1440
stonewall rank 693: 1069 of 1440
stonewall rank 579: 1086 of 1440
stonewall rank 536: 1026 of 1440
stonewall rank 845: 910 of 1440
stonewall rank 889: 869 of 1440
stonewall rank 28: 950 of 1440
stonewall rank 703: 1049 of 1440
stonewall rank 553: 1114 of 1440
stonewall rank 643: 999 of 1440
stonewall rank 119: 1025 of 1440
stonewall rank 719: 996 of 1440
stonewall rank 785: 1195 of 1440
stonewall rank 327: 1327 of 1440
stonewall rank 461: 893 of 1440
stonewall rank 521: 1323 of 1440
stonewall rank 183: 851 of 1440
stonewall rank 582: 1014 of 1440
stonewall rank 47: 1301 of 1440
stonewall rank 729: 1064 of 1440
stonewall rank 434: 1061 of 1440
stonewall rank 815: 1069 of 1440
stonewall rank 345: 945 of 1440
stonewall rank 771: 1092 of 1440
stonewall rank 476: 1124 of 1440
stonewall rank 402: 1111 of 1440
stonewall rank 494: 1003 of 1440
stonewall rank 702: 1244 of 1440
stonewall rank 100: 1237 of 1440
stonewall rank 134: 1162 of 1440
stonewall rank 115: 1067 of 1440
stonewall rank 378: 1033 of 1440
stonewall rank 299: 949 of 1440
stonewall rank 836: 1024 of 1440
stonewall rank 498: 902 of 1440
stonewall rank 54: 1090 of 1440
stonewall rank 232: 1111 of 1440
stonewall rank 948: 1212 of 1440
stonewall rank 416: 936 of 1440
stonewall rank 851: 934 of 1440
stonewall rank 397: 1079 of 1440
stonewall rank 493: 1063 of 1440
stonewall rank 82: 981 of 1440
stonewall rank 15: 949 of 1440
stonewall rank 198: 1256 of 1440
stonewall rank 58: 1077 of 1440
stonewall rank 140: 1109 of 1440
stonewall rank 619: 1067 of 1440
stonewall rank 280: 889 of 1440
stonewall rank 271: 928 of 1440
stonewall rank 354: 1004 of 1440
stonewall rank 793: 1060 of 1440
stonewall rank 27: 1294 of 1440
stonewall rank 510: 1014 of 1440
stonewall rank 66: 1083 of 1440
stonewall rank 233: 1079 of 1440
stonewall rank 443: 1032 of 1440
stonewall rank 425: 1256 of 1440
stonewall rank 816: 1000 of 1440
stonewall rank 847: 1065 of 1440
stonewall rank 762: 995 of 1440
stonewall rank 882: 1034 of 1440
stonewall rank 969: 1051 of 1440
stonewall rank 831: 1323 of 1440
stonewall rank 392: 1030 of 1440
stonewall rank 997: 1074 of 1440
stonewall rank 598: 961 of 1440
stonewall rank 747: 1005 of 1440
stonewall rank 61: 1139 of 1440
stonewall rank 41: 1025 of 1440
stonewall rank 635: 1334 of 1440
stonewall rank 279: 940 of 1440
stonewall rank 366: 983 of 1440
stonewall rank 807: 1092 of 1440
stonewall rank 1002: 902 of 1440
stonewall rank 178: 1050 of 1440
stonewall rank 78: 1026 of 1440
stonewall rank 104: 1141 of 1440
stonewall rank 624: 971 of 1440
stonewall rank 228: 1083 of 1440
stonewall rank 947: 1040 of 1440
stonewall rank 849: 944 of 1440
stonewall rank 477: 1138 of 1440
stonewall rank 331: 847 of 1440
stonewall rank 81: 1124 of 1440
stonewall rank 159: 970 of 1440
stonewall rank 698: 1021 of 1440
stonewall rank 241: 966 of 1440
stonewall rank 562: 1066 of 1440
stonewall rank 143: 1027 of 1440
stonewall rank 44: 1147 of 1440
stonewall rank 618: 1058 of 1440
stonewall rank 223: 1038 of 1440
stonewall rank 929: 1357 of 1440
stonewall rank 377: 1135 of 1440
stonewall rank 365: 1082 of 1440
stonewall rank 403: 1131 of 1440
stonewall rank 458: 874 of 1440
stonewall rank 532: 951 of 1440
stonewall rank 8: 996 of 1440
stonewall rank 682: 1064 of 1440
stonewall rank 580: 991 of 1440
stonewall rank 608: 1140 of 1440
stonewall rank 170: 1153 of 1440
stonewall rank 311: 1071 of 1440
stonewall rank 381: 1261 of 1440
stonewall rank 396: 1041 of 1440
stonewall rank 84: 1056 of 1440
stonewall rank 653: 1166 of 1440
stonewall rank 745: 967 of 1440
stonewall rank 565: 1143 of 1440
stonewall rank 135: 1058 of 1440
stonewall rank 551: 1278 of 1440
stonewall rank 718: 1151 of 1440
stonewall rank 432: 1148 of 1440
stonewall rank 781: 1098 of 1440
stonewall rank 850: 1124 of 1440
stonewall rank 913: 923 of 1440
stonewall rank 860: 1121 of 1440
stonewall rank 486: 1189 of 1440
stonewall rank 962: 876 of 1440
stonewall rank 530: 980 of 1440
stonewall rank 685: 1043 of 1440
stonewall rank 570: 1083 of 1440
stonewall rank 230: 1195 of 1440
stonewall rank 674: 1151 of 1440
stonewall rank 421: 998 of 1440
stonewall rank 305: 956 of 1440
stonewall rank 341: 913 of 1440
stonewall rank 763: 1135 of 1440
stonewall rank 881: 1135 of 1440
stonewall rank 298: 1138 of 1440
stonewall rank 839: 832 of 1440
stonewall rank 896: 1030 of 1440
stonewall rank 587: 1102 of 1440
stonewall rank 659: 1024 of 1440
stonewall rank 196: 1144 of 1440
stonewall rank 749: 953 of 1440
stonewall rank 129: 1049 of 1440
stonewall rank 214: 1151 of 1440
stonewall rank 277: 1090 of 1440
stonewall rank 988: 956 of 1440
stonewall rank 533: 1275 of 1440
stonewall rank 687: 847 of 1440
stonewall rank 173: 826 of 1440
stonewall rank 940: 1044 of 1440
stonewall rank 873: 1084 of 1440
stonewall rank 980: 1133 of 1440
stonewall rank 408: 1253 of 1440
stonewall rank 359: 1119 of 1440
stonewall rank 326: 1255 of 1440
stonewall rank 888: 1243 of 1440
stonewall rank 963: 1063 of 1440
stonewall rank 663: 969 of 1440
stonewall rank 753: 839 of 1440
stonewall rank 576: 1126 of 1440
stonewall rank 558: 1092 of 1440
stonewall rank 644: 1110 of 1440
stonewall rank 602: 992 of 1440
stonewall rank 626: 1088 of 1440
stonewall rank 923: 1112 of 1440
stonewall rank 272: 1174 of 1440
stonewall rank 340: 883 of 1440
stonewall rank 410: 957 of 1440
stonewall rank 483: 1143 of 1440
stonewall rank 29: 1275 of 1440
stonewall rank 585: 1119 of 1440
stonewall rank 200: 1060 of 1440
stonewall rank 79: 1048 of 1440
stonewall rank 168: 839 of 1440
stonewall rank 936: 1147 of 1440
stonewall rank 846: 1051 of 1440
stonewall rank 968: 1123 of 1440
stonewall rank 384: 1255 of 1440
stonewall rank 501: 895 of 1440
stonewall rank 965: 892 of 1440
stonewall rank 92: 862 of 1440
stonewall rank 145: 1199 of 1440
stonewall rank 197: 1159 of 1440
stonewall rank 744: 960 of 1440
stonewall rank 36: 1023 of 1440
stonewall rank 112: 916 of 1440
stonewall rank 713: 928 of 1440
stonewall rank 930: 1006 of 1440
stonewall rank 369: 1085 of 1440
stonewall rank 918: 1060 of 1440
stonewall rank 363: 1286 of 1440
stonewall rank 987: 1142 of 1440
stonewall rank 25: 1107 of 1440
stonewall rank 695: 881 of 1440
stonewall rank 583: 1044 of 1440
stonewall rank 734: 1192 of 1440
stonewall rank 436: 1014 of 1440
stonewall rank 822: 1054 of 1440
stonewall rank 259: 1130 of 1440
stonewall rank 291: 1101 of 1440
stonewall rank 1005: 900 of 1440
stonewall rank 586: 975 of 1440
stonewall rank 155: 1022 of 1440
stonewall rank 545: 1024 of 1440
stonewall rank 647: 1016 of 1440
stonewall rank 117: 1060 of 1440
stonewall rank 210: 1139 of 1440
stonewall rank 872: 1242 of 1440
stonewall rank 975: 982 of 1440
stonewall rank 869: 1053 of 1440
stonewall rank 830: 1255 of 1440
stonewall rank 529: 1107 of 1440
stonewall rank 191: 883 of 1440
stonewall rank 71: 1001 of 1440
stonewall rank 613: 1044 of 1440
stonewall rank 672: 846 of 1440
stonewall rank 319: 1243 of 1440
stonewall rank 766: 949 of 1440
stonewall rank 469: 1135 of 1440
stonewall rank 906: 1113 of 1440
stonewall rank 897: 993 of 1440
stonewall rank 6: 1007 of 1440
stonewall rank 149: 1167 of 1440
stonewall rank 206: 959 of 1440
stonewall rank 48: 1125 of 1440
stonewall rank 540: 1024 of 1440
stonewall rank 216: 971 of 1440
stonewall rank 419: 1115 of 1440
stonewall rank 787: 1152 of 1440
stonewall rank 409: 942 of 1440
stonewall rank 867: 1289 of 1440
stonewall rank 324: 1125 of 1440
stonewall rank 492: 999 of 1440
stonewall rank 575: 1011 of 1440
stonewall rank 105: 981 of 1440
stonewall rank 229: 983 of 1440
stonewall rank 171: 1053 of 1440
stonewall rank 823: 1241 of 1440
stonewall rank 797: 1147 of 1440
stonewall rank 894: 937 of 1440
stonewall rank 704: 1074 of 1440
stonewall rank 249: 859 of 1440
stonewall rank 59: 942 of 1440
stonewall rank 633: 1047 of 1440
stonewall rank 715: 1240 of 1440
stonewall rank 426: 1006 of 1440
stonewall rank 776: 1174 of 1440
stonewall rank 346: 1108 of 1440
stonewall rank 914: 1006 of 1440
stonewall rank 996: 1023 of 1440
stonewall rank 463: 935 of 1440
stonewall rank 31: 1021 of 1440
stonewall rank 179: 986 of 1440
stonewall rank 649: 1180 of 1440
stonewall rank 96: 948 of 1440
stonewall rank 437: 1304 of 1440
stonewall rank 312: 1001 of 1440
stonewall rank 258: 994 of 1440
stonewall rank 844: 884 of 1440
stonewall rank 974: 1071 of 1440
stonewall rank 293: 1146 of 1440
stonewall rank 83: 1106 of 1440
stonewall rank 505: 1171 of 1440
stonewall rank 156: 1041 of 1440
stonewall rank 700: 1128 of 1440
stonewall rank 247: 1002 of 1440
stonewall rank 566: 1123 of 1440
stonewall rank 933: 1116 of 1440
stonewall rank 885: 1260 of 1440
stonewall rank 413: 1229 of 1440
stonewall rank 865: 950 of 1440
stonewall rank 990: 1188 of 1440
stonewall rank 531: 1295 of 1440
stonewall rank 506: 1045 of 1440
stonewall rank 683: 969 of 1440
stonewall rank 76: 1000 of 1440
stonewall rank 609: 980 of 1440
stonewall rank 116: 1031 of 1440
stonewall rank 736: 1110 of 1440
stonewall rank 287: 1039 of 1440
stonewall rank 304: 1040 of 1440
stonewall rank 262: 933 of 1440
stonewall rank 471: 981 of 1440
stonewall rank 795: 1110 of 1440
stonewall rank 321: 1100 of 1440
stonewall rank 958: 955 of 1440
stonewall rank 660: 1049 of 1440
stonewall rank 751: 1001 of 1440
stonewall rank 710: 963 of 1440
stonewall rank 552: 1125 of 1440
stonewall rank 34: 1180 of 1440
stonewall rank 638: 1163 of 1440
stonewall rank 720: 941 of 1440
stonewall rank 920: 895 of 1440
stonewall rank 283: 1156 of 1440
stonewall rank 835: 847 of 1440
stonewall rank 453: 1053 of 1440
stonewall rank 26: 980 of 1440
stonewall rank 611: 1117 of 1440
stonewall rank 127: 1267 of 1440
stonewall rank 237: 966 of 1440
stonewall rank 162: 1190 of 1440
stonewall rank 318: 1047 of 1440
stonewall rank 848: 1140 of 1440
stonewall rank 773: 1082 of 1440
stonewall rank 474: 988 of 1440
stonewall rank 792: 914 of 1440
stonewall rank 398: 964 of 1440
stonewall rank 588: 1049 of 1440
stonewall rank 205: 1175 of 1440
stonewall rank 538: 1182 of 1440
stonewall rank 382: 1070 of 1440
stonewall rank 911: 970 of 1440
stonewall rank 329: 1122 of 1440
stonewall rank 995: 889 of 1440
stonewall rank 1007: 1054 of 1440
stonewall rank 454: 968 of 1440
stonewall rank 527: 921 of 1440
stonewall rank 10: 1000 of 1440
stonewall rank 181: 1043 of 1440
stonewall rank 97: 1126 of 1440
stonewall rank 733: 981 of 1440
stonewall rank 665: 885 of 1440
stonewall rank 941: 1306 of 1440
stonewall rank 313: 1318 of 1440
stonewall rank 794: 1167 of 1440
stonewall rank 899: 1020 of 1440
stonewall rank 596: 862 of 1440
stonewall rank 252: 911 of 1440
stonewall rank 563: 942 of 1440
stonewall rank 37: 1017 of 1440
stonewall rank 139: 999 of 1440
stonewall rank 126: 872 of 1440
stonewall rank 211: 1241 of 1440
stonewall rank 779: 1024 of 1440
stonewall rank 368: 1254 of 1440
stonewall rank 467: 1083 of 1440
stonewall rank 871: 1255 of 1440
stonewall rank 828: 1124 of 1440
stonewall rank 452: 966 of 1440
stonewall rank 109: 1048 of 1440
stonewall rank 625: 903 of 1440
stonewall rank 236: 1359 of 1440
stonewall rank 666: 1188 of 1440
stonewall rank 285: 947 of 1440
stonewall rank 269: 1110 of 1440
stonewall rank 901: 1038 of 1440
stonewall rank 594: 1381 of 1440
stonewall rank 1: 1174 of 1440
stonewall rank 202: 881 of 1440
stonewall rank 68: 1067 of 1440
stonewall rank 786: 1088 of 1440
stonewall rank 879: 1028 of 1440
stonewall rank 323: 1395 of 1440
stonewall rank 535: 1026 of 1440
stonewall rank 185: 1070 of 1440
stonewall rank 631: 1279 of 1440
stonewall rank 675: 1058 of 1440
stonewall rank 944: 1241 of 1440
stonewall rank 464: 1119 of 1440
stonewall rank 912: 1253 of 1440
stonewall rank 387: 1049 of 1440
stonewall rank 490: 999 of 1440
stonewall rank 957: 1051 of 1440
stonewall rank 158: 1223 of 1440
stonewall rank 245: 952 of 1440
stonewall rank 62: 1121 of 1440
stonewall rank 548: 1147 of 1440
stonewall rank 726: 949 of 1440
stonewall rank 931: 914 of 1440
stonewall rank 335: 830 of 1440
stonewall rank 503: 1055 of 1440
stonewall rank 514: 1001 of 1440
stonewall rank 689: 1070 of 1440
stonewall rank 741: 939 of 1440
stonewall rank 620: 1031 of 1440
stonewall rank 673: 999 of 1440
stonewall rank 440: 1240 of 1440
stonewall rank 808: 1053 of 1440
stonewall rank 268: 1306 of 1440
stonewall rank 978: 986 of 1440
stonewall rank 288: 906 of 1440
stonewall rank 395: 1018 of 1440
stonewall rank 998: 965 of 1440
stonewall rank 93: 1149 of 1440
stonewall rank 754: 1086 of 1440
stonewall rank 43: 1171 of 1440
stonewall rank 610: 828 of 1440
stonewall rank 344: 1141 of 1440
stonewall rank 379: 938 of 1440
stonewall rank 357: 1242 of 1440
stonewall rank 448: 1114 of 1440
stonewall rank 4: 1206 of 1440
stonewall rank 601: 1128 of 1440
stonewall rank 226: 1122 of 1440
stonewall rank 943: 943 of 1440
stonewall rank 282: 1088 of 1440
stonewall rank 290: 1196 of 1440
stonewall rank 706: 885 of 1440
stonewall rank 756: 911 of 1440
stonewall rank 547: 1176 of 1440
stonewall rank 132: 848 of 1440
stonewall rank 222: 967 of 1440
stonewall rank 164: 1038 of 1440
stonewall rank 934: 895 of 1440
stonewall rank 772: 1280 of 1440
stonewall rank 880: 1034 of 1440
stonewall rank 355: 1105 of 1440
stonewall rank 917: 1223 of 1440
stonewall rank 572: 1067 of 1440
stonewall rank 621: 1066 of 1440
stonewall rank 949: 1026 of 1440
stonewall rank 479: 1060 of 1440
stonewall rank 902: 965 of 1440
stonewall rank 491: 930 of 1440
stonewall rank 90: 1382 of 1440
stonewall rank 152: 1271 of 1440
stonewall rank 541: 1018 of 1440
stonewall rank 136: 1278 of 1440
stonewall rank 212: 869 of 1440
stonewall rank 429: 1141 of 1440
stonewall rank 789: 946 of 1440
stonewall rank 349: 827 of 1440
stonewall rank 876: 1004 of 1440
stonewall rank 859: 1111 of 1440
stonewall rank 520: 972 of 1440
stonewall rank 459: 1088 of 1440
stonewall rank 2: 1046 of 1440
stonewall rank 67: 966 of 1440
stonewall rank 98: 992 of 1440
stonewall rank 118: 1192 of 1440
stonewall rank 817: 1318 of 1440
stonewall rank 296: 872 of 1440
stonewall rank 87: 972 of 1440
stonewall rank 662: 1224 of 1440
stonewall rank 240: 955 of 1440
stonewall rank 45: 1177 of 1440
stonewall rank 636: 850 of 1440
stonewall rank 235: 1055 of 1440
stonewall rank 213: 1108 of 1440
stonewall rank 275: 1013 of 1440
stonewall rank 430: 895 of 1440
stonewall rank 372: 1011 of 1440
stonewall rank 400: 1089 of 1440
stonewall rank 16: 969 of 1440
stonewall rank 107: 1117 of 1440
stonewall rank 113: 1158 of 1440
stonewall rank 740: 1274 of 1440
stonewall rank 446: 1182 of 1440
stonewall rank 820: 898 of 1440
stonewall rank 983: 1060 of 1440
stonewall rank 800: 873 of 1440
stonewall rank 393: 991 of 1440
stonewall rank 994: 993 of 1440
stonewall rank 597: 1153 of 1440
stonewall rank 709: 993 of 1440
stonewall rank 550: 964 of 1440
stonewall rank 640: 1292 of 1440
stonewall rank 169: 967 of 1440
stonewall rank 350: 919 of 1440
stonewall rank 774: 950 of 1440
stonewall rank 863: 1120 of 1440
stonewall rank 833: 1099 of 1440
stonewall rank 956: 967 of 1440
stonewall rank 508: 1207 of 1440
stonewall rank 571: 966 of 1440
stonewall rank 106: 807 of 1440
stonewall rank 742: 1214 of 1440
stonewall rank 445: 1016 of 1440
stonewall rank 481: 1153 of 1440
stonewall rank 656: 1260 of 1440
stonewall rank 194: 1029 of 1440
stonewall rank 668: 1040 of 1440
stonewall rank 853: 826 of 1440
stonewall rank 263: 1202 of 1440
stonewall rank 886: 1070 of 1440
stonewall rank 827: 1412 of 1440
stonewall rank 952: 1116 of 1440
stonewall rank 23: 918 of 1440
stonewall rank 77: 900 of 1440
stonewall rank 600: 947 of 1440
stonewall rank 739: 1059 of 1440
stonewall rank 717: 1108 of 1440
stonewall rank 435: 1008 of 1440
stonewall rank 316: 899 of 1440
stonewall rank 915: 1138 of 1440
stonewall rank 358: 1046 of 1440
stonewall rank 302: 980 of 1440
stonewall rank 985: 1164 of 1440
stonewall rank 250: 1085 of 1440
stonewall rank 163: 1234 of 1440
stonewall rank 423: 912 of 1440
stonewall rank 784: 915 of 1440
stonewall rank 348: 945 of 1440
stonewall rank 767: 1203 of 1440
stonewall rank 207: 1092 of 1440
stonewall rank 238: 1205 of 1440
stonewall rank 209: 945 of 1440
stonewall rank 442: 1148 of 1440
stonewall rank 907: 1105 of 1440
stonewall rank 466: 1214 of 1440
stonewall rank 862: 1046 of 1440
stonewall rank 301: 1054 of 1440
stonewall rank 891: 1046 of 1440
stonewall rank 246: 1177 of 1440
stonewall rank 427: 915 of 1440
stonewall rank 581: 898 of 1440
stonewall rank 616: 919 of 1440
stonewall rank 730: 1124 of 1440
stonewall rank 716: 870 of 1440
stonewall rank 950: 1196 of 1440
stonewall rank 791: 1037 of 1440
stonewall rank 854: 920 of 1440
stonewall rank 973: 1131 of 1440
stonewall rank 916: 1070 of 1440
stonewall rank 367: 1196 of 1440
stonewall rank 832: 1156 of 1440
stonewall rank 390: 918 of 1440
stonewall rank 591: 976 of 1440
stonewall rank 549: 1167 of 1440
stonewall rank 161: 885 of 1440
stonewall rank 927: 912 of 1440
stonewall rank 842: 938 of 1440
stonewall rank 760: 1007 of 1440
stonewall rank 376: 1034 of 1440
stonewall rank 292: 1238 of 1440
stonewall rank 617: 1162 of 1440
stonewall rank 439: 940 of 1440
stonewall rank 971: 1082 of 1440
stonewall rank 411: 1138 of 1440
stonewall rank 394: 1279 of 1440
stonewall rank 484: 957 of 1440
stonewall rank 88: 1086 of 1440
stonewall rank 711: 1075 of 1440
stonewall rank 750: 1187 of 1440
stonewall rank 270: 948 of 1440
stonewall rank 883: 939 of 1440
stonewall rank 574: 864 of 1440
stonewall rank 125: 975 of 1440
stonewall rank 942: 1231 of 1440
stonewall rank 407: 979 of 1440
stonewall rank 334: 821 of 1440
stonewall rank 805: 1054 of 1440
stonewall rank 592: 1088 of 1440
stonewall rank 242: 1015 of 1440
stonewall rank 338: 935 of 1440
stonewall rank 874: 1111 of 1440
stonewall rank 70: 869 of 1440
stonewall rank 622: 1192 of 1440
stonewall rank 438: 1235 of 1440
stonewall rank 295: 996 of 1440
stonewall rank 333: 1253 of 1440
stonewall rank 451: 1021 of 1440
stonewall rank 696: 1324 of 1440
stonewall rank 667: 1243 of 1440
stonewall rank 898: 1281 of 1440
stonewall rank 122: 1086 of 1440
stonewall rank 721: 1191 of 1440
stonewall rank 946: 1145 of 1440
stonewall rank 284: 976 of 1440
stonewall rank 972: 1170 of 1440
stonewall rank 362: 1103 of 1440
stonewall rank 806: 980 of 1440
stonewall rank 328: 1142 of 1440
stonewall rank 192: 1320 of 1440
stonewall rank 248: 907 of 1440
stonewall rank 852: 946 of 1440
stonewall rank 954: 1119 of 1440
stonewall rank 939: 1011 of 1440
stonewall rank 970: 1228 of 1440
stonewall rank 904: 1092 of 1440
stonewall rank 866: 1132 of 1440
stonewall rank 799: 994 of 1440
stonewall rank 752: 901 of 1440
stonewall rank 375: 1024 of 1440
stonewall rank 955: 1021 of 1440
stonewall rank 227: 969 of 1440
stonewall rank 468: 1158 of 1440
stonewall rank 406: 1180 of 1440
stonewall rank 746: 1018 of 1440
stonewall rank 337: 1069 of 1440
stonewall rank 370: 1110 of 1440
stonewall rank 629: 974 of 1440
stonewall rank 731: 952 of 1440
stonewall rank 121: 903 of 1440
stonewall rank 360: 895 of 1440
stonewall rank 838: 857 of 1440
stonewall rank 841: 1081 of 1440
stonewall rank 877: 1069 of 1440
stonewall rank 412: 1051 of 1440
stonewall rank 796: 1132 of 1440
stonewall rank 356: 1103 of 1440
stonewall rank 837: 1256 of 1440
stonewall rank 864: 896 of 1440
stonewall rank 910: 1179 of 1440
stonewall rank 450: 1105 of 1440
stonewall rank 658: 1082 of 1440
stonewall rank 124: 1174 of 1440
stonewall rank 42: 929 of 1440
stonewall rank 909: 1013 of 1440
stonewall rank 154: 1064 of 1440
stonewall rank 628: 1177 of 1440
stonewall rank 546: 939 of 1440
stonewall rank 788: 976 of 1440
stonewall rank 405: 1005 of 1440
stonewall rank 217: 1209 of 1440
stonewall rank 256: 996 of 1440
stonewall rank 46: 964 of 1440
stonewall rank 373: 1069 of 1440
- result_summary
-
[RESULT] BW phase 1 ior_easy_write 2158.700 GB/s : time 362.34 seconds
[RESULT] IOPS phase 1 mdtest_easy_write 3071.260 kiops : time 352.36 seconds
[RESULT] BW phase 2 ior_hard_write 0.572 GB/s : time 462.76 seconds
[RESULT] IOPS phase 2 mdtest_hard_write 24.375 kiops : time 327.20 seconds
[RESULT] IOPS phase 3 find 21780.030 kiops : time 46.35 seconds
[RESULT] BW phase 3 ior_easy_read 1788.320 GB/s : time 437.39 seconds
[RESULT] IOPS phase 4 mdtest_easy_stat 28769.400 kiops : time 52.52 seconds
[RESULT] BW phase 4 ior_hard_read 27.403 GB/s : time 9.66 seconds
[RESULT] IOPS phase 5 mdtest_hard_stat 560.886 kiops : time 19.36 seconds
[RESULT] IOPS phase 6 mdtest_easy_delete 2699.000 kiops : time 398.34 seconds
[RESULT] IOPS phase 7 mdtest_hard_read 15354.700 kiops : time 17.02 seconds
[RESULT] IOPS phase 8 mdtest_hard_delete 26.504 kiops : time 181.79 seconds
[SCORE] Bandwidth 88.2049 GB/s : IOPS 1522.68 kiops : TOTAL 366.48