Exascaler-on-GCP

Institution Google
Client Procs Per Node
Client Operating System Exascaler
Client Operating System Version 5.0 PRE CentOS 7.x
Client Kernel Version 3.10.0-862.14.4.el7_lustre.ddn0.x86_64

DATA SERVER

Storage Type NVMe Local
Volatile Memory 60GB
Storage Interface NVMe
Network Ethernet 16Gbps
Software Version Lustre 2.11.56 (MASTER)
OS Version Exascaler 5.0 PRE

INFORMATION

Client Nodes 60
Client Total Procs 480
Metadata Nodes 5
Metadata Storage Devices 4
Data Nodes 60
Data Storage Devices 4

METADATA

Easy Write 173.61 kIOP/s
Easy Stat 347.68 kIOP/s
Easy Delete 98.22 kIOP/s
Hard Write 1.10 kIOP/s
Hard Read 174.06 kIOP/s
Hard Stat 81.64 kIOP/s
Hard Delete 57.25 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_stonewall_timer=300 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=/lustre/lgcpfs/client/io500/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=/lustre/lgcpfs/client/io500/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
  lfs setdirstripe -c 20 $io500_workdir/mdt_easy
  lfs setdirstripe -c 20 $io500_workdir/mdt_hard
  lfs setdirstripe -c 20 -D $io500_workdir/mdt_easy
  lfs setdirstripe -c 20 -D $io500_workdir/mdt_hard
  mkdir -p $io500_workdir/ior_hard
  lfs setstripe -c -1 -S 47104k $io500_workdir/ior_hard
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 480 -ppn 1 -f /lustre/lgcpfs/client/cfg/machines.txt"
}

function setup_ior_easy {
  # io500_ior_easy_size is the amount of data written per rank in MiB units,
  # but it can be any number as long as it is somehow used to scale the IOR
  # runtime as part of io500_ior_easy_params
  #io500_ior_easy_size=2000
  io500_ior_easy_size=60000
  # 2M writes, 2 GB per proc, file per proc
  io500_ior_easy_params="-t 2048k -b ${io500_ior_easy_size}m -F"
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
  #io500_mdtest_easy_files_per_proc=25000
  io500_mdtest_easy_files_per_proc=335000
}

function setup_ior_hard {
  #io500_ior_hard_writes_per_proc=10000
  io500_ior_hard_writes_per_proc=38600
  io500_ior_hard_other_options="" #e.g., -E to keep precreated files using lfs setstripe, or -a MPIIO
}

function setup_mdt_hard {
  #io500_mdtest_hard_files_per_proc=5000
  io500_mdtest_hard_files_per_proc=60000
  io500_mdtest_hard_other_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind 
  io500_find_cmd_args="-s $io500_stonewall_timer -r $io500_result_dir/pfind_results"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  io500_mdreal_params="-P=5000 -I=1000"
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Add key/value pairs defining your system
# Feel free to add extra ones if you'd like
function extra_description {
  # top level info
  io500_info_system_name='Google Cloud Platform'      # e.g. Oakforest-PACS
  io500_info_institute_name='Google and DDN'   # e.g. JCAHPC
  io500_info_storage_age_in_months='0'# not install date but age since last refresh
  io500_info_storage_install_date='11/18'  # MM/YY
  io500_info_filesystem='Lustre'     # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
  io500_info_filesystem_version='2.11.56'
  io500_info_filesystem_vendor='DDN'
  # client side info
  io500_info_num_client_nodes='60'
  io500_info_procs_per_node='8'
  # server side info
  io500_info_num_metadata_server_nodes='xxx'
  io500_info_num_data_server_nodes='xxx'
  io500_info_num_data_storage_devices='xxx'  # if you have 5 data servers, and each has 5 drives, then this number is 25
  io500_info_num_metadata_storage_devices='xxx'  # if you have 2 metadata servers, and each has 5 drives, then this number is 10
  io500_info_data_storage_type='xxx' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_metadata_storage_type='xxx' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_storage_network='xxx' # infiniband, omnipath, ethernet, etc
  io500_info_storage_interface='xxx' # SAS, SATA, NVMe, etc
  # miscellaneous
  io500_info_whatever='WhateverElseYouThinkRelevant'
}

main
ior_easy_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Sat Nov 10 13:56:12 2018
Command line        : /scratch/io-500-dev-master/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 2048k -b 60000m -F -o /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_easy/ior_file_easy -O stoneWallingStatusFile=/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_easy/stonewall
Machine             : Linux lgcp-client0
TestID              : 0
StartTime           : Sat Nov 10 13:56:12 2018
Path                : /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_easy
FS                  : 86.6 TiB   Used FS: 32.6%   Inodes: 2400.0 Mi   Used Inodes: 2.7%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 480
clients per node    : 8
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 58.59 GiB
aggregate filesize  : 27.47 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      52585      61440000   2048.00    0.015864   547.66     0.009039   547.68     0   
Max Read:  52585.14 MiB/sec (55139.51 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        52585.14   52585.14   52585.14       0.00   26292.57   26292.57   26292.57       0.00  547.68329     0    480   8    1   1     1        1         0    0      1 62914560000  2097152 28800000.0 POSIX      0
Finished            : Sat Nov 10 14:05:20 2018
ior_easy_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Sat Nov 10 13:04:41 2018
Command line        : /scratch/io-500-dev-master/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 2048k -b 60000m -F -o /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_easy/ior_file_easy -O stoneWallingStatusFile=/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux lgcp-client0
TestID              : 0
StartTime           : Sat Nov 10 13:04:41 2018
Path                : /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_easy
FS                  : 86.6 TiB   Used FS: 0.0%   Inodes: 702.8 Mi   Used Inodes: 0.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 480
clients per node    : 8
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 58.59 GiB
aggregate filesize  : 27.47 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 15835 max: 30000 -- min data: 30.9 GiB mean data: 56.3 GiB time: 300.3s
write     71400      61440000   2048.00    0.017389   403.32     0.016542   403.36     0   
Max Write: 71400.31 MiB/sec (74868.65 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       71400.31   71400.31   71400.31       0.00   35700.16   35700.16   35700.16       0.00  403.35959     0    480   8    1   1     1        1         0    0      1 62914560000  2097152 28800000.0 POSIX      0
Finished            : Sat Nov 10 13:11:25 2018
ior_hard_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Sat Nov 10 14:08:27 2018
Command line        : /scratch/io-500-dev-master/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 38600 -o /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_hard/IOR_file -O stoneWallingStatusFile=/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_hard/stonewall
Machine             : Linux lgcp-client0
TestID              : 0
StartTime           : Sat Nov 10 14:08:27 2018
Path                : /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_hard
FS                  : 86.6 TiB   Used FS: 32.6%   Inodes: 2400.0 Mi   Used Inodes: 2.7%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 38600
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 480
clients per node    : 8
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 811.15 GiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      8293       45.91      45.91      0.025722   100.12     0.014362   100.16     0   
Max Read:  8292.68 MiB/sec (8695.51 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         8292.68    8292.68    8292.68       0.00  184979.29  184979.29  184979.29       0.00  100.16256     0    480   8    1   0     1        1         0    0  38600    47008    47008  830616.2 POSIX      0
Finished            : Sat Nov 10 14:10:08 2018
ior_hard_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Sat Nov 10 13:17:39 2018
Command line        : /scratch/io-500-dev-master/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 38600 -o /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_hard/IOR_file -O stoneWallingStatusFile=/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux lgcp-client0
TestID              : 0
StartTime           : Sat Nov 10 13:17:39 2018
Path                : /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_hard
FS                  : 86.6 TiB   Used FS: 31.7%   Inodes: 2400.0 Mi   Used Inodes: 2.6%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 38600
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 480
clients per node    : 8
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 811.15 GiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 17575 max: 38600 -- min data: 0.8 GiB mean data: 1.2 GiB time: 300.0s
write     1481.75    45.91      45.91      0.031263   560.48     0.052326   560.57     0   
Max Write: 1481.75 MiB/sec (1553.72 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        1481.75    1481.75    1481.75       0.00   33052.33   33052.33   33052.33       0.00  560.56556     0    480   8    1   0     1        1         0    0  38600    47008    47008  830616.2 POSIX      0
Finished            : Sat Nov 10 13:27:00 2018
mdtest_easy_delete
-- started at 11/10/2018 14:10:32 --

mdtest-1.9.3 was launched with 480 total task(s) on 60 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-r" "-F" "-d" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_easy" "-n" "135000" "-u" "-L" "-x" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_easy-stonewall"
Path: /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41
FS: 86.6 TiB   Used FS: 32.6%   Inodes: 2400.0 Mi   Used Inodes: 2.7%

480 tasks, 64800000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :      98216.736      98216.736      98216.736          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.672          0.672          0.672          0.000

-- finished at 11/10/2018 14:21:33 --
mdtest_easy_stat
-- started at 11/10/2018 14:05:21 --

mdtest-1.9.3 was launched with 480 total task(s) on 60 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-T" "-F" "-d" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_easy" "-n" "135000" "-u" "-L" "-x" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_easy-stonewall"
Path: /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41
FS: 86.6 TiB   Used FS: 32.6%   Inodes: 2400.0 Mi   Used Inodes: 2.7%

480 tasks, 64800000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :     347679.535     347679.535     347679.535          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/10/2018 14:08:27 --
mdtest_easy_write
-- started at 11/10/2018 13:11:25 --

mdtest-1.9.3 was launched with 480 total task(s) on 60 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-C" "-F" "-d" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_easy" "-n" "135000" "-u" "-L" "-x" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_easy-stonewall" "-W" "300"
Path: /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41
FS: 86.6 TiB   Used FS: 31.7%   Inodes: 2400.0 Mi   Used Inodes: 0.0%

480 tasks, 64800000 files
stonewall rank 367: 121205 of 135000 
stonewall rank 420: 110948 of 135000 
stonewall rank 1: 126344 of 135000 
stonewall rank 422: 119976 of 135000 
stonewall rank 423: 110898 of 135000 
stonewall rank 305: 119561 of 135000 
stonewall rank 7: 121897 of 135000 
stonewall rank 70: 121655 of 135000 
stonewall rank 135: 119773 of 135000 
stonewall rank 79: 125643 of 135000 
stonewall rank 261: 111260 of 135000 
stonewall rank 86: 121419 of 135000 
stonewall rank 151: 120952 of 135000 
stonewall rank 33: 119202 of 135000 
stonewall rank 39: 116399 of 135000 
stonewall rank 103: 129484 of 135000 
stonewall rank 49: 125633 of 135000 
stonewall rank 110: 112564 of 135000 
stonewall rank 111: 123169 of 135000 
stonewall rank 239: 120061 of 135000 
stonewall rank 60: 111260 of 135000 
stonewall rank 61: 126332 of 135000 
stonewall rank 302: 120381 of 135000 
stonewall rank 303: 111239 of 135000 
stonewall rank 5: 120538 of 135000 
stonewall rank 127: 122546 of 135000 
stonewall rank 430: 121726 of 135000 
stonewall rank 369: 121390 of 135000 
stonewall rank 15: 120038 of 135000 
stonewall rank 199: 125143 of 135000 
stonewall rank 321: 110798 of 135000 
stonewall rank 206: 120607 of 135000 
stonewall rank 391: 121376 of 135000 
stonewall rank 153: 118521 of 135000 
stonewall rank 166: 125206 of 135000 
stonewall rank 229: 126044 of 135000 
stonewall rank 230: 112403 of 135000 
stonewall rank 231: 121702 of 135000 
stonewall rank 359: 120764 of 135000 
stonewall rank 180: 111064 of 135000 
stonewall rank 181: 126241 of 135000 
stonewall rank 62: 119479 of 135000 
stonewall rank 63: 110364 of 135000 
stonewall rank 65: 120367 of 135000 
stonewall rank 366: 124819 of 135000 
stonewall rank 247: 121720 of 135000 
stonewall rank 10: 120676 of 135000 
stonewall rank 71: 111754 of 135000 
stonewall rank 69: 120082 of 135000 
stonewall rank 255: 120311 of 135000 
stonewall rank 433: 118414 of 135000 
stonewall rank 134: 127327 of 135000 
stonewall rank 19: 124701 of 135000 
stonewall rank 22: 126797 of 135000 
stonewall rank 381: 110031 of 135000 
stonewall rank 23: 117835 of 135000 
stonewall rank 326: 121467 of 135000 
stonewall rank 271: 122312 of 135000 
stonewall rank 150: 127762 of 135000 
stonewall rank 93: 118028 of 135000 
stonewall rank 38: 121978 of 135000 
stonewall rank 279: 117459 of 135000 
stonewall rank 343: 129576 of 135000 
stonewall rank 101: 116347 of 135000 
stonewall rank 102: 111770 of 135000 
stonewall rank 406: 125453 of 135000 
stonewall rank 165: 113240 of 135000 
stonewall rank 167: 119141 of 135000 
stonewall rank 289: 125421 of 135000 
stonewall rank 350: 113228 of 135000 
stonewall rank 471: 123250 of 135000 
stonewall rank 295: 112933 of 135000 
stonewall rank 177: 121433 of 135000 
stonewall rank 238: 112898 of 135000 
stonewall rank 119: 120520 of 135000 
stonewall rank 300: 111114 of 135000 
stonewall rank 241: 125837 of 135000 
stonewall rank 182: 119514 of 135000 
stonewall rank 183: 110427 of 135000 
stonewall rank 185: 120342 of 135000 
stonewall rank 6: 125190 of 135000 
stonewall rank 187: 122060 of 135000 
stonewall rank 190: 121181 of 135000 
stonewall rank 431: 111762 of 135000 
stonewall rank 129: 120971 of 135000 
stonewall rank 375: 120192 of 135000 
stonewall rank 133: 118750 of 135000 
stonewall rank 14: 127786 of 135000 
stonewall rank 139: 124832 of 135000 
stonewall rank 262: 127019 of 135000 
stonewall rank 441: 111185 of 135000 
stonewall rank 263: 117891 of 135000 
stonewall rank 146: 121108 of 135000 
stonewall rank 451: 121334 of 135000 
stonewall rank 390: 127016 of 135000 
stonewall rank 393: 118430 of 135000 
stonewall rank 278: 122006 of 135000 
stonewall rank 459: 116661 of 135000 
stonewall rank 223: 130281 of 135000 
stonewall rank 342: 112372 of 135000 
stonewall rank 46: 125095 of 135000 
stonewall rank 285: 112836 of 135000 
stonewall rank 407: 119969 of 135000 
stonewall rank 349: 126126 of 135000 
stonewall rank 410: 113121 of 135000 
stonewall rank 351: 122752 of 135000 
stonewall rank 175: 113004 of 135000 
stonewall rank 357: 120283 of 135000 
stonewall rank 358: 113041 of 135000 
stonewall rank 179: 119396 of 135000 
stonewall rank 421: 126765 of 135000 
stonewall rank 362: 120062 of 135000 
stonewall rank 123: 110785 of 135000 
stonewall rank 125: 120614 of 135000 
stonewall rank 126: 125282 of 135000 
stonewall rank 307: 121890 of 135000 
stonewall rank 310: 120631 of 135000 
stonewall rank 11: 111942 of 135000 
stonewall rank 309: 120404 of 135000 
stonewall rank 435: 119039 of 135000 
stonewall rank 313: 118115 of 135000 
stonewall rank 254: 127964 of 135000 
stonewall rank 78: 123676 of 135000 
stonewall rank 319: 125068 of 135000 
stonewall rank 142: 127043 of 135000 
stonewall rank 201: 111105 of 135000 
stonewall rank 143: 117774 of 135000 
stonewall rank 26: 121091 of 135000 
stonewall rank 388: 122194 of 135000 
stonewall rank 31: 120786 of 135000 
stonewall rank 270: 126915 of 135000 
stonewall rank 389: 109730 of 135000 
stonewall rank 213: 118725 of 135000 
stonewall rank 214: 112206 of 135000 
stonewall rank 398: 122354 of 135000 
stonewall rank 399: 117197 of 135000 
stonewall rank 463: 130115 of 135000 
stonewall rank 161: 115850 of 135000 
stonewall rank 222: 111827 of 135000 
stonewall rank 37: 106023 of 135000 
stonewall rank 466: 124345 of 135000 
stonewall rank 345: 113117 of 135000 
stonewall rank 467: 119966 of 135000 
stonewall rank 228: 128420 of 135000 
stonewall rank 50: 112818 of 135000 
stonewall rank 51: 122233 of 135000 
stonewall rank 55: 113762 of 135000 
stonewall rank 417: 121025 of 135000 
stonewall rank 118: 112575 of 135000 
stonewall rank 479: 119435 of 135000 
stonewall rank 121: 125657 of 135000 
stonewall rank 122: 119875 of 135000 
stonewall rank 363: 110162 of 135000 
stonewall rank 245: 120601 of 135000 
stonewall rank 246: 124493 of 135000 
stonewall rank 427: 122357 of 135000 
stonewall rank 370: 121105 of 135000 
stonewall rank 191: 112326 of 135000 
stonewall rank 189: 121110 of 135000 
stonewall rank 315: 119660 of 135000 
stonewall rank 373: 119518 of 135000 
stonewall rank 374: 127942 of 135000 
stonewall rank 198: 123351 of 135000 
stonewall rank 382: 127109 of 135000 
stonewall rank 81: 111258 of 135000 
stonewall rank 383: 118492 of 135000 
stonewall rank 266: 121549 of 135000 
stonewall rank 91: 121321 of 135000 
stonewall rank 30: 126806 of 135000 
stonewall rank 29: 109689 of 135000 
stonewall rank 334: 112444 of 135000 
stonewall rank 158: 122694 of 135000 
stonewall rank 159: 117473 of 135000 
stonewall rank 283: 130103 of 135000 
stonewall rank 281: 115196 of 135000 
stonewall rank 402: 111715 of 135000 
stonewall rank 97: 106271 of 135000 
stonewall rank 286: 124543 of 135000 
stonewall rank 47: 119750 of 135000 
stonewall rank 170: 113002 of 135000 
stonewall rank 411: 122050 of 135000 
stonewall rank 292: 120090 of 135000 
stonewall rank 415: 113092 of 135000 
stonewall rank 57: 121646 of 135000 
stonewall rank 178: 113123 of 135000 
stonewall rank 301: 126697 of 135000 
stonewall rank 2: 120043 of 135000 
stonewall rank 3: 110671 of 135000 
stonewall rank 425: 120200 of 135000 
stonewall rank 186: 125802 of 135000 
stonewall rank 67: 122296 of 135000 
stonewall rank 250: 121871 of 135000 
stonewall rank 311: 112092 of 135000 
stonewall rank 132: 123982 of 135000 
stonewall rank 9: 120869 of 135000 
stonewall rank 195: 119252 of 135000 
stonewall rank 253: 118640 of 135000 
stonewall rank 434: 127139 of 135000 
stonewall rank 18: 123220 of 135000 
stonewall rank 439: 124694 of 135000 
stonewall rank 442: 127145 of 135000 
stonewall rank 21: 111402 of 135000 
stonewall rank 83: 117767 of 135000 
stonewall rank 446: 120849 of 135000 
stonewall rank 87: 124454 of 135000 
stonewall rank 28: 122711 of 135000 
stonewall rank 211: 122060 of 135000 
stonewall rank 90: 127045 of 135000 
stonewall rank 89: 109487 of 135000 
stonewall rank 333: 117782 of 135000 
stonewall rank 94: 113076 of 135000 
stonewall rank 339: 116784 of 135000 
stonewall rank 100: 119690 of 135000 
stonewall rank 403: 130125 of 135000 
stonewall rank 41: 115865 of 135000 
stonewall rank 42: 111680 of 135000 
stonewall rank 157: 106686 of 135000 
stonewall rank 287: 119722 of 135000 
stonewall rank 469: 126258 of 135000 
stonewall rank 348: 127645 of 135000 
stonewall rank 290: 112544 of 135000 
stonewall rank 171: 122685 of 135000 
stonewall rank 174: 123229 of 135000 
stonewall rank 52: 120426 of 135000 
stonewall rank 115: 113100 of 135000 
stonewall rank 117: 120600 of 135000 
stonewall rank 58: 112737 of 135000 
stonewall rank 120: 111132 of 135000 
stonewall rank 361: 126115 of 135000 
stonewall rank 242: 119669 of 135000 
stonewall rank 243: 110937 of 135000 
stonewall rank 4: 125095 of 135000 
stonewall rank 365: 119864 of 135000 
stonewall rank 306: 125154 of 135000 
stonewall rank 130: 121413 of 135000 
stonewall rank 371: 111857 of 135000 
stonewall rank 372: 125091 of 135000 
stonewall rank 429: 120665 of 135000 
stonewall rank 75: 119341 of 135000 
stonewall rank 73: 118682 of 135000 
stonewall rank 314: 127240 of 135000 
stonewall rank 318: 123704 of 135000 
stonewall rank 379: 124932 of 135000 
stonewall rank 82: 126301 of 135000 
stonewall rank 141: 111683 of 135000 
stonewall rank 443: 118015 of 135000 
stonewall rank 386: 121596 of 135000 
stonewall rank 207: 124778 of 135000 
stonewall rank 88: 121607 of 135000 
stonewall rank 331: 121525 of 135000 
stonewall rank 210: 127288 of 135000 
stonewall rank 273: 117643 of 135000 
stonewall rank 215: 128203 of 135000 
stonewall rank 154: 112187 of 135000 
stonewall rank 338: 121527 of 135000 
stonewall rank 99: 116628 of 135000 
stonewall rank 280: 119275 of 135000 
stonewall rank 43: 130108 of 135000 
stonewall rank 461: 115951 of 135000 
stonewall rank 282: 112196 of 135000 
stonewall rank 217: 106314 of 135000 
stonewall rank 346: 125487 of 135000 
stonewall rank 45: 112494 of 135000 
stonewall rank 164: 119901 of 135000 
stonewall rank 107: 119263 of 135000 
stonewall rank 109: 125823 of 135000 
stonewall rank 108: 128384 of 135000 
stonewall rank 470: 113130 of 135000 
stonewall rank 291: 122074 of 135000 
stonewall rank 294: 123015 of 135000 
stonewall rank 412: 119392 of 135000 
stonewall rank 235: 113264 of 135000 
stonewall rank 237: 121565 of 135000 
stonewall rank 298: 113216 of 135000 
stonewall rank 59: 119727 of 135000 
stonewall rank 124: 123632 of 135000 
stonewall rank 426: 125368 of 135000 
stonewall rank 251: 112233 of 135000 
stonewall rank 252: 125387 of 135000 
stonewall rank 249: 120099 of 135000 
stonewall rank 193: 119406 of 135000 
stonewall rank 194: 127407 of 135000 
stonewall rank 197: 120185 of 135000 
stonewall rank 138: 124392 of 135000 
stonewall rank 259: 124021 of 135000 
stonewall rank 322: 126895 of 135000 
stonewall rank 323: 117576 of 135000 
stonewall rank 327: 124430 of 135000 
stonewall rank 148: 122332 of 135000 
stonewall rank 450: 127576 of 135000 
stonewall rank 453: 118534 of 135000 
stonewall rank 335: 127455 of 135000 
stonewall rank 274: 112142 of 135000 
stonewall rank 98: 121271 of 135000 
stonewall rank 219: 117269 of 135000 
stonewall rank 40: 119443 of 135000 
stonewall rank 163: 130241 of 135000 
stonewall rank 401: 116164 of 135000 
stonewall rank 462: 112075 of 135000 
stonewall rank 457: 106319 of 135000 
stonewall rank 106: 125018 of 135000 
stonewall rank 105: 113206 of 135000 
stonewall rank 284: 119006 of 135000 
stonewall rank 347: 120305 of 135000 
stonewall rank 169: 126651 of 135000 
stonewall rank 408: 128596 of 135000 
stonewall rank 475: 113582 of 135000 
stonewall rank 477: 120362 of 135000 
stonewall rank 478: 113353 of 135000 
stonewall rank 299: 119181 of 135000 
stonewall rank 244: 123843 of 135000 
stonewall rank 66: 125052 of 135000 
stonewall rank 131: 112273 of 135000 
stonewall rank 12: 126126 of 135000 
stonewall rank 13: 118910 of 135000 
stonewall rank 74: 127883 of 135000 
stonewall rank 257: 119406 of 135000 
stonewall rank 438: 123861 of 135000 
stonewall rank 202: 126617 of 135000 
stonewall rank 203: 116979 of 135000 
stonewall rank 147: 124638 of 135000 
stonewall rank 268: 122723 of 135000 
stonewall rank 330: 127565 of 135000 
stonewall rank 329: 110020 of 135000 
stonewall rank 455: 127692 of 135000 
stonewall rank 394: 112350 of 135000 
stonewall rank 218: 122156 of 135000 
stonewall rank 460: 118979 of 135000 
stonewall rank 341: 115598 of 135000 
stonewall rank 162: 111451 of 135000 
stonewall rank 337: 106558 of 135000 
stonewall rank 226: 125490 of 135000 
stonewall rank 225: 112330 of 135000 
stonewall rank 344: 119398 of 135000 
stonewall rank 227: 119665 of 135000 
stonewall rank 409: 125104 of 135000 
stonewall rank 168: 128031 of 135000 
stonewall rank 355: 114143 of 135000 
stonewall rank 297: 120834 of 135000 
stonewall rank 418: 112315 of 135000 
stonewall rank 419: 120083 of 135000 
stonewall rank 360: 111480 of 135000 
stonewall rank 364: 123558 of 135000 
stonewall rank 68: 110597 of 135000 
stonewall rank 72: 125648 of 135000 
stonewall rank 317: 120194 of 135000 
stonewall rank 378: 123100 of 135000 
stonewall rank 260: 123741 of 135000 
stonewall rank 25: 123781 of 135000 
stonewall rank 27: 125008 of 135000 
stonewall rank 328: 122542 of 135000 
stonewall rank 449: 109222 of 135000 
stonewall rank 92: 110536 of 135000 
stonewall rank 95: 127053 of 135000 
stonewall rank 34: 111993 of 135000 
stonewall rank 36: 120632 of 135000 
stonewall rank 458: 121178 of 135000 
stonewall rank 340: 118991 of 135000 
stonewall rank 221: 115550 of 135000 
stonewall rank 277: 105879 of 135000 
stonewall rank 465: 113461 of 135000 
stonewall rank 44: 119342 of 135000 
stonewall rank 48: 127686 of 135000 
stonewall rank 414: 123125 of 135000 
stonewall rank 232: 120245 of 135000 
stonewall rank 240: 110778 of 135000 
stonewall rank 424: 124262 of 135000 
stonewall rank 188: 110276 of 135000 
stonewall rank 312: 125048 of 135000 
stonewall rank 258: 123127 of 135000 
stonewall rank 380: 123484 of 135000 
stonewall rank 325: 124165 of 135000 
stonewall rank 267: 125358 of 135000 
stonewall rank 208: 122406 of 135000 
stonewall rank 149: 110005 of 135000 
stonewall rank 152: 110234 of 135000 
stonewall rank 155: 127527 of 135000 
stonewall rank 454: 112731 of 135000 
stonewall rank 156: 119799 of 135000 
stonewall rank 220: 119495 of 135000 
stonewall rank 397: 106472 of 135000 
stonewall rank 405: 113016 of 135000 
stonewall rank 104: 119627 of 135000 
stonewall rank 468: 128922 of 135000 
stonewall rank 54: 123104 of 135000 
stonewall rank 172: 120787 of 135000 
stonewall rank 184: 124413 of 135000 
stonewall rank 308: 110258 of 135000 
stonewall rank 432: 125509 of 135000 
stonewall rank 437: 120686 of 135000 
stonewall rank 20: 123699 of 135000 
stonewall rank 385: 123507 of 135000 
stonewall rank 447: 125112 of 135000 
stonewall rank 448: 122276 of 135000 
stonewall rank 209: 109853 of 135000 
stonewall rank 212: 110915 of 135000 
stonewall rank 400: 119367 of 135000 
stonewall rank 288: 129782 of 135000 
stonewall rank 234: 123023 of 135000 
stonewall rank 472: 119998 of 135000 
stonewall rank 304: 124210 of 135000 
stonewall rank 428: 110287 of 135000 
stonewall rank 192: 125313 of 135000 
stonewall rank 377: 120631 of 135000 
stonewall rank 445: 123980 of 135000 
stonewall rank 387: 124815 of 135000 
stonewall rank 269: 110320 of 135000 
stonewall rank 32: 110898 of 135000 
stonewall rank 276: 119880 of 135000 
stonewall rank 160: 119053 of 135000 
stonewall rank 114: 123521 of 135000 
stonewall rank 112: 119503 of 135000 
stonewall rank 64: 123626 of 135000 
stonewall rank 248: 110068 of 135000 
stonewall rank 85: 124605 of 135000 
stonewall rank 332: 110944 of 135000 
stonewall rank 216: 120679 of 135000 
stonewall rank 474: 123517 of 135000 
stonewall rank 352: 120062 of 135000 
stonewall rank 24: 109193 of 135000 
stonewall rank 354: 123297 of 135000 
stonewall rank 324: 108750 of 135000 
stonewall rank 196: 110653 of 135000 
stonewall rank 316: 110285 of 135000 
stonewall rank 444: 108998 of 135000 
stonewall rank 436: 109891 of 135000 
stonewall rank 84: 108610 of 135000 
stonewall rank 396: 119798 of 135000 
stonewall rank 76: 110733 of 135000 
stonewall rank 264: 109003 of 135000 
stonewall rank 376: 110955 of 135000 
stonewall rank 272: 110186 of 135000 
stonewall rank 404: 119510 of 135000 
stonewall rank 392: 110725 of 135000 
stonewall rank 464: 119380 of 135000 
stonewall rank 452: 111055 of 135000 
stonewall rank 224: 119545 of 135000 
stonewall rank 140: 123763 of 135000 
stonewall rank 440: 123191 of 135000 
stonewall rank 80: 122785 of 135000 
stonewall rank 200: 122563 of 135000 
stonewall rank 136: 110653 of 135000 
stonewall rank 144: 109090 of 135000 
stonewall rank 8: 110447 of 135000 
stonewall rank 16: 111167 of 135000 
stonewall rank 204: 108759 of 135000 
stonewall rank 368: 110671 of 135000 
stonewall rank 137: 120060 of 135000 
stonewall rank 384: 108973 of 135000 
stonewall rank 128: 110850 of 135000 
stonewall rank 17: 120026 of 135000 
stonewall rank 77: 120144 of 135000 
stonewall rank 256: 110924 of 135000 
stonewall rank 320: 122844 of 135000 
stonewall rank 275: 127440 of 135000 
stonewall rank 336: 120873 of 135000 
stonewall rank 145: 124113 of 135000 
stonewall rank 395: 127688 of 135000 
stonewall rank 456: 120904 of 135000 
Continue stonewall hit min: 105879 max: 135000 avg: 119872.7 
stonewall rank 0: 111852 of 135000 
stonewall rank 265: 124644 of 135000 
stonewall rank 35: 128015 of 135000 
stonewall rank 96: 119878 of 135000 
stonewall rank 205: 124300 of 135000 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :     173605.766     173605.766     173605.766          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          2.552          2.552          2.552          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/10/2018 13:17:39 --
mdtest_hard_delete
-- started at 11/10/2018 14:21:45 --

mdtest-1.9.3 was launched with 480 total task(s) on 60 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-r" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_hard" "-n" "60000" "-x" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_hard-stonewall"
Path: /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41
FS: 86.6 TiB   Used FS: 32.6%   Inodes: 2400.0 Mi   Used Inodes: 0.1%

480 tasks, 28800000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :      57247.539      57247.539      57247.539          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :         11.024         11.024         11.024          0.000

-- finished at 11/10/2018 14:22:18 --
mdtest_hard_read
-- started at 11/10/2018 14:21:34 --

mdtest-1.9.3 was launched with 480 total task(s) on 60 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-E" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_hard" "-n" "60000" "-x" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_hard-stonewall"
Path: /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41
FS: 86.6 TiB   Used FS: 32.6%   Inodes: 2400.0 Mi   Used Inodes: 0.1%

480 tasks, 28800000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :     174057.486     174057.486     174057.486          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/10/2018 14:21:45 --
mdtest_hard_stat
-- started at 11/10/2018 14:10:08 --

mdtest-1.9.3 was launched with 480 total task(s) on 60 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-T" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_hard" "-n" "60000" "-x" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_hard-stonewall"
Path: /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41
FS: 86.6 TiB   Used FS: 32.6%   Inodes: 2400.0 Mi   Used Inodes: 2.7%

480 tasks, 28800000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :      81643.462      81643.462      81643.462          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/10/2018 14:10:32 --
mdtest_hard_write
-- started at 11/10/2018 13:27:01 --

mdtest-1.9.3 was launched with 480 total task(s) on 60 node(s)
Command line used: /scratch/io-500-dev-master/bin/mdtest "-C" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_hard" "-n" "60000" "-x" "/lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41/mdt_hard-stonewall" "-W" "300"
Path: /lustre/lgcpfs/client/io500/datafiles/io500.2018.11.10-13.04.41
FS: 86.6 TiB   Used FS: 32.6%   Inodes: 2400.0 Mi   Used Inodes: 2.6%

480 tasks, 28800000 files
stonewall rank 301: 3140 of 3956 
stonewall rank 421: 3152 of 3956 
stonewall rank 93: 3005 of 3956 
stonewall rank 285: 3556 of 3956 
stonewall rank 349: 3930 of 3956 
stonewall rank 61: 3159 of 3956 
stonewall rank 181: 3145 of 3956 
stonewall rank 1: 3168 of 3956 
stonewall rank 469: 3940 of 3956 
stonewall rank 121: 3167 of 3956 
stonewall rank 343: 3847 of 3956 
stonewall rank 241: 3142 of 3956 
stonewall rank 213: 2960 of 3956 
stonewall rank 109: 3908 of 3956 
stonewall rank 361: 3162 of 3956 
stonewall rank 333: 2995 of 3956 
stonewall rank 229: 3942 of 3956 
stonewall rank 94: 3276 of 3956 
stonewall rank 273: 2989 of 3956 
stonewall rank 45: 3547 of 3956 
stonewall rank 289: 3931 of 3956 
stonewall rank 350: 3317 of 3956 
stonewall rank 55: 3643 of 3956 
stonewall rank 5: 3163 of 3956 
stonewall rank 71: 3201 of 3956 
stonewall rank 69: 2620 of 3956 
stonewall rank 374: 3177 of 3956 
stonewall rank 198: 3441 of 3956 
stonewall rank 375: 3078 of 3956 
stonewall rank 19: 2499 of 3956 
stonewall rank 261: 3121 of 3956 
stonewall rank 87: 2933 of 3956 
stonewall rank 31: 3054 of 3956 
stonewall rank 389: 2961 of 3956 
stonewall rank 35: 3317 of 3956 
stonewall rank 214: 3248 of 3956 
stonewall rank 33: 3013 of 3956 
stonewall rank 102: 3363 of 3956 
stonewall rank 43: 3849 of 3956 
stonewall rank 165: 3570 of 3956 
stonewall rank 286: 3630 of 3956 
stonewall rank 287: 3061 of 3956 
stonewall rank 169: 3928 of 3956 
stonewall rank 50: 3297 of 3956 
stonewall rank 295: 3658 of 3956 
stonewall rank 354: 3305 of 3956 
stonewall rank 118: 3408 of 3956 
stonewall rank 60: 3016 of 3956 
stonewall rank 245: 3158 of 3956 
stonewall rank 311: 3243 of 3956 
stonewall rank 309: 2626 of 3956 
stonewall rank 381: 3110 of 3956 
stonewall rank 327: 2952 of 3956 
stonewall rank 151: 3089 of 3956 
stonewall rank 95: 3380 of 3956 
stonewall rank 454: 3271 of 3956 
stonewall rank 453: 2964 of 3956 
stonewall rank 222: 3383 of 3956 
stonewall rank 103: 3861 of 3956 
stonewall rank 405: 3569 of 3956 
stonewall rank 406: 3607 of 3956 
stonewall rank 407: 3078 of 3956 
stonewall rank 409: 3903 of 3956 
stonewall rank 170: 3299 of 3956 
stonewall rank 355: 3668 of 3956 
stonewall rank 474: 3292 of 3956 
stonewall rank 358: 3376 of 3956 
stonewall rank 119: 3033 of 3956 
stonewall rank 365: 3158 of 3956 
stonewall rank 7: 2956 of 3956 
stonewall rank 11: 3232 of 3956 
stonewall rank 189: 2617 of 3956 
stonewall rank 14: 3187 of 3956 
stonewall rank 373: 2955 of 3956 
stonewall rank 18: 3448 of 3956 
stonewall rank 15: 3067 of 3956 
stonewall rank 199: 2519 of 3956 
stonewall rank 141: 3138 of 3956 
stonewall rank 26: 3245 of 3956 
stonewall rank 27: 2930 of 3956 
stonewall rank 211: 3036 of 3956 
stonewall rank 449: 2966 of 3956 
stonewall rank 215: 3315 of 3956 
stonewall rank 34: 3250 of 3956 
stonewall rank 37: 3040 of 3956 
stonewall rank 393: 2999 of 3956 
stonewall rank 39: 2900 of 3956 
stonewall rank 342: 3435 of 3956 
stonewall rank 163: 3836 of 3956 
stonewall rank 345: 3572 of 3956 
stonewall rank 466: 3630 of 3956 
stonewall rank 467: 3071 of 3956 
stonewall rank 230: 3294 of 3956 
stonewall rank 415: 3670 of 3956 
stonewall rank 54: 3322 of 3956 
stonewall rank 178: 3391 of 3956 
stonewall rank 117: 3010 of 3956 
stonewall rank 359: 3050 of 3956 
stonewall rank 180: 3028 of 3956 
stonewall rank 182: 3049 of 3956 
stonewall rank 3: 3099 of 3956 
stonewall rank 67: 2975 of 3956 
stonewall rank 131: 3215 of 3956 
stonewall rank 70: 3288 of 3956 
stonewall rank 429: 2634 of 3956 
stonewall rank 74: 3214 of 3956 
stonewall rank 13: 2975 of 3956 
stonewall rank 78: 3475 of 3956 
stonewall rank 75: 3066 of 3956 
stonewall rank 259: 2518 of 3956 
stonewall rank 21: 3140 of 3956 
stonewall rank 85: 3364 of 3956 
stonewall rank 23: 2885 of 3956 
stonewall rank 86: 3243 of 3956 
stonewall rank 147: 2965 of 3956 
stonewall rank 271: 3058 of 3956 
stonewall rank 29: 2962 of 3956 
stonewall rank 275: 3305 of 3956 
stonewall rank 154: 3256 of 3956 
stonewall rank 278: 3246 of 3956 
stonewall rank 397: 3055 of 3956 
stonewall rank 153: 2988 of 3956 
stonewall rank 99: 2934 of 3956 
stonewall rank 402: 3381 of 3956 
stonewall rank 223: 3855 of 3956 
stonewall rank 465: 3578 of 3956 
stonewall rank 106: 3649 of 3956 
stonewall rank 107: 3075 of 3956 
stonewall rank 290: 3305 of 3956 
stonewall rank 412: 3132 of 3956 
stonewall rank 53: 3200 of 3956 
stonewall rank 475: 3663 of 3956 
stonewall rank 114: 3261 of 3956 
stonewall rank 298: 3401 of 3956 
stonewall rank 357: 3025 of 3956 
stonewall rank 351: 3256 of 3956 
stonewall rank 419: 3061 of 3956 
stonewall rank 420: 2985 of 3956 
stonewall rank 362: 3027 of 3956 
stonewall rank 183: 3121 of 3956 
stonewall rank 125: 3177 of 3956 
stonewall rank 247: 2959 of 3956 
stonewall rank 371: 3209 of 3956 
stonewall rank 310: 3304 of 3956 
stonewall rank 129: 2639 of 3956 
stonewall rank 134: 3217 of 3956 
stonewall rank 133: 2980 of 3956 
stonewall rank 258: 3439 of 3956 
stonewall rank 135: 3078 of 3956 
stonewall rank 79: 2556 of 3956 
stonewall rank 81: 3119 of 3956 
stonewall rank 205: 3364 of 3956 
stonewall rank 83: 2872 of 3956 
stonewall rank 326: 3239 of 3956 
stonewall rank 207: 2942 of 3956 
stonewall rank 331: 3055 of 3956 
stonewall rank 149: 2945 of 3956 
stonewall rank 335: 3345 of 3956 
stonewall rank 274: 3259 of 3956 
stonewall rank 338: 3261 of 3956 
stonewall rank 157: 3038 of 3956 
stonewall rank 159: 2950 of 3956 
stonewall rank 462: 3368 of 3956 
stonewall rank 283: 3853 of 3956 
stonewall rank 225: 3554 of 3956 
stonewall rank 166: 3613 of 3956 
stonewall rank 284: 3106 of 3956 
stonewall rank 167: 3083 of 3956 
stonewall rank 410: 3328 of 3956 
stonewall rank 173: 3173 of 3956 
stonewall rank 115: 3642 of 3956 
stonewall rank 294: 3284 of 3956 
stonewall rank 418: 3404 of 3956 
stonewall rank 237: 3032 of 3956 
stonewall rank 116: 3223 of 3956 
stonewall rank 51: 3265 of 3956 
stonewall rank 2: 3025 of 3956 
stonewall rank 363: 3102 of 3956 
stonewall rank 65: 3156 of 3956 
stonewall rank 307: 2985 of 3956 
stonewall rank 431: 3229 of 3956 
stonewall rank 370: 3263 of 3956 
stonewall rank 369: 2617 of 3956 
stonewall rank 194: 3178 of 3956 
stonewall rank 253: 2961 of 3956 
stonewall rank 438: 3399 of 3956 
stonewall rank 195: 3031 of 3956 
stonewall rank 139: 2513 of 3956 
stonewall rank 321: 3140 of 3956 
stonewall rank 325: 3382 of 3956 
stonewall rank 143: 2911 of 3956 
stonewall rank 386: 3237 of 3956 
stonewall rank 148: 3119 of 3956 
stonewall rank 267: 2936 of 3956 
stonewall rank 150: 3272 of 3956 
stonewall rank 391: 3061 of 3956 
stonewall rank 269: 2972 of 3956 
stonewall rank 455: 3358 of 3956 
stonewall rank 334: 3250 of 3956 
stonewall rank 38: 3253 of 3956 
stonewall rank 277: 3035 of 3956 
stonewall rank 279: 2916 of 3956 
stonewall rank 42: 3385 of 3956 
stonewall rank 101: 2740 of 3956 
stonewall rank 403: 3845 of 3956 
stonewall rank 105: 3562 of 3956 
stonewall rank 226: 3635 of 3956 
stonewall rank 404: 3120 of 3956 
stonewall rank 227: 3084 of 3956 
stonewall rank 470: 3297 of 3956 
stonewall rank 293: 3201 of 3956 
stonewall rank 235: 3666 of 3956 
stonewall rank 414: 3310 of 3956 
stonewall rank 238: 3381 of 3956 
stonewall rank 417: 3003 of 3956 
stonewall rank 356: 3255 of 3956 
stonewall rank 171: 3253 of 3956 
stonewall rank 179: 3033 of 3956 
stonewall rank 422: 3047 of 3956 
stonewall rank 423: 3123 of 3956 
stonewall rank 305: 3153 of 3956 
stonewall rank 6: 2933 of 3956 
stonewall rank 427: 2943 of 3956 
stonewall rank 251: 3243 of 3956 
stonewall rank 10: 3271 of 3956 
stonewall rank 9: 2628 of 3956 
stonewall rank 434: 3223 of 3956 
stonewall rank 193: 2979 of 3956 
stonewall rank 138: 3438 of 3956 
stonewall rank 435: 3063 of 3956 
stonewall rank 439: 2492 of 3956 
stonewall rank 201: 3109 of 3956 
stonewall rank 445: 3370 of 3956 
stonewall rank 263: 2874 of 3956 
stonewall rank 146: 3247 of 3956 
stonewall rank 328: 3139 of 3956 
stonewall rank 387: 2927 of 3956 
stonewall rank 210: 3287 of 3956 
stonewall rank 451: 3040 of 3956 
stonewall rank 92: 2907 of 3956 
stonewall rank 89: 2953 of 3956 
stonewall rank 155: 3334 of 3956 
stonewall rank 394: 3245 of 3956 
stonewall rank 36: 3129 of 3956 
stonewall rank 98: 3248 of 3956 
stonewall rank 337: 3046 of 3956 
stonewall rank 339: 2949 of 3956 
stonewall rank 162: 3379 of 3956 
stonewall rank 221: 2735 of 3956 
stonewall rank 463: 3853 of 3956 
stonewall rank 46: 3634 of 3956 
stonewall rank 47: 3074 of 3956 
stonewall rank 110: 3319 of 3956 
stonewall rank 52: 3136 of 3956 
stonewall rank 413: 3174 of 3956 
stonewall rank 175: 3651 of 3956 
stonewall rank 234: 3306 of 3956 
stonewall rank 58: 3367 of 3956 
stonewall rank 297: 3007 of 3956 
stonewall rank 236: 3242 of 3956 
stonewall rank 231: 3251 of 3956 
stonewall rank 299: 3021 of 3956 
stonewall rank 242: 3034 of 3956 
stonewall rank 243: 3134 of 3956 
stonewall rank 425: 3164 of 3956 
stonewall rank 246: 2922 of 3956 
stonewall rank 367: 2946 of 3956 
stonewall rank 191: 3236 of 3956 
stonewall rank 130: 3290 of 3956 
stonewall rank 249: 2634 of 3956 
stonewall rank 132: 3634 of 3956 
stonewall rank 314: 3191 of 3956 
stonewall rank 73: 2941 of 3956 
stonewall rank 318: 3439 of 3956 
stonewall rank 255: 3065 of 3956 
stonewall rank 319: 2541 of 3956 
stonewall rank 22: 3136 of 3956 
stonewall rank 441: 3111 of 3956 
stonewall rank 145: 3337 of 3956 
stonewall rank 323: 2881 of 3956 
stonewall rank 206: 3275 of 3956 
stonewall rank 268: 3100 of 3956 
stonewall rank 447: 2942 of 3956 
stonewall rank 270: 3228 of 3956 
stonewall rank 91: 3060 of 3956 
stonewall rank 212: 2911 of 3956 
stonewall rank 209: 2981 of 3956 
stonewall rank 395: 3342 of 3956 
stonewall rank 276: 3097 of 3956 
stonewall rank 158: 3237 of 3956 
stonewall rank 457: 3049 of 3956 
stonewall rank 399: 2927 of 3956 
stonewall rank 282: 3393 of 3956 
stonewall rank 100: 2756 of 3956 
stonewall rank 341: 2722 of 3956 
stonewall rank 346: 3624 of 3956 
stonewall rank 347: 3066 of 3956 
stonewall rank 348: 3320 of 3956 
stonewall rank 292: 3135 of 3956 
stonewall rank 353: 3186 of 3956 
stonewall rank 174: 3272 of 3956 
stonewall rank 478: 3364 of 3956 
stonewall rank 57: 3024 of 3956 
stonewall rank 476: 3225 of 3956 
stonewall rank 291: 3250 of 3956 
stonewall rank 239: 3051 of 3956 
stonewall rank 302: 3035 of 3956 
stonewall rank 303: 3134 of 3956 
stonewall rank 185: 3146 of 3956 
stonewall rank 426: 2941 of 3956 
stonewall rank 127: 2960 of 3956 
stonewall rank 430: 3312 of 3956 
stonewall rank 372: 3637 of 3956 
stonewall rank 254: 3197 of 3956 
stonewall rank 313: 2952 of 3956 
stonewall rank 378: 3432 of 3956 
stonewall rank 315: 3066 of 3956 
stonewall rank 77: 3056 of 3956 
stonewall rank 379: 2516 of 3956 
stonewall rank 82: 3138 of 3956 
stonewall rank 265: 3385 of 3956 
stonewall rank 203: 2898 of 3956 
stonewall rank 266: 3272 of 3956 
stonewall rank 388: 3133 of 3956 
stonewall rank 330: 3239 of 3956 
stonewall rank 332: 2891 of 3956 
stonewall rank 329: 2971 of 3956 
stonewall rank 156: 3115 of 3956 
stonewall rank 398: 3267 of 3956 
stonewall rank 97: 3035 of 3956 
stonewall rank 459: 2913 of 3956 
stonewall rank 220: 2760 of 3956 
stonewall rank 41: 2727 of 3956 
stonewall rank 164: 3122 of 3956 
stonewall rank 228: 3298 of 3956 
stonewall rank 232: 3132 of 3956 
stonewall rank 473: 3183 of 3956 
stonewall rank 177: 3004 of 3956 
stonewall rank 56: 3266 of 3956 
stonewall rank 411: 3251 of 3956 
stonewall rank 59: 3055 of 3956 
stonewall rank 62: 3020 of 3956 
stonewall rank 63: 3126 of 3956 
stonewall rank 66: 2893 of 3956 
stonewall rank 187: 2958 of 3956 
stonewall rank 250: 3284 of 3956 
stonewall rank 12: 3619 of 3956 
stonewall rank 433: 2963 of 3956 
stonewall rank 317: 3046 of 3956 
stonewall rank 385: 3380 of 3956 
stonewall rank 383: 2897 of 3956 
stonewall rank 446: 3242 of 3956 
stonewall rank 28: 3128 of 3956 
stonewall rank 390: 3260 of 3956 
stonewall rank 452: 2904 of 3956 
stonewall rank 396: 3123 of 3956 
stonewall rank 458: 3238 of 3956 
stonewall rank 217: 3015 of 3956 
stonewall rank 219: 2932 of 3956 
stonewall rank 161: 2748 of 3956 
stonewall rank 44: 3106 of 3956 
stonewall rank 408: 3320 of 3956 
stonewall rank 172: 3128 of 3956 
stonewall rank 113: 3194 of 3956 
stonewall rank 477: 3019 of 3956 
stonewall rank 176: 3235 of 3956 
stonewall rank 471: 3251 of 3956 
stonewall rank 479: 3054 of 3956 
stonewall rank 122: 3042 of 3956 
stonewall rank 244: 3422 of 3956 
stonewall rank 123: 3104 of 3956 
stonewall rank 306: 2937 of 3956 
stonewall rank 190: 3291 of 3956 
stonewall rank 72: 3630 of 3956 
stonewall rank 196: 3083 of 3956 
stonewall rank 437: 3058 of 3956 
stonewall rank 20: 3510 of 3956 
stonewall rank 262: 3153 of 3956 
stonewall rank 25: 3330 of 3956 
stonewall rank 443: 2905 of 3956 
stonewall rank 204: 3152 of 3956 
stonewall rank 392: 2884 of 3956 
stonewall rank 456: 3111 of 3956 
stonewall rank 218: 3230 of 3956 
stonewall rank 340: 2738 of 3956 
stonewall rank 104: 3089 of 3956 
stonewall rank 472: 3141 of 3956 
stonewall rank 233: 3180 of 3956 
stonewall rank 416: 3256 of 3956 
stonewall rank 111: 3240 of 3956 
stonewall rank 4: 3393 of 3956 
stonewall rank 312: 3624 of 3956 
stonewall rank 436: 3086 of 3956 
stonewall rank 17: 3058 of 3956 
stonewall rank 140: 3538 of 3956 
stonewall rank 322: 3141 of 3956 
stonewall rank 324: 3195 of 3956 
stonewall rank 216: 3126 of 3956 
stonewall rank 280: 2757 of 3956 
stonewall rank 344: 3095 of 3956 
stonewall rank 252: 3628 of 3956 
stonewall rank 76: 3095 of 3956 
stonewall rank 257: 3050 of 3956 
stonewall rank 84: 3188 of 3956 
stonewall rank 336: 3109 of 3956 
stonewall rank 400: 2742 of 3956 
stonewall rank 464: 3116 of 3956 
stonewall rank 68: 3175 of 3956 
stonewall rank 432: 3604 of 3956 
stonewall rank 136: 3088 of 3956 
stonewall rank 260: 3537 of 3956 
stonewall rank 142: 3140 of 3956 
stonewall rank 24: 3186 of 3956 
stonewall rank 360: 3021 of 3956 
stonewall rank 364: 3428 of 3956 
stonewall rank 308: 3174 of 3956 
stonewall rank 200: 3508 of 3956 
stonewall rank 264: 3155 of 3956 
stonewall rank 300: 3013 of 3956 
stonewall rank 424: 3400 of 3956 
stonewall rank 366: 2961 of 3956 
stonewall rank 8: 3165 of 3956 
stonewall rank 192: 3624 of 3956 
stonewall rank 16: 3079 of 3956 
stonewall rank 377: 3062 of 3956 
stonewall rank 380: 3543 of 3956 
stonewall rank 202: 3162 of 3956 
stonewall rank 144: 3157 of 3956 
stonewall rank 88: 3135 of 3956 
stonewall rank 30: 3244 of 3956 
stonewall rank 152: 2902 of 3956 
stonewall rank 96: 3147 of 3956 
stonewall rank 40: 2739 of 3956 
stonewall rank 281: 2722 of 3956 
stonewall rank 224: 3102 of 3956 
stonewall rank 468: 3283 of 3956 
stonewall rank 352: 3129 of 3956 
stonewall rank 296: 3237 of 3956 
stonewall rank 120: 3015 of 3956 
stonewall rank 124: 3380 of 3956 
stonewall rank 126: 2901 of 3956 
stonewall rank 428: 3148 of 3956 
stonewall rank 316: 3102 of 3956 
stonewall rank 137: 3051 of 3956 
stonewall rank 80: 3519 of 3956 
stonewall rank 382: 3145 of 3956 
stonewall rank 444: 3183 of 3956 
stonewall rank 208: 3094 of 3956 
stonewall rank 450: 3240 of 3956 
stonewall rank 272: 2929 of 3956 
stonewall rank 460: 2738 of 3956 
stonewall rank 401: 2717 of 3956 
stonewall rank 108: 3300 of 3956 
stonewall rank 112: 3127 of 3956 
stonewall rank 240: 3009 of 3956 
stonewall rank 304: 3419 of 3956 
stonewall rank 186: 2957 of 3956 
stonewall rank 188: 3160 of 3956 
stonewall rank 376: 3073 of 3956 
stonewall rank 197: 3072 of 3956 
stonewall rank 440: 3531 of 3956 
stonewall rank 442: 3148 of 3956 
stonewall rank 384: 3154 of 3956 
stonewall rank 448: 3115 of 3956 
stonewall rank 90: 3241 of 3956 
stonewall rank 32: 2897 of 3956 
stonewall rank 160: 2733 of 3956 
stonewall rank 461: 2725 of 3956 
stonewall rank 168: 3292 of 3956 
Continue stonewall hit min: 2492 max: 3956 avg: 3174.8 
stonewall rank 0: 3019 of 3956 
stonewall rank 184: 3399 of 3956 
stonewall rank 248: 3166 of 3956 
stonewall rank 256: 3091 of 3956 
stonewall rank 320: 3541 of 3956 
stonewall rank 48: 3289 of 3956 
stonewall rank 64: 3391 of 3956 
stonewall rank 368: 3170 of 3956 
stonewall rank 288: 3284 of 3956 
stonewall rank 128: 3162 of 3956 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :      16645.603      16645.603      16645.603          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          2.273          2.273          2.273          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/10/2018 13:55:51 --
result_summary
[RESULT] BW   phase 1            ior_easy_write               69.727 GB/s : time 403.36 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              173.606 kiops : time 374.11 seconds
[RESULT] BW   phase 2            ior_hard_write                1.447 GB/s : time 560.57 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               16.646 kiops : time 1731.14 seconds
[RESULT] IOPS phase 3                      find             3226.800 kiops : time  20.67 seconds
[RESULT] BW   phase 3             ior_easy_read               51.353 GB/s : time 547.68 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              347.680 kiops : time 186.84 seconds
[RESULT] BW   phase 4             ior_hard_read                8.099 GB/s : time 100.16 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               81.644 kiops : time  23.72 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               98.217 kiops : time 661.74 seconds
[RESULT] IOPS phase 7          mdtest_hard_read              174.057 kiops : time  11.34 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               57.248 kiops : time  33.63 seconds
[SCORE] Bandwidth 14.3124 GB/s : IOPS 150.2 kiops : TOTAL 46.3651