SharedScratch

Institution Janelia Research Campus, HHMI
Client Procs Per Node
Client Operating System Scientific-Linux
Client Operating System Version 7.7
Client Kernel Version 3.10.0-1062.9.1.el7.x86_64

DATA SERVER

Storage Type Flash
Volatile Memory 192GB
Storage Interface NVMe
Network 100GbE
Software Version 3.7.0
OS Version 7.7

INFORMATION

Client Nodes 10
Client Total Procs 600
Metadata Nodes 0
Metadata Storage Devices 0
Data Nodes 20
Data Storage Devices 80

METADATA

Easy Write 37.49 kIOP/s
Easy Stat 201.86 kIOP/s
Easy Delete 33.22 kIOP/s
Hard Write 14.29 kIOP/s
Hard Read 38.34 kIOP/s
Hard Stat 60.21 kIOP/s
Hard Delete 14.79 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 600 -ppn 60 -f $PWD/hostfile-10-h09 --bootstrap ssh"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 09:31:50 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -r -R -a POSIX -t 4m --posix.odirect -v -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datadir/2020.06.02-08.56.31-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datadir/2020.06.02-08.56.31-scr/ior_easy/stonewall
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 09:31:50 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr/ior_easy
FS                  : 50.0 TiB   Used FS: 44.0%   Inodes: 858.3 Mi   Used Inodes: 5.0%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-08.56.31-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 4 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 5676.27 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Tue Jun  2 09:31:50 2020

WARNING: Expected aggregate file size       = 6241124352000000.
WARNING: Stat() of aggregate file size      = 9708974899200.
WARNING: Using actual aggregate bytes moved = 9708974899200.
read      16037      4009       0.000233    10158080000 4096       0.008144   577.35     0.002564   577.36     0   
Max Read:  16037.08 MiB/sec (16816.10 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        16037.08   16037.08   16037.08       0.00    4009.27    4009.27    4009.27       0.00  577.36201         NA            NA     0    600  60    1   1     1        1         0    0      1 10401873920000  4194304 9259200.0 POSIX      0
Finished            : Tue Jun  2 09:41:28 2020
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 08:56:45 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -w -a POSIX -t 4m --posix.odirect -v -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datadir/2020.06.02-08.56.31-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datadir/2020.06.02-08.56.31-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 08:56:45 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr/ior_easy
FS                  : 50.0 TiB   Used FS: 17.5%   Inodes: 858.3 Mi   Used Inodes: 1.0%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-08.56.31-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 4 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 5676.27 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Tue Jun  2 08:56:45 2020
478: stonewalling pairs accessed: 2877
115: stonewalling pairs accessed: 2096
181: stonewalling pairs accessed: 1836
551: stonewalling pairs accessed: 1787
409: stonewalling pairs accessed: 1997
334: stonewalling pairs accessed: 2040
415: stonewalling pairs accessed: 2113
28: stonewalling pairs accessed: 1903
562: stonewalling pairs accessed: 1873
66: stonewalling pairs accessed: 2043
358: stonewalling pairs accessed: 3019
419: stonewalling pairs accessed: 1979
58: stonewalling pairs accessed: 1964
233: stonewalling pairs accessed: 2063
479: stonewalling pairs accessed: 2611
581: stonewalling pairs accessed: 3290
526: stonewalling pairs accessed: 1993
100: stonewalling pairs accessed: 2932
313: stonewalling pairs accessed: 1872
241: stonewalling pairs accessed: 2112
360: stonewalling pairs accessed: 3196
14: stonewalling pairs accessed: 2069
447: stonewalling pairs accessed: 2516
587: stonewalling pairs accessed: 1907
498: stonewalling pairs accessed: 2866
103: stonewalling pairs accessed: 2085
359: stonewalling pairs accessed: 3023
397: stonewalling pairs accessed: 2135
59: stonewalling pairs accessed: 1924
207: stonewalling pairs accessed: 1876
443: stonewalling pairs accessed: 1869
599: stonewalling pairs accessed: 2711
534: stonewalling pairs accessed: 3107
106: stonewalling pairs accessed: 2689
337: stonewalling pairs accessed: 3224
242: stonewalling pairs accessed: 3341
365: stonewalling pairs accessed: 2891
2: stonewalling pairs accessed: 2127
230: stonewalling pairs accessed: 1890
449: stonewalling pairs accessed: 2020
580: stonewalling pairs accessed: 2890
520: stonewalling pairs accessed: 2005
107: stonewalling pairs accessed: 3175
309: stonewalling pairs accessed: 1906
361: stonewalling pairs accessed: 3096
36: stonewalling pairs accessed: 2996
431: stonewalling pairs accessed: 2951
576: stonewalling pairs accessed: 2888
513: stonewalling pairs accessed: 2903
118: stonewalling pairs accessed: 2045
301: stonewalling pairs accessed: 3160
254: stonewalling pairs accessed: 2005
403: stonewalling pairs accessed: 1938
15: stonewalling pairs accessed: 2111
180: stonewalling pairs accessed: 2891
448: stonewalling pairs accessed: 2025
547: stonewalling pairs accessed: 1802
499: stonewalling pairs accessed: 2936
102: stonewalling pairs accessed: 3265
319: stonewalling pairs accessed: 1958
404: stonewalling pairs accessed: 3044
16: stonewalling pairs accessed: 3153
437: stonewalling pairs accessed: 2894
569: stonewalling pairs accessed: 2063
510: stonewalling pairs accessed: 1902
91: stonewalling pairs accessed: 1795
306: stonewalling pairs accessed: 2130
413: stonewalling pairs accessed: 2138
7: stonewalling pairs accessed: 3060
214: stonewalling pairs accessed: 1977
475: stonewalling pairs accessed: 1874
545: stonewalling pairs accessed: 2034
523: stonewalling pairs accessed: 1838
93: stonewalling pairs accessed: 1867
317: stonewalling pairs accessed: 1813
283: stonewalling pairs accessed: 1951
381: stonewalling pairs accessed: 2884
41: stonewalling pairs accessed: 2904
420: stonewalling pairs accessed: 2745
567: stonewalling pairs accessed: 1894
489: stonewalling pairs accessed: 1892
99: stonewalling pairs accessed: 3335
342: stonewalling pairs accessed: 1909
250: stonewalling pairs accessed: 2105
392: stonewalling pairs accessed: 2017
40: stonewalling pairs accessed: 1884
238: stonewalling pairs accessed: 3157
469: stonewalling pairs accessed: 2676
577: stonewalling pairs accessed: 3322
497: stonewalling pairs accessed: 3146
71: stonewalling pairs accessed: 3141
344: stonewalling pairs accessed: 3037
405: stonewalling pairs accessed: 3045
22: stonewalling pairs accessed: 3275
445: stonewalling pairs accessed: 2754
571: stonewalling pairs accessed: 2914
503: stonewalling pairs accessed: 1838
98: stonewalling pairs accessed: 2031
307: stonewalling pairs accessed: 1855
373: stonewalling pairs accessed: 1963
8: stonewalling pairs accessed: 3139
456: stonewalling pairs accessed: 2729
555: stonewalling pairs accessed: 2044
521: stonewalling pairs accessed: 2009
90: stonewalling pairs accessed: 1923
339: stonewalling pairs accessed: 1995
380: stonewalling pairs accessed: 1993
51: stonewalling pairs accessed: 3372
450: stonewalling pairs accessed: 1834
598: stonewalling pairs accessed: 1863
493: stonewalling pairs accessed: 2878
81: stonewalling pairs accessed: 2036
308: stonewalling pairs accessed: 3112
390: stonewalling pairs accessed: 3069
5: stonewalling pairs accessed: 2999
440: stonewalling pairs accessed: 1911
549: stonewalling pairs accessed: 1924
538: stonewalling pairs accessed: 1959
109: stonewalling pairs accessed: 2682
310: stonewalling pairs accessed: 2064
394: stonewalling pairs accessed: 3136
18: stonewalling pairs accessed: 2159
225: stonewalling pairs accessed: 1845
468: stonewalling pairs accessed: 2105
592: stonewalling pairs accessed: 2806
516: stonewalling pairs accessed: 1818
62: stonewalling pairs accessed: 2757
340: stonewalling pairs accessed: 1997
374: stonewalling pairs accessed: 3123
57: stonewalling pairs accessed: 1892
232: stonewalling pairs accessed: 1962
444: stonewalling pairs accessed: 3064
572: stonewalling pairs accessed: 2704
524: stonewalling pairs accessed: 2256
69: stonewalling pairs accessed: 3214
354: stonewalling pairs accessed: 3445
367: stonewalling pairs accessed: 3120
47: stonewalling pairs accessed: 2184
446: stonewalling pairs accessed: 1839
583: stonewalling pairs accessed: 1960
525: stonewalling pairs accessed: 2108
77: stonewalling pairs accessed: 2864
352: stonewalling pairs accessed: 3168
395: stonewalling pairs accessed: 3179
52: stonewalling pairs accessed: 2919
463: stonewalling pairs accessed: 2023
574: stonewalling pairs accessed: 3047
486: stonewalling pairs accessed: 2920
67: stonewalling pairs accessed: 1967
355: stonewalling pairs accessed: 3858
396: stonewalling pairs accessed: 2027
44: stonewalling pairs accessed: 1775
236: stonewalling pairs accessed: 1940
425: stonewalling pairs accessed: 3044
573: stonewalling pairs accessed: 2004
509: stonewalling pairs accessed: 2011
78: stonewalling pairs accessed: 3151
328: stonewalling pairs accessed: 1949
406: stonewalling pairs accessed: 3461
23: stonewalling pairs accessed: 1906
461: stonewalling pairs accessed: 1893
579: stonewalling pairs accessed: 3029
492: stonewalling pairs accessed: 3005
108: stonewalling pairs accessed: 2125
331: stonewalling pairs accessed: 3189
366: stonewalling pairs accessed: 3261
53: stonewalling pairs accessed: 2837
204: stonewalling pairs accessed: 2894
467: stonewalling pairs accessed: 1765
544: stonewalling pairs accessed: 2090
488: stonewalling pairs accessed: 3161
84: stonewalling pairs accessed: 3046
333: stonewalling pairs accessed: 3106
383: stonewalling pairs accessed: 3618
6: stonewalling pairs accessed: 3027
192: stonewalling pairs accessed: 1941
441: stonewalling pairs accessed: 1917
586: stonewalling pairs accessed: 1851
511: stonewalling pairs accessed: 2145
89: stonewalling pairs accessed: 1844
327: stonewalling pairs accessed: 1974
258: stonewalling pairs accessed: 2119
385: stonewalling pairs accessed: 2310
12: stonewalling pairs accessed: 2221
452: stonewalling pairs accessed: 2847
582: stonewalling pairs accessed: 2808
491: stonewalling pairs accessed: 3604
61: stonewalling pairs accessed: 3071
356: stonewalling pairs accessed: 2980
378: stonewalling pairs accessed: 3077
21: stonewalling pairs accessed: 3048
189: stonewalling pairs accessed: 1895
424: stonewalling pairs accessed: 2761
564: stonewalling pairs accessed: 2033
502: stonewalling pairs accessed: 2638
68: stonewalling pairs accessed: 3190
304: stonewalling pairs accessed: 2075
389: stonewalling pairs accessed: 2070
56: stonewalling pairs accessed: 3170
422: stonewalling pairs accessed: 3006
585: stonewalling pairs accessed: 2873
490: stonewalling pairs accessed: 3396
114: stonewalling pairs accessed: 1912
350: stonewalling pairs accessed: 3077
417: stonewalling pairs accessed: 3051
11: stonewalling pairs accessed: 1913
436: stonewalling pairs accessed: 2037
590: stonewalling pairs accessed: 2952
484: stonewalling pairs accessed: 3012
110: stonewalling pairs accessed: 2007
347: stonewalling pairs accessed: 3151
379: stonewalling pairs accessed: 2070
27: stonewalling pairs accessed: 2287
429: stonewalling pairs accessed: 2943
553: stonewalling pairs accessed: 1942
480: stonewalling pairs accessed: 2866
80: stonewalling pairs accessed: 3028
336: stonewalling pairs accessed: 2984
268: stonewalling pairs accessed: 2175
407: stonewalling pairs accessed: 2340
13: stonewalling pairs accessed: 1782
434: stonewalling pairs accessed: 2067
570: stonewalling pairs accessed: 3094
504: stonewalling pairs accessed: 2996
92: stonewalling pairs accessed: 2167
348: stonewalling pairs accessed: 1942
401: stonewalling pairs accessed: 2034
26: stonewalling pairs accessed: 1822
462: stonewalling pairs accessed: 2696
595: stonewalling pairs accessed: 2917
485: stonewalling pairs accessed: 2979
64: stonewalling pairs accessed: 1802
349: stonewalling pairs accessed: 2111
261: stonewalling pairs accessed: 2144
382: stonewalling pairs accessed: 3406
43: stonewalling pairs accessed: 3054
191: stonewalling pairs accessed: 2740
428: stonewalling pairs accessed: 3102
593: stonewalling pairs accessed: 1879
494: stonewalling pairs accessed: 1972
79: stonewalling pairs accessed: 3234
325: stonewalling pairs accessed: 2175
257: stonewalling pairs accessed: 2012
402: stonewalling pairs accessed: 1980
0: stonewalling pairs accessed: 1815
432: stonewalling pairs accessed: 2043
541: stonewalling pairs accessed: 2132
507: stonewalling pairs accessed: 3154
104: stonewalling pairs accessed: 1906
341: stonewalling pairs accessed: 1890
155: stonewalling pairs accessed: 3151
371: stonewalling pairs accessed: 3068
50: stonewalling pairs accessed: 3445
211: stonewalling pairs accessed: 1953
459: stonewalling pairs accessed: 2438
575: stonewalling pairs accessed: 1979
487: stonewalling pairs accessed: 2937
73: stonewalling pairs accessed: 1925
311: stonewalling pairs accessed: 1914
370: stonewalling pairs accessed: 2857
48: stonewalling pairs accessed: 3185
227: stonewalling pairs accessed: 2994
477: stonewalling pairs accessed: 1909
589: stonewalling pairs accessed: 2748
529: stonewalling pairs accessed: 1924
86: stonewalling pairs accessed: 1763
343: stonewalling pairs accessed: 2078
410: stonewalling pairs accessed: 2074
9: stonewalling pairs accessed: 2941
228: stonewalling pairs accessed: 1969
471: stonewalling pairs accessed: 2705
568: stonewalling pairs accessed: 2114
527: stonewalling pairs accessed: 2190
111: stonewalling pairs accessed: 1870
338: stonewalling pairs accessed: 2083
159: stonewalling pairs accessed: 2588
393: stonewalling pairs accessed: 2792
38: stonewalling pairs accessed: 3140
220: stonewalling pairs accessed: 1889
426: stonewalling pairs accessed: 2890
596: stonewalling pairs accessed: 2794
518: stonewalling pairs accessed: 2086
95: stonewalling pairs accessed: 2073
324: stonewalling pairs accessed: 1911
178: stonewalling pairs accessed: 1940
398: stonewalling pairs accessed: 1955
35: stonewalling pairs accessed: 2222
195: stonewalling pairs accessed: 2026
457: stonewalling pairs accessed: 1992
597: stonewalling pairs accessed: 1978
539: stonewalling pairs accessed: 1752
88: stonewalling pairs accessed: 1945
357: stonewalling pairs accessed: 1966
364: stonewalling pairs accessed: 3118
49: stonewalling pairs accessed: 2040
215: stonewalling pairs accessed: 3247
427: stonewalling pairs accessed: 2794
563: stonewalling pairs accessed: 1915
508: stonewalling pairs accessed: 1897
63: stonewalling pairs accessed: 1991
305: stonewalling pairs accessed: 1981
173: stonewalling pairs accessed: 2782
416: stonewalling pairs accessed: 1896
25: stonewalling pairs accessed: 2059
470: stonewalling pairs accessed: 2834
543: stonewalling pairs accessed: 1955
532: stonewalling pairs accessed: 1939
97: stonewalling pairs accessed: 1855
322: stonewalling pairs accessed: 2150
165: stonewalling pairs accessed: 2607
391: stonewalling pairs accessed: 2208
39: stonewalling pairs accessed: 2226
229: stonewalling pairs accessed: 1985
476: stonewalling pairs accessed: 2111
565: stonewalling pairs accessed: 1860
522: stonewalling pairs accessed: 1865
83: stonewalling pairs accessed: 1777
320: stonewalling pairs accessed: 2087
384: stonewalling pairs accessed: 2028
19: stonewalling pairs accessed: 1957
433: stonewalling pairs accessed: 2031
552: stonewalling pairs accessed: 1966
535: stonewalling pairs accessed: 2157
65: stonewalling pairs accessed: 1981
321: stonewalling pairs accessed: 1973
154: stonewalling pairs accessed: 3183
375: stonewalling pairs accessed: 2080
34: stonewalling pairs accessed: 2044
442: stonewalling pairs accessed: 2139
559: stonewalling pairs accessed: 1816
537: stonewalling pairs accessed: 2037
94: stonewalling pairs accessed: 2007
353: stonewalling pairs accessed: 1989
127: stonewalling pairs accessed: 2568
408: stonewalling pairs accessed: 2082
45: stonewalling pairs accessed: 1970
454: stonewalling pairs accessed: 1935
584: stonewalling pairs accessed: 2150
506: stonewalling pairs accessed: 2033
60: stonewalling pairs accessed: 1858
326: stonewalling pairs accessed: 2065
126: stonewalling pairs accessed: 2692
411: stonewalling pairs accessed: 2043
31: stonewalling pairs accessed: 1928
473: stonewalling pairs accessed: 2013
588: stonewalling pairs accessed: 2896
482: stonewalling pairs accessed: 2017
87: stonewalling pairs accessed: 1802
318: stonewalling pairs accessed: 2035
150: stonewalling pairs accessed: 2721
369: stonewalling pairs accessed: 1946
33: stonewalling pairs accessed: 2182
453: stonewalling pairs accessed: 2677
550: stonewalling pairs accessed: 2010
533: stonewalling pairs accessed: 1963
119: stonewalling pairs accessed: 2064
312: stonewalling pairs accessed: 2017
141: stonewalling pairs accessed: 1936
414: stonewalling pairs accessed: 2039
20: stonewalling pairs accessed: 1681
474: stonewalling pairs accessed: 1897
561: stonewalling pairs accessed: 1913
517: stonewalling pairs accessed: 2153
113: stonewalling pairs accessed: 2045
323: stonewalling pairs accessed: 2117
171: stonewalling pairs accessed: 1917
399: stonewalling pairs accessed: 2279
10: stonewalling pairs accessed: 2018
435: stonewalling pairs accessed: 2809
540: stonewalling pairs accessed: 1890
483: stonewalling pairs accessed: 2096
74: stonewalling pairs accessed: 1862
332: stonewalling pairs accessed: 1959
167: stonewalling pairs accessed: 1945
368: stonewalling pairs accessed: 2046
1: stonewalling pairs accessed: 1908
466: stonewalling pairs accessed: 1866
557: stonewalling pairs accessed: 1947
536: stonewalling pairs accessed: 1935
112: stonewalling pairs accessed: 2206
330: stonewalling pairs accessed: 1811
148: stonewalling pairs accessed: 1851
263: stonewalling pairs accessed: 3265
362: stonewalling pairs accessed: 1964
55: stonewalling pairs accessed: 2214
182: stonewalling pairs accessed: 3039
423: stonewalling pairs accessed: 2042
566: stonewalling pairs accessed: 1838
531: stonewalling pairs accessed: 1936
116: stonewalling pairs accessed: 1949
300: stonewalling pairs accessed: 2068
387: stonewalling pairs accessed: 2277
17: stonewalling pairs accessed: 1920
188: stonewalling pairs accessed: 2111
472: stonewalling pairs accessed: 2014
578: stonewalling pairs accessed: 2007
512: stonewalling pairs accessed: 1889
75: stonewalling pairs accessed: 1942
302: stonewalling pairs accessed: 2029
124: stonewalling pairs accessed: 2551
372: stonewalling pairs accessed: 1811
42: stonewalling pairs accessed: 1915
438: stonewalling pairs accessed: 1816
548: stonewalling pairs accessed: 1907
519: stonewalling pairs accessed: 2063
70: stonewalling pairs accessed: 1816
335: stonewalling pairs accessed: 1869
400: stonewalling pairs accessed: 1966
3: stonewalling pairs accessed: 1821
458: stonewalling pairs accessed: 1834
542: stonewalling pairs accessed: 1994
514: stonewalling pairs accessed: 2038
85: stonewalling pairs accessed: 1770
314: stonewalling pairs accessed: 1933
418: stonewalling pairs accessed: 2039
54: stonewalling pairs accessed: 1711
226: stonewalling pairs accessed: 3251
430: stonewalling pairs accessed: 1924
556: stonewalling pairs accessed: 1874
500: stonewalling pairs accessed: 1851
82: stonewalling pairs accessed: 1859
303: stonewalling pairs accessed: 1913
158: stonewalling pairs accessed: 2738
274: stonewalling pairs accessed: 1952
376: stonewalling pairs accessed: 2146
37: stonewalling pairs accessed: 2165
460: stonewalling pairs accessed: 1844
554: stonewalling pairs accessed: 1930
530: stonewalling pairs accessed: 1866
76: stonewalling pairs accessed: 2074
315: stonewalling pairs accessed: 2229
168: stonewalling pairs accessed: 2858
412: stonewalling pairs accessed: 2205
4: stonewalling pairs accessed: 1867
455: stonewalling pairs accessed: 1779
558: stonewalling pairs accessed: 1879
515: stonewalling pairs accessed: 2017
105: stonewalling pairs accessed: 2115
316: stonewalling pairs accessed: 1984
175: stonewalling pairs accessed: 2889
377: stonewalling pairs accessed: 1976
46: stonewalling pairs accessed: 2089
451: stonewalling pairs accessed: 1926
594: stonewalling pairs accessed: 1871
501: stonewalling pairs accessed: 2081
101: stonewalling pairs accessed: 2046
351: stonewalling pairs accessed: 1873
121: stonewalling pairs accessed: 1889
363: stonewalling pairs accessed: 2083
29: stonewalling pairs accessed: 1741
216: stonewalling pairs accessed: 1874
439: stonewalling pairs accessed: 1959
591: stonewalling pairs accessed: 1942
528: stonewalling pairs accessed: 1975
96: stonewalling pairs accessed: 1837
329: stonewalling pairs accessed: 2186
143: stonewalling pairs accessed: 1799
259: stonewalling pairs accessed: 2182
388: stonewalling pairs accessed: 2078
32: stonewalling pairs accessed: 2218
184: stonewalling pairs accessed: 1831
465: stonewalling pairs accessed: 1849
560: stonewalling pairs accessed: 1775
496: stonewalling pairs accessed: 1863
72: stonewalling pairs accessed: 2060
345: stonewalling pairs accessed: 2039
140: stonewalling pairs accessed: 1860
386: stonewalling pairs accessed: 2193
30: stonewalling pairs accessed: 1787
421: stonewalling pairs accessed: 1874
546: stonewalling pairs accessed: 2173
505: stonewalling pairs accessed: 1828
117: stonewalling pairs accessed: 2042
346: stonewalling pairs accessed: 1959
166: stonewalling pairs accessed: 1852
24: stonewalling pairs accessed: 2106
464: stonewalling pairs accessed: 1727
481: stonewalling pairs accessed: 2071
179: stonewalling pairs accessed: 1873
495: stonewalling pairs accessed: 1857
172: stonewalling pairs accessed: 2796
170: stonewalling pairs accessed: 2841
156: stonewalling pairs accessed: 2836
177: stonewalling pairs accessed: 2767
133: stonewalling pairs accessed: 1681
120: stonewalling pairs accessed: 1891
135: stonewalling pairs accessed: 1888
153: stonewalling pairs accessed: 1906
149: stonewalling pairs accessed: 1865
122: stonewalling pairs accessed: 2051
231: stonewalling pairs accessed: 2885
201: stonewalling pairs accessed: 1978
292: stonewalling pairs accessed: 2102
123: stonewalling pairs accessed: 1755
144: stonewalling pairs accessed: 1951
161: stonewalling pairs accessed: 1884
160: stonewalling pairs accessed: 1847
138: stonewalling pairs accessed: 1829
163: stonewalling pairs accessed: 1992
131: stonewalling pairs accessed: 1825
169: stonewalling pairs accessed: 1868
129: stonewalling pairs accessed: 2001
176: stonewalling pairs accessed: 1958
202: stonewalling pairs accessed: 3001
174: stonewalling pairs accessed: 1895
146: stonewalling pairs accessed: 1757
151: stonewalling pairs accessed: 1897
157: stonewalling pairs accessed: 1954
134: stonewalling pairs accessed: 1939
147: stonewalling pairs accessed: 1933
219: stonewalling pairs accessed: 1894
152: stonewalling pairs accessed: 1736
132: stonewalling pairs accessed: 1935
164: stonewalling pairs accessed: 1767
162: stonewalling pairs accessed: 1791
142: stonewalling pairs accessed: 1810
130: stonewalling pairs accessed: 2009
128: stonewalling pairs accessed: 1887
139: stonewalling pairs accessed: 1840
145: stonewalling pairs accessed: 1877
187: stonewalling pairs accessed: 2009
186: stonewalling pairs accessed: 2004
253: stonewalling pairs accessed: 1980
223: stonewalling pairs accessed: 1797
136: stonewalling pairs accessed: 2049
218: stonewalling pairs accessed: 2165
239: stonewalling pairs accessed: 1755
125: stonewalling pairs accessed: 1916
210: stonewalling pairs accessed: 1960
137: stonewalling pairs accessed: 1908
199: stonewalling pairs accessed: 1958
197: stonewalling pairs accessed: 1935
272: stonewalling pairs accessed: 2078
237: stonewalling pairs accessed: 1827
190: stonewalling pairs accessed: 2701
208: stonewalling pairs accessed: 1955
198: stonewalling pairs accessed: 2919
234: stonewalling pairs accessed: 2769
273: stonewalling pairs accessed: 2061
252: stonewalling pairs accessed: 2055
185: stonewalling pairs accessed: 2928
209: stonewalling pairs accessed: 1892
196: stonewalling pairs accessed: 1932
224: stonewalling pairs accessed: 2027
213: stonewalling pairs accessed: 1869
235: stonewalling pairs accessed: 2741
287: stonewalling pairs accessed: 2053
206: stonewalling pairs accessed: 2189
221: stonewalling pairs accessed: 1948
200: stonewalling pairs accessed: 1929
217: stonewalling pairs accessed: 3037
222: stonewalling pairs accessed: 1927
203: stonewalling pairs accessed: 2751
205: stonewalling pairs accessed: 2734
212: stonewalling pairs accessed: 2796
194: stonewalling pairs accessed: 2014
289: stonewalling pairs accessed: 1995
193: stonewalling pairs accessed: 1973
183: stonewalling pairs accessed: 1873
291: stonewalling pairs accessed: 2020
288: stonewalling pairs accessed: 3124
251: stonewalling pairs accessed: 3355
256: stonewalling pairs accessed: 2146
260: stonewalling pairs accessed: 2109
282: stonewalling pairs accessed: 2037
285: stonewalling pairs accessed: 2032
264: stonewalling pairs accessed: 2194
243: stonewalling pairs accessed: 3159
286: stonewalling pairs accessed: 2141
293: stonewalling pairs accessed: 2032
290: stonewalling pairs accessed: 3108
275: stonewalling pairs accessed: 2088
284: stonewalling pairs accessed: 1979
294: stonewalling pairs accessed: 3293
255: stonewalling pairs accessed: 1953
248: stonewalling pairs accessed: 1990
278: stonewalling pairs accessed: 1897
295: stonewalling pairs accessed: 2103
271: stonewalling pairs accessed: 2157
267: stonewalling pairs accessed: 2155
281: stonewalling pairs accessed: 2159
262: stonewalling pairs accessed: 3312
240: stonewalling pairs accessed: 2113
296: stonewalling pairs accessed: 3379
265: stonewalling pairs accessed: 1987
244: stonewalling pairs accessed: 3413
279: stonewalling pairs accessed: 2091
276: stonewalling pairs accessed: 2199
280: stonewalling pairs accessed: 2055
269: stonewalling pairs accessed: 1945
298: stonewalling pairs accessed: 1942
266: stonewalling pairs accessed: 2001
249: stonewalling pairs accessed: 3232
247: stonewalling pairs accessed: 3400
277: stonewalling pairs accessed: 2018
299: stonewalling pairs accessed: 3263
245: stonewalling pairs accessed: 3474
270: stonewalling pairs accessed: 3215
246: stonewalling pairs accessed: 2172
297: stonewalling pairs accessed: 2032
stonewalling pairs accessed min: 1681 max: 3858 -- min data: 6.6 GiB mean data: 8.9 GiB time: 300.6s
WARNING: Expected aggregate file size       = 6241124352000000.
WARNING: Stat() of aggregate file size      = 9708974899200.
WARNING: Using actual aggregate bytes moved = 9708974899200.
WARNING: maybe caused by deadlineForStonewalling
write     14548      3638       0.000255    10158080000 4096       0.079455   636.35     0.019625   636.45     0   
Max Write: 14548.18 MiB/sec (15254.87 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       14548.18   14548.18   14548.18       0.00    3637.05    3637.05    3637.05       0.00  636.45070     300.63      18248.68     0    600  60    1   1     1        1         0    0      1 10401873920000  4194304 9259200.0 POSIX      0
Finished            : Tue Jun  2 09:07:21 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 09:47:19 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -r -R -s 1900000 -a POSIX --posix.odirect -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datadir/2020.06.02-08.56.31-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datadir/2020.06.02-08.56.31-scr/ior_hard/stonewall
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 09:47:19 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr/ior_hard
FS                  : 50.0 TiB   Used FS: 44.0%   Inodes: 858.3 Mi   Used Inodes: 5.0%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-08.56.31-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 48.74 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Tue Jun  2 09:47:19 2020

WARNING: Expected aggregate file size       = 53589120000000.
WARNING: Stat() of aggregate file size      = 4698976089600.
WARNING: Using actual aggregate bytes moved = 4698976089600.
read      9946       221865     450.55      45.91      45.91      0.002965   450.55     0.000405   450.55     0   
Max Read:  9946.22 MiB/sec (10429.37 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         9946.22    9946.22    9946.22       0.00  221863.68  221863.68  221863.68       0.00  450.55234         NA            NA     0    600  60    1   0     1        1         0    0 1900000    47008    47008 4481293.0 POSIX      0
Finished            : Tue Jun  2 09:54:49 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 09:15:49 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -w -s 1900000 -a POSIX --posix.odirect -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datadir/2020.06.02-08.56.31-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datadir/2020.06.02-08.56.31-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 09:15:49 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr/ior_hard
FS                  : 50.0 TiB   Used FS: 35.3%   Inodes: 858.3 Mi   Used Inodes: 4.0%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-08.56.31-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 48.74 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Tue Jun  2 09:15:49 2020
359: stonewalling pairs accessed: 166601
419: stonewalling pairs accessed: 166601
539: stonewalling pairs accessed: 166601
479: stonewalling pairs accessed: 166601
599: stonewalling pairs accessed: 166601
59: stonewalling pairs accessed: 166602
179: stonewalling pairs accessed: 166602
119: stonewalling pairs accessed: 166602
239: stonewalling pairs accessed: 166602
418: stonewalling pairs accessed: 36
598: stonewalling pairs accessed: 38
178: stonewalling pairs accessed: 41
358: stonewalling pairs accessed: 40
299: stonewalling pairs accessed: 166602
118: stonewalling pairs accessed: 40
538: stonewalling pairs accessed: 47
478: stonewalling pairs accessed: 47
298: stonewalling pairs accessed: 38
160: stonewalling pairs accessed: 46
332: stonewalling pairs accessed: 55
394: stonewalling pairs accessed: 36
459: stonewalling pairs accessed: 45
591: stonewalling pairs accessed: 66
177: stonewalling pairs accessed: 35
99: stonewalling pairs accessed: 52
190: stonewalling pairs accessed: 34
335: stonewalling pairs accessed: 32
407: stonewalling pairs accessed: 30
494: stonewalling pairs accessed: 34
461: stonewalling pairs accessed: 49
282: stonewalling pairs accessed: 37
24: stonewalling pairs accessed: 50
584: stonewalling pairs accessed: 58
170: stonewalling pairs accessed: 46
98: stonewalling pairs accessed: 50
224: stonewalling pairs accessed: 41
347: stonewalling pairs accessed: 34
398: stonewalling pairs accessed: 28
500: stonewalling pairs accessed: 36
469: stonewalling pairs accessed: 35
259: stonewalling pairs accessed: 36
26: stonewalling pairs accessed: 38
590: stonewalling pairs accessed: 65
138: stonewalling pairs accessed: 50
92: stonewalling pairs accessed: 40
184: stonewalling pairs accessed: 33
349: stonewalling pairs accessed: 31
379: stonewalling pairs accessed: 38
496: stonewalling pairs accessed: 39
476: stonewalling pairs accessed: 60
276: stonewalling pairs accessed: 32
11: stonewalling pairs accessed: 40
595: stonewalling pairs accessed: 64
157: stonewalling pairs accessed: 54
112: stonewalling pairs accessed: 36
201: stonewalling pairs accessed: 40
318: stonewalling pairs accessed: 61
372: stonewalling pairs accessed: 42
510: stonewalling pairs accessed: 40
425: stonewalling pairs accessed: 61
257: stonewalling pairs accessed: 34
37: stonewalling pairs accessed: 57
548: stonewalling pairs accessed: 55
129: stonewalling pairs accessed: 61
114: stonewalling pairs accessed: 32
208: stonewalling pairs accessed: 44
325: stonewalling pairs accessed: 56
392: stonewalling pairs accessed: 39
481: stonewalling pairs accessed: 32
427: stonewalling pairs accessed: 35
284: stonewalling pairs accessed: 36
20: stonewalling pairs accessed: 34
571: stonewalling pairs accessed: 51
137: stonewalling pairs accessed: 58
64: stonewalling pairs accessed: 51
215: stonewalling pairs accessed: 43
326: stonewalling pairs accessed: 43
411: stonewalling pairs accessed: 30
488: stonewalling pairs accessed: 28
434: stonewalling pairs accessed: 57
242: stonewalling pairs accessed: 63
15: stonewalling pairs accessed: 41
580: stonewalling pairs accessed: 50
143: stonewalling pairs accessed: 58
74: stonewalling pairs accessed: 47
236: stonewalling pairs accessed: 72
357: stonewalling pairs accessed: 53
413: stonewalling pairs accessed: 46
495: stonewalling pairs accessed: 49
472: stonewalling pairs accessed: 33
243: stonewalling pairs accessed: 62
31: stonewalling pairs accessed: 42
557: stonewalling pairs accessed: 61
168: stonewalling pairs accessed: 59
85: stonewalling pairs accessed: 32
188: stonewalling pairs accessed: 51
308: stonewalling pairs accessed: 64
404: stonewalling pairs accessed: 37
503: stonewalling pairs accessed: 35
477: stonewalling pairs accessed: 39
254: stonewalling pairs accessed: 42
14: stonewalling pairs accessed: 36
545: stonewalling pairs accessed: 41
164: stonewalling pairs accessed: 42
68: stonewalling pairs accessed: 42
209: stonewalling pairs accessed: 38
314: stonewalling pairs accessed: 62
361: stonewalling pairs accessed: 31
505: stonewalling pairs accessed: 36
435: stonewalling pairs accessed: 53
255: stonewalling pairs accessed: 35
40: stonewalling pairs accessed: 32
553: stonewalling pairs accessed: 62
142: stonewalling pairs accessed: 39
69: stonewalling pairs accessed: 30
230: stonewalling pairs accessed: 40
315: stonewalling pairs accessed: 35
366: stonewalling pairs accessed: 33
507: stonewalling pairs accessed: 34
464: stonewalling pairs accessed: 31
278: stonewalling pairs accessed: 49
2: stonewalling pairs accessed: 35
569: stonewalling pairs accessed: 42
152: stonewalling pairs accessed: 60
93: stonewalling pairs accessed: 28
218: stonewalling pairs accessed: 35
342: stonewalling pairs accessed: 40
387: stonewalling pairs accessed: 36
518: stonewalling pairs accessed: 39
475: stonewalling pairs accessed: 36
247: stonewalling pairs accessed: 56
9: stonewalling pairs accessed: 55
570: stonewalling pairs accessed: 61
128: stonewalling pairs accessed: 56
75: stonewalling pairs accessed: 43
232: stonewalling pairs accessed: 44
346: stonewalling pairs accessed: 37
397: stonewalling pairs accessed: 34
519: stonewalling pairs accessed: 28
437: stonewalling pairs accessed: 36
258: stonewalling pairs accessed: 32
25: stonewalling pairs accessed: 54
549: stonewalling pairs accessed: 36
136: stonewalling pairs accessed: 34
95: stonewalling pairs accessed: 50
182: stonewalling pairs accessed: 36
322: stonewalling pairs accessed: 61
360: stonewalling pairs accessed: 47
523: stonewalling pairs accessed: 37
424: stonewalling pairs accessed: 47
279: stonewalling pairs accessed: 52
36: stonewalling pairs accessed: 50
559: stonewalling pairs accessed: 46
153: stonewalling pairs accessed: 42
107: stonewalling pairs accessed: 41
198: stonewalling pairs accessed: 31
356: stonewalling pairs accessed: 48
363: stonewalling pairs accessed: 45
526: stonewalling pairs accessed: 34
442: stonewalling pairs accessed: 31
291: stonewalling pairs accessed: 45
51: stonewalling pairs accessed: 31
586: stonewalling pairs accessed: 47
156: stonewalling pairs accessed: 59
103: stonewalling pairs accessed: 34
229: stonewalling pairs accessed: 33
330: stonewalling pairs accessed: 56
368: stonewalling pairs accessed: 37
529: stonewalling pairs accessed: 30
474: stonewalling pairs accessed: 59
244: stonewalling pairs accessed: 60
57: stonewalling pairs accessed: 59
594: stonewalling pairs accessed: 50
161: stonewalling pairs accessed: 59
77: stonewalling pairs accessed: 42
220: stonewalling pairs accessed: 35
351: stonewalling pairs accessed: 42
369: stonewalling pairs accessed: 35
536: stonewalling pairs accessed: 31
432: stonewalling pairs accessed: 47
245: stonewalling pairs accessed: 35
8: stonewalling pairs accessed: 32
561: stonewalling pairs accessed: 57
176: stonewalling pairs accessed: 59
106: stonewalling pairs accessed: 53
237: stonewalling pairs accessed: 37
324: stonewalling pairs accessed: 60
378: stonewalling pairs accessed: 39
480: stonewalling pairs accessed: 33
441: stonewalling pairs accessed: 36
277: stonewalling pairs accessed: 45
33: stonewalling pairs accessed: 54
540: stonewalling pairs accessed: 47
120: stonewalling pairs accessed: 32
82: stonewalling pairs accessed: 49
191: stonewalling pairs accessed: 45
333: stonewalling pairs accessed: 52
380: stonewalling pairs accessed: 35
515: stonewalling pairs accessed: 33
426: stonewalling pairs accessed: 41
249: stonewalling pairs accessed: 39
30: stonewalling pairs accessed: 57
572: stonewalling pairs accessed: 59
124: stonewalling pairs accessed: 55
90: stonewalling pairs accessed: 36
225: stonewalling pairs accessed: 32
348: stonewalling pairs accessed: 57
382: stonewalling pairs accessed: 30
483: stonewalling pairs accessed: 31
448: stonewalling pairs accessed: 40
240: stonewalling pairs accessed: 39
16: stonewalling pairs accessed: 57
542: stonewalling pairs accessed: 68
125: stonewalling pairs accessed: 52
108: stonewalling pairs accessed: 28
226: stonewalling pairs accessed: 31
327: stonewalling pairs accessed: 57
383: stonewalling pairs accessed: 29
532: stonewalling pairs accessed: 50
471: stonewalling pairs accessed: 52
285: stonewalling pairs accessed: 32
21: stonewalling pairs accessed: 54
585: stonewalling pairs accessed: 35
126: stonewalling pairs accessed: 42
63: stonewalling pairs accessed: 34
207: stonewalling pairs accessed: 31
317: stonewalling pairs accessed: 63
385: stonewalling pairs accessed: 31
522: stonewalling pairs accessed: 31
451: stonewalling pairs accessed: 39
287: stonewalling pairs accessed: 31
27: stonewalling pairs accessed: 34
593: stonewalling pairs accessed: 56
130: stonewalling pairs accessed: 60
78: stonewalling pairs accessed: 34
192: stonewalling pairs accessed: 32
300: stonewalling pairs accessed: 39
396: stonewalling pairs accessed: 29
501: stonewalling pairs accessed: 28
430: stonewalling pairs accessed: 61
297: stonewalling pairs accessed: 56
38: stonewalling pairs accessed: 57
563: stonewalling pairs accessed: 35
131: stonewalling pairs accessed: 35
104: stonewalling pairs accessed: 54
200: stonewalling pairs accessed: 45
302: stonewalling pairs accessed: 74
408: stonewalling pairs accessed: 41
509: stonewalling pairs accessed: 31
429: stonewalling pairs accessed: 36
264: stonewalling pairs accessed: 46
56: stonewalling pairs accessed: 60
551: stonewalling pairs accessed: 43
132: stonewalling pairs accessed: 61
65: stonewalling pairs accessed: 28
206: stonewalling pairs accessed: 45
305: stonewalling pairs accessed: 72
409: stonewalling pairs accessed: 28
530: stonewalling pairs accessed: 33
450: stonewalling pairs accessed: 31
272: stonewalling pairs accessed: 46
0: stonewalling pairs accessed: 36
581: stonewalling pairs accessed: 34
133: stonewalling pairs accessed: 60
87: stonewalling pairs accessed: 32
217: stonewalling pairs accessed: 44
350: stonewalling pairs accessed: 40
416: stonewalling pairs accessed: 31
517: stonewalling pairs accessed: 28
446: stonewalling pairs accessed: 31
274: stonewalling pairs accessed: 31
17: stonewalling pairs accessed: 54
547: stonewalling pairs accessed: 50
134: stonewalling pairs accessed: 60
111: stonewalling pairs accessed: 32
180: stonewalling pairs accessed: 28
303: stonewalling pairs accessed: 34
417: stonewalling pairs accessed: 33
537: stonewalling pairs accessed: 39
447: stonewalling pairs accessed: 33
251: stonewalling pairs accessed: 48
1: stonewalling pairs accessed: 34
558: stonewalling pairs accessed: 34
135: stonewalling pairs accessed: 42
101: stonewalling pairs accessed: 55
189: stonewalling pairs accessed: 53
323: stonewalling pairs accessed: 55
384: stonewalling pairs accessed: 31
485: stonewalling pairs accessed: 33
470: stonewalling pairs accessed: 39
271: stonewalling pairs accessed: 39
7: stonewalling pairs accessed: 36
550: stonewalling pairs accessed: 65
139: stonewalling pairs accessed: 34
88: stonewalling pairs accessed: 51
227: stonewalling pairs accessed: 31
329: stonewalling pairs accessed: 57
403: stonewalling pairs accessed: 36
497: stonewalling pairs accessed: 39
458: stonewalling pairs accessed: 33
275: stonewalling pairs accessed: 31
13: stonewalling pairs accessed: 54
566: stonewalling pairs accessed: 39
140: stonewalling pairs accessed: 39
86: stonewalling pairs accessed: 28
210: stonewalling pairs accessed: 31
334: stonewalling pairs accessed: 31
364: stonewalling pairs accessed: 31
521: stonewalling pairs accessed: 33
462: stonewalling pairs accessed: 50
263: stonewalling pairs accessed: 31
58: stonewalling pairs accessed: 28
582: stonewalling pairs accessed: 63
146: stonewalling pairs accessed: 60
61: stonewalling pairs accessed: 28
211: stonewalling pairs accessed: 28
353: stonewalling pairs accessed: 38
365: stonewalling pairs accessed: 45
535: stonewalling pairs accessed: 31
421: stonewalling pairs accessed: 39
265: stonewalling pairs accessed: 32
3: stonewalling pairs accessed: 32
560: stonewalling pairs accessed: 65
147: stonewalling pairs accessed: 32
109: stonewalling pairs accessed: 28
203: stonewalling pairs accessed: 32
336: stonewalling pairs accessed: 56
388: stonewalling pairs accessed: 41
487: stonewalling pairs accessed: 31
423: stonewalling pairs accessed: 70
270: stonewalling pairs accessed: 31
55: stonewalling pairs accessed: 32
577: stonewalling pairs accessed: 63
162: stonewalling pairs accessed: 59
91: stonewalling pairs accessed: 32
221: stonewalling pairs accessed: 32
316: stonewalling pairs accessed: 39
402: stonewalling pairs accessed: 43
504: stonewalling pairs accessed: 31
443: stonewalling pairs accessed: 31
266: stonewalling pairs accessed: 31
32: stonewalling pairs accessed: 32
552: stonewalling pairs accessed: 59
163: stonewalling pairs accessed: 34
94: stonewalling pairs accessed: 32
231: stonewalling pairs accessed: 32
309: stonewalling pairs accessed: 68
414: stonewalling pairs accessed: 39
531: stonewalling pairs accessed: 39
467: stonewalling pairs accessed: 33
288: stonewalling pairs accessed: 31
43: stonewalling pairs accessed: 37
576: stonewalling pairs accessed: 63
167: stonewalling pairs accessed: 60
67: stonewalling pairs accessed: 32
195: stonewalling pairs accessed: 28
345: stonewalling pairs accessed: 32
370: stonewalling pairs accessed: 31
508: stonewalling pairs accessed: 28
436: stonewalling pairs accessed: 41
283: stonewalling pairs accessed: 32
50: stonewalling pairs accessed: 32
574: stonewalling pairs accessed: 46
169: stonewalling pairs accessed: 32
113: stonewalling pairs accessed: 32
181: stonewalling pairs accessed: 28
307: stonewalling pairs accessed: 59
362: stonewalling pairs accessed: 41
511: stonewalling pairs accessed: 33
456: stonewalling pairs accessed: 31
295: stonewalling pairs accessed: 41
54: stonewalling pairs accessed: 60
583: stonewalling pairs accessed: 63
175: stonewalling pairs accessed: 53
79: stonewalling pairs accessed: 28
216: stonewalling pairs accessed: 32
339: stonewalling pairs accessed: 45
373: stonewalling pairs accessed: 31
527: stonewalling pairs accessed: 39
463: stonewalling pairs accessed: 39
273: stonewalling pairs accessed: 31
46: stonewalling pairs accessed: 32
579: stonewalling pairs accessed: 33
150: stonewalling pairs accessed: 61
102: stonewalling pairs accessed: 28
185: stonewalling pairs accessed: 32
311: stonewalling pairs accessed: 68
389: stonewalling pairs accessed: 31
484: stonewalling pairs accessed: 33
420: stonewalling pairs accessed: 50
269: stonewalling pairs accessed: 32
10: stonewalling pairs accessed: 32
589: stonewalling pairs accessed: 45
172: stonewalling pairs accessed: 37
110: stonewalling pairs accessed: 32
205: stonewalling pairs accessed: 32
355: stonewalling pairs accessed: 60
415: stonewalling pairs accessed: 28
493: stonewalling pairs accessed: 28
438: stonewalling pairs accessed: 33
267: stonewalling pairs accessed: 32
44: stonewalling pairs accessed: 60
564: stonewalling pairs accessed: 59
121: stonewalling pairs accessed: 32
70: stonewalling pairs accessed: 28
233: stonewalling pairs accessed: 32
338: stonewalling pairs accessed: 31
393: stonewalling pairs accessed: 39
489: stonewalling pairs accessed: 31
422: stonewalling pairs accessed: 41
241: stonewalling pairs accessed: 32
42: stonewalling pairs accessed: 60
588: stonewalling pairs accessed: 43
144: stonewalling pairs accessed: 32
117: stonewalling pairs accessed: 32
199: stonewalling pairs accessed: 28
344: stonewalling pairs accessed: 58
377: stonewalling pairs accessed: 31
516: stonewalling pairs accessed: 31
444: stonewalling pairs accessed: 41
280: stonewalling pairs accessed: 32
49: stonewalling pairs accessed: 28
541: stonewalling pairs accessed: 33
123: stonewalling pairs accessed: 32
96: stonewalling pairs accessed: 28
183: stonewalling pairs accessed: 32
306: stonewalling pairs accessed: 32
367: stonewalling pairs accessed: 45
514: stonewalling pairs accessed: 28
473: stonewalling pairs accessed: 33
248: stonewalling pairs accessed: 32
6: stonewalling pairs accessed: 32
597: stonewalling pairs accessed: 59
171: stonewalling pairs accessed: 32
115: stonewalling pairs accessed: 32
212: stonewalling pairs accessed: 32
331: stonewalling pairs accessed: 31
371: stonewalling pairs accessed: 31
533: stonewalling pairs accessed: 33
433: stonewalling pairs accessed: 31
292: stonewalling pairs accessed: 32
35: stonewalling pairs accessed: 32
546: stonewalling pairs accessed: 39
122: stonewalling pairs accessed: 32
116: stonewalling pairs accessed: 32
214: stonewalling pairs accessed: 28
340: stonewalling pairs accessed: 58
386: stonewalling pairs accessed: 41
498: stonewalling pairs accessed: 31
468: stonewalling pairs accessed: 31
293: stonewalling pairs accessed: 32
18: stonewalling pairs accessed: 32
578: stonewalling pairs accessed: 50
141: stonewalling pairs accessed: 32
97: stonewalling pairs accessed: 28
219: stonewalling pairs accessed: 32
354: stonewalling pairs accessed: 45
376: stonewalling pairs accessed: 41
534: stonewalling pairs accessed: 31
454: stonewalling pairs accessed: 31
256: stonewalling pairs accessed: 32
34: stonewalling pairs accessed: 32
568: stonewalling pairs accessed: 59
127: stonewalling pairs accessed: 32
76: stonewalling pairs accessed: 28
187: stonewalling pairs accessed: 32
328: stonewalling pairs accessed: 35
374: stonewalling pairs accessed: 31
482: stonewalling pairs accessed: 31
460: stonewalling pairs accessed: 33
246: stonewalling pairs accessed: 32
52: stonewalling pairs accessed: 28
587: stonewalling pairs accessed: 68
165: stonewalling pairs accessed: 32
83: stonewalling pairs accessed: 28
238: stonewalling pairs accessed: 32
343: stonewalling pairs accessed: 39
391: stonewalling pairs accessed: 31
492: stonewalling pairs accessed: 50
455: stonewalling pairs accessed: 31
252: stonewalling pairs accessed: 32
19: stonewalling pairs accessed: 32
573: stonewalling pairs accessed: 33
174: stonewalling pairs accessed: 32
66: stonewalling pairs accessed: 28
197: stonewalling pairs accessed: 28
319: stonewalling pairs accessed: 41
395: stonewalling pairs accessed: 28
486: stonewalling pairs accessed: 28
457: stonewalling pairs accessed: 33
286: stonewalling pairs accessed: 32
48: stonewalling pairs accessed: 32
596: stonewalling pairs accessed: 68
149: stonewalling pairs accessed: 32
84: stonewalling pairs accessed: 28
193: stonewalling pairs accessed: 32
320: stonewalling pairs accessed: 39
375: stonewalling pairs accessed: 31
524: stonewalling pairs accessed: 28
440: stonewalling pairs accessed: 31
289: stonewalling pairs accessed: 32
28: stonewalling pairs accessed: 32
556: stonewalling pairs accessed: 59
145: stonewalling pairs accessed: 32
62: stonewalling pairs accessed: 28
222: stonewalling pairs accessed: 32
337: stonewalling pairs accessed: 41
406: stonewalling pairs accessed: 28
490: stonewalling pairs accessed: 31
466: stonewalling pairs accessed: 31
294: stonewalling pairs accessed: 32
4: stonewalling pairs accessed: 32
562: stonewalling pairs accessed: 33
166: stonewalling pairs accessed: 32
60: stonewalling pairs accessed: 28
213: stonewalling pairs accessed: 28
301: stonewalling pairs accessed: 39
401: stonewalling pairs accessed: 31
528: stonewalling pairs accessed: 31
439: stonewalling pairs accessed: 31
268: stonewalling pairs accessed: 32
12: stonewalling pairs accessed: 32
592: stonewalling pairs accessed: 68
173: stonewalling pairs accessed: 32
71: stonewalling pairs accessed: 28
202: stonewalling pairs accessed: 32
321: stonewalling pairs accessed: 39
410: stonewalling pairs accessed: 28
520: stonewalling pairs accessed: 28
428: stonewalling pairs accessed: 31
253: stonewalling pairs accessed: 32
29: stonewalling pairs accessed: 32
554: stonewalling pairs accessed: 33
159: stonewalling pairs accessed: 32
105: stonewalling pairs accessed: 28
223: stonewalling pairs accessed: 32
310: stonewalling pairs accessed: 32
390: stonewalling pairs accessed: 31
491: stonewalling pairs accessed: 31
465: stonewalling pairs accessed: 31
281: stonewalling pairs accessed: 32
5: stonewalling pairs accessed: 32
565: stonewalling pairs accessed: 33
151: stonewalling pairs accessed: 32
72: stonewalling pairs accessed: 28
235: stonewalling pairs accessed: 32
352: stonewalling pairs accessed: 41
381: stonewalling pairs accessed: 31
502: stonewalling pairs accessed: 28
453: stonewalling pairs accessed: 31
296: stonewalling pairs accessed: 32
41: stonewalling pairs accessed: 32
555: stonewalling pairs accessed: 33
158: stonewalling pairs accessed: 32
100: stonewalling pairs accessed: 28
186: stonewalling pairs accessed: 32
341: stonewalling pairs accessed: 41
399: stonewalling pairs accessed: 28
499: stonewalling pairs accessed: 28
431: stonewalling pairs accessed: 31
250: stonewalling pairs accessed: 32
47: stonewalling pairs accessed: 32
575: stonewalling pairs accessed: 33
148: stonewalling pairs accessed: 32
80: stonewalling pairs accessed: 28
234: stonewalling pairs accessed: 32
304: stonewalling pairs accessed: 32
412: stonewalling pairs accessed: 28
512: stonewalling pairs accessed: 28
449: stonewalling pairs accessed: 31
290: stonewalling pairs accessed: 32
22: stonewalling pairs accessed: 32
543: stonewalling pairs accessed: 33
154: stonewalling pairs accessed: 32
89: stonewalling pairs accessed: 28
204: stonewalling pairs accessed: 32
312: stonewalling pairs accessed: 32
405: stonewalling pairs accessed: 28
525: stonewalling pairs accessed: 28
445: stonewalling pairs accessed: 31
260: stonewalling pairs accessed: 32
39: stonewalling pairs accessed: 32
567: stonewalling pairs accessed: 33
155: stonewalling pairs accessed: 32
73: stonewalling pairs accessed: 28
228: stonewalling pairs accessed: 32
313: stonewalling pairs accessed: 32
400: stonewalling pairs accessed: 28
506: stonewalling pairs accessed: 28
452: stonewalling pairs accessed: 31
262: stonewalling pairs accessed: 32
53: stonewalling pairs accessed: 28
544: stonewalling pairs accessed: 33
81: stonewalling pairs accessed: 28
196: stonewalling pairs accessed: 32
513: stonewalling pairs accessed: 28
261: stonewalling pairs accessed: 32
23: stonewalling pairs accessed: 32
194: stonewalling pairs accessed: 32
45: stonewalling pairs accessed: 28
stonewalling pairs accessed min: 28 max: 166602 -- min data: 0.0 GiB mean data: 0.1 GiB time: 300.0s
WARNING: Expected aggregate file size       = 53589120000000.
WARNING: Stat() of aggregate file size      = 4698976089600.
WARNING: Using actual aggregate bytes moved = 4698976089600.
WARNING: maybe caused by deadlineForStonewalling
write     9337       208268     475.30      45.91      45.91      0.002308   479.96     0.000553   479.97     0   
Max Write: 9336.68 MiB/sec (9790.21 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        9336.68    9336.68    9336.68       0.00  208266.98  208266.98  208266.98       0.00  479.96662     300.01        252.41     0    600  60    1   0     1        1         0    0 1900000    47008    47008 4481293.0 POSIX      0
Finished            : Tue Jun  2 09:23:49 2020
mdtest_easy_delete
-- started at 06/02/2020 10:00:45 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-r' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_easy' '-n' '29000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 44.0%   Inodes: 858.3 Mi   Used Inodes: 5.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 17400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      33220.564      33220.534      33220.543          0.008
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.061          2.061          2.061          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        523.772        523.772        523.772          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.485          0.485          0.485          0.000
-- finished at 06/02/2020 10:13:05 --

mdtest_easy_stat
-- started at 06/02/2020 09:42:28 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-T' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_easy' '-n' '29000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 44.0%   Inodes: 858.3 Mi   Used Inodes: 5.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 17400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     201857.104     201856.898     201857.019          0.045
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         86.200         86.200         86.200          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 09:47:17 --

mdtest_easy_write
-- started at 06/02/2020 09:07:23 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-C' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_easy' '-n' '29000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 35.2%   Inodes: 858.3 Mi   Used Inodes: 2.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 17400000 files
Continue stonewall hit min: 424 max: 29000 avg: 19386.4 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      37494.209      37494.189      37494.202          0.005
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      38650.713             NA
   Tree creation             :         18.330         18.330         18.330          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        464.072        464.072        464.072          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.947             NA
   Tree creation             :          0.055          0.055          0.055          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 09:15:47 --

mdtest_hard_delete
-- started at 06/02/2020 10:20:20 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-r' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 43.8%   Inodes: 858.3 Mi   Used Inodes: 3.1%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      14791.778      14791.761      14791.767          0.004
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.322          0.322          0.322          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        315.703        315.702        315.703          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.104          3.104          3.104          0.000
-- finished at 06/02/2020 10:29:58 --

mdtest_hard_read
-- started at 06/02/2020 10:14:05 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 43.8%   Inodes: 858.3 Mi   Used Inodes: 3.1%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      38342.094      38342.058      38342.075          0.009
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        121.793        121.793        121.793          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 10:20:18 --

mdtest_hard_stat
-- started at 06/02/2020 09:55:50 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-T' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 44.0%   Inodes: 858.3 Mi   Used Inodes: 5.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      60207.388      60207.257      60207.287          0.035
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         77.562         77.562         77.562          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 10:00:43 --

mdtest_hard_write
-- started at 06/02/2020 09:23:51 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-C' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 43.9%   Inodes: 858.3 Mi   Used Inodes: 4.5%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files
Continue stonewall hit min: 7605 max: 7783 avg: 7710.0 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      14285.209      14285.198      14285.204          0.002
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      15417.669             NA
   Tree creation             :        974.513        974.513        974.513          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        326.898        326.898        326.898          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.047             NA
   Tree creation             :          0.001          0.001          0.001          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 09:30:32 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               14.207 GiB/s : time 636.35 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               37.494 kiops : time 464.07 seconds
[RESULT] BW   phase 2            ior_hard_write                9.118 GiB/s : time 479.96 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               14.285 kiops : time 326.90 seconds
[RESULT] IOPS phase 3                      find              288.630 kiops : time  76.47 seconds
[RESULT] BW   phase 3             ior_easy_read               15.661 GiB/s : time 577.35 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              201.857 kiops : time  86.20 seconds
[RESULT] BW   phase 4             ior_hard_read                9.713 GiB/s : time 450.55 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               60.207 kiops : time  77.56 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               33.221 kiops : time 523.77 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               38.342 kiops : time 121.79 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               14.792 kiops : time 579.92 seconds
[SCORE] Bandwidth 11.848 GiB/s : IOPS 49.3883 kiops : TOTAL 24.1899