SharedScratch

Institution Janelia Research Campus, HHMI
Client Procs Per Node
Client Operating System Scientific-Linux
Client Operating System Version 7.7
Client Kernel Version 3.10.0-1062.9.1.el7.x86_64

DATA SERVER

Storage Type Flash
Volatile Memory 192GB
Storage Interface NVMe
Network 100GbE
Software Version 3.7.0
OS Version 7.7

INFORMATION

Client Nodes 10
Client Total Procs 600

METADATA

Easy Write 37.49 kIOP/s
Easy Stat 201.86 kIOP/s
Easy Delete 33.22 kIOP/s
Hard Write 14.29 kIOP/s
Hard Read 38.34 kIOP/s
Hard Stat 60.21 kIOP/s
Hard Delete 14.79 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 600 -ppn 60 -f $PWD/hostfile-10-h09 --bootstrap ssh"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 09:31:50 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -r -R -a POSIX -t 4m --posix.odirect -v -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datadir/2020.06.02-08.56.31-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datadir/2020.06.02-08.56.31-scr/ior_easy/stonewall
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 09:31:50 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr/ior_easy
FS                  : 50.0 TiB   Used FS: 44.0%   Inodes: 858.3 Mi   Used Inodes: 5.0%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-08.56.31-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 4 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 5676.27 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Tue Jun  2 09:31:50 2020

WARNING: Expected aggregate file size       = 6241124352000000.
WARNING: Stat() of aggregate file size      = 9708974899200.
WARNING: Using actual aggregate bytes moved = 9708974899200.
read      16037      4009       0.000233    10158080000 4096       0.008144   577.35     0.002564   577.36     0   
Max Read:  16037.08 MiB/sec (16816.10 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        16037.08   16037.08   16037.08       0.00    4009.27    4009.27    4009.27       0.00  577.36201         NA            NA     0    600  60    1   1     1        1         0    0      1 10401873920000  4194304 9259200.0 POSIX      0
Finished            : Tue Jun  2 09:41:28 2020
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 08:56:45 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -w -a POSIX -t 4m --posix.odirect -v -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datadir/2020.06.02-08.56.31-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datadir/2020.06.02-08.56.31-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 08:56:45 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr/ior_easy
FS                  : 50.0 TiB   Used FS: 17.5%   Inodes: 858.3 Mi   Used Inodes: 1.0%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-08.56.31-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 4 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 5676.27 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Tue Jun  2 08:56:45 2020
478: stonewalling pairs accessed: 2877
115: stonewalling pairs accessed: 2096
181: stonewalling pairs accessed: 1836
551: stonewalling pairs accessed: 1787
409: stonewalling pairs accessed: 1997
334: stonewalling pairs accessed: 2040
415: stonewalling pairs accessed: 2113
28: stonewalling pairs accessed: 1903
562: stonewalling pairs accessed: 1873
66: stonewalling pairs accessed: 2043
358: stonewalling pairs accessed: 3019
419: stonewalling pairs accessed: 1979
58: stonewalling pairs accessed: 1964
233: stonewalling pairs accessed: 2063
479: stonewalling pairs accessed: 2611
581: stonewalling pairs accessed: 3290
526: stonewalling pairs accessed: 1993
100: stonewalling pairs accessed: 2932
313: stonewalling pairs accessed: 1872
241: stonewalling pairs accessed: 2112
360: stonewalling pairs accessed: 3196
14: stonewalling pairs accessed: 2069
447: stonewalling pairs accessed: 2516
587: stonewalling pairs accessed: 1907
498: stonewalling pairs accessed: 2866
103: stonewalling pairs accessed: 2085
359: stonewalling pairs accessed: 3023
397: stonewalling pairs accessed: 2135
59: stonewalling pairs accessed: 1924
207: stonewalling pairs accessed: 1876
443: stonewalling pairs accessed: 1869
599: stonewalling pairs accessed: 2711
534: stonewalling pairs accessed: 3107
106: stonewalling pairs accessed: 2689
337: stonewalling pairs accessed: 3224
242: stonewalling pairs accessed: 3341
365: stonewalling pairs accessed: 2891
2: stonewalling pairs accessed: 2127
230: stonewalling pairs accessed: 1890
449: stonewalling pairs accessed: 2020
580: stonewalling pairs accessed: 2890
520: stonewalling pairs accessed: 2005
107: stonewalling pairs accessed: 3175
309: stonewalling pairs accessed: 1906
361: stonewalling pairs accessed: 3096
36: stonewalling pairs accessed: 2996
431: stonewalling pairs accessed: 2951
576: stonewalling pairs accessed: 2888
513: stonewalling pairs accessed: 2903
118: stonewalling pairs accessed: 2045
301: stonewalling pairs accessed: 3160
254: stonewalling pairs accessed: 2005
403: stonewalling pairs accessed: 1938
15: stonewalling pairs accessed: 2111
180: stonewalling pairs accessed: 2891
448: stonewalling pairs accessed: 2025
547: stonewalling pairs accessed: 1802
499: stonewalling pairs accessed: 2936
102: stonewalling pairs accessed: 3265
319: stonewalling pairs accessed: 1958
404: stonewalling pairs accessed: 3044
16: stonewalling pairs accessed: 3153
437: stonewalling pairs accessed: 2894
569: stonewalling pairs accessed: 2063
510: stonewalling pairs accessed: 1902
91: stonewalling pairs accessed: 1795
306: stonewalling pairs accessed: 2130
413: stonewalling pairs accessed: 2138
7: stonewalling pairs accessed: 3060
214: stonewalling pairs accessed: 1977
475: stonewalling pairs accessed: 1874
545: stonewalling pairs accessed: 2034
523: stonewalling pairs accessed: 1838
93: stonewalling pairs accessed: 1867
317: stonewalling pairs accessed: 1813
283: stonewalling pairs accessed: 1951
381: stonewalling pairs accessed: 2884
41: stonewalling pairs accessed: 2904
420: stonewalling pairs accessed: 2745
567: stonewalling pairs accessed: 1894
489: stonewalling pairs accessed: 1892
99: stonewalling pairs accessed: 3335
342: stonewalling pairs accessed: 1909
250: stonewalling pairs accessed: 2105
392: stonewalling pairs accessed: 2017
40: stonewalling pairs accessed: 1884
238: stonewalling pairs accessed: 3157
469: stonewalling pairs accessed: 2676
577: stonewalling pairs accessed: 3322
497: stonewalling pairs accessed: 3146
71: stonewalling pairs accessed: 3141
344: stonewalling pairs accessed: 3037
405: stonewalling pairs accessed: 3045
22: stonewalling pairs accessed: 3275
445: stonewalling pairs accessed: 2754
571: stonewalling pairs accessed: 2914
503: stonewalling pairs accessed: 1838
98: stonewalling pairs accessed: 2031
307: stonewalling pairs accessed: 1855
373: stonewalling pairs accessed: 1963
8: stonewalling pairs accessed: 3139
456: stonewalling pairs accessed: 2729
555: stonewalling pairs accessed: 2044
521: stonewalling pairs accessed: 2009
90: stonewalling pairs accessed: 1923
339: stonewalling pairs accessed: 1995
380: stonewalling pairs accessed: 1993
51: stonewalling pairs accessed: 3372
450: stonewalling pairs accessed: 1834
598: stonewalling pairs accessed: 1863
493: stonewalling pairs accessed: 2878
81: stonewalling pairs accessed: 2036
308: stonewalling pairs accessed: 3112
390: stonewalling pairs accessed: 3069
5: stonewalling pairs accessed: 2999
440: stonewalling pairs accessed: 1911
549: stonewalling pairs accessed: 1924
538: stonewalling pairs accessed: 1959
109: stonewalling pairs accessed: 2682
310: stonewalling pairs accessed: 2064
394: stonewalling pairs accessed: 3136
18: stonewalling pairs accessed: 2159
225: stonewalling pairs accessed: 1845
468: stonewalling pairs accessed: 2105
592: stonewalling pairs accessed: 2806
516: stonewalling pairs accessed: 1818
62: stonewalling pairs accessed: 2757
340: stonewalling pairs accessed: 1997
374: stonewalling pairs accessed: 3123
57: stonewalling pairs accessed: 1892
232: stonewalling pairs accessed: 1962
444: stonewalling pairs accessed: 3064
572: stonewalling pairs accessed: 2704
524: stonewalling pairs accessed: 2256
69: stonewalling pairs accessed: 3214
354: stonewalling pairs accessed: 3445
367: stonewalling pairs accessed: 3120
47: stonewalling pairs accessed: 2184
446: stonewalling pairs accessed: 1839
583: stonewalling pairs accessed: 1960
525: stonewalling pairs accessed: 2108
77: stonewalling pairs accessed: 2864
352: stonewalling pairs accessed: 3168
395: stonewalling pairs accessed: 3179
52: stonewalling pairs accessed: 2919
463: stonewalling pairs accessed: 2023
574: stonewalling pairs accessed: 3047
486: stonewalling pairs accessed: 2920
67: stonewalling pairs accessed: 1967
355: stonewalling pairs accessed: 3858
396: stonewalling pairs accessed: 2027
44: stonewalling pairs accessed: 1775
236: stonewalling pairs accessed: 1940
425: stonewalling pairs accessed: 3044
573: stonewalling pairs accessed: 2004
509: stonewalling pairs accessed: 2011
78: stonewalling pairs accessed: 3151
328: stonewalling pairs accessed: 1949
406: stonewalling pairs accessed: 3461
23: stonewalling pairs accessed: 1906
461: stonewalling pairs accessed: 1893
579: stonewalling pairs accessed: 3029
492: stonewalling pairs accessed: 3005
108: stonewalling pairs accessed: 2125
331: stonewalling pairs accessed: 3189
366: stonewalling pairs accessed: 3261
53: stonewalling pairs accessed: 2837
204: stonewalling pairs accessed: 2894
467: stonewalling pairs accessed: 1765
544: stonewalling pairs accessed: 2090
488: stonewalling pairs accessed: 3161
84: stonewalling pairs accessed: 3046
333: stonewalling pairs accessed: 3106
383: stonewalling pairs accessed: 3618
6: stonewalling pairs accessed: 3027
192: stonewalling pairs accessed: 1941
441: stonewalling pairs accessed: 1917
586: stonewalling pairs accessed: 1851
511: stonewalling pairs accessed: 2145
89: stonewalling pairs accessed: 1844
327: stonewalling pairs accessed: 1974
258: stonewalling pairs accessed: 2119
385: stonewalling pairs accessed: 2310
12: stonewalling pairs accessed: 2221
452: stonewalling pairs accessed: 2847
582: stonewalling pairs accessed: 2808
491: stonewalling pairs accessed: 3604
61: stonewalling pairs accessed: 3071
356: stonewalling pairs accessed: 2980
378: stonewalling pairs accessed: 3077
21: stonewalling pairs accessed: 3048
189: stonewalling pairs accessed: 1895
424: stonewalling pairs accessed: 2761
564: stonewalling pairs accessed: 2033
502: stonewalling pairs accessed: 2638
68: stonewalling pairs accessed: 3190
304: stonewalling pairs accessed: 2075
389: stonewalling pairs accessed: 2070
56: stonewalling pairs accessed: 3170
422: stonewalling pairs accessed: 3006
585: stonewalling pairs accessed: 2873
490: stonewalling pairs accessed: 3396
114: stonewalling pairs accessed: 1912
350: stonewalling pairs accessed: 3077
417: stonewalling pairs accessed: 3051
11: stonewalling pairs accessed: 1913
436: stonewalling pairs accessed: 2037
590: stonewalling pairs accessed: 2952
484: stonewalling pairs accessed: 3012
110: stonewalling pairs accessed: 2007
347: stonewalling pairs accessed: 3151
379: stonewalling pairs accessed: 2070
27: stonewalling pairs accessed: 2287
429: stonewalling pairs accessed: 2943
553: stonewalling pairs accessed: 1942
480: stonewalling pairs accessed: 2866
80: stonewalling pairs accessed: 3028
336: stonewalling pairs accessed: 2984
268: stonewalling pairs accessed: 2175
407: stonewalling pairs accessed: 2340
13: stonewalling pairs accessed: 1782
434: stonewalling pairs accessed: 2067
570: stonewalling pairs accessed: 3094
504: stonewalling pairs accessed: 2996
92: stonewalling pairs accessed: 2167
348: stonewalling pairs accessed: 1942
401: stonewalling pairs accessed: 2034
26: stonewalling pairs accessed: 1822
462: stonewalling pairs accessed: 2696
595: stonewalling pairs accessed: 2917
485: stonewalling pairs accessed: 2979
64: stonewalling pairs accessed: 1802
349: stonewalling pairs accessed: 2111
261: stonewalling pairs accessed: 2144
382: stonewalling pairs accessed: 3406
43: stonewalling pairs accessed: 3054
191: stonewalling pairs accessed: 2740
428: stonewalling pairs accessed: 3102
593: stonewalling pairs accessed: 1879
494: stonewalling pairs accessed: 1972
79: stonewalling pairs accessed: 3234
325: stonewalling pairs accessed: 2175
257: stonewalling pairs accessed: 2012
402: stonewalling pairs accessed: 1980
0: stonewalling pairs accessed: 1815
432: stonewalling pairs accessed: 2043
541: stonewalling pairs accessed: 2132
507: stonewalling pairs accessed: 3154
104: stonewalling pairs accessed: 1906
341: stonewalling pairs accessed: 1890
155: stonewalling pairs accessed: 3151
371: stonewalling pairs accessed: 3068
50: stonewalling pairs accessed: 3445
211: stonewalling pairs accessed: 1953
459: stonewalling pairs accessed: 2438
575: stonewalling pairs accessed: 1979
487: stonewalling pairs accessed: 2937
73: stonewalling pairs accessed: 1925
311: stonewalling pairs accessed: 1914
370: stonewalling pairs accessed: 2857
48: stonewalling pairs accessed: 3185
227: stonewalling pairs accessed: 2994
477: stonewalling pairs accessed: 1909
589: stonewalling pairs accessed: 2748
529: stonewalling pairs accessed: 1924
86: stonewalling pairs accessed: 1763
343: stonewalling pairs accessed: 2078
410: stonewalling pairs accessed: 2074
9: stonewalling pairs accessed: 2941
228: stonewalling pairs accessed: 1969
471: stonewalling pairs accessed: 2705
568: stonewalling pairs accessed: 2114
527: stonewalling pairs accessed: 2190
111: stonewalling pairs accessed: 1870
338: stonewalling pairs accessed: 2083
159: stonewalling pairs accessed: 2588
393: stonewalling pairs accessed: 2792
38: stonewalling pairs accessed: 3140
220: stonewalling pairs accessed: 1889
426: stonewalling pairs accessed: 2890
596: stonewalling pairs accessed: 2794
518: stonewalling pairs accessed: 2086
95: stonewalling pairs accessed: 2073
324: stonewalling pairs accessed: 1911
178: stonewalling pairs accessed: 1940
398: stonewalling pairs accessed: 1955
35: stonewalling pairs accessed: 2222
195: stonewalling pairs accessed: 2026
457: stonewalling pairs accessed: 1992
597: stonewalling pairs accessed: 1978
539: stonewalling pairs accessed: 1752
88: stonewalling pairs accessed: 1945
357: stonewalling pairs accessed: 1966
364: stonewalling pairs accessed: 3118
49: stonewalling pairs accessed: 2040
215: stonewalling pairs accessed: 3247
427: stonewalling pairs accessed: 2794
563: stonewalling pairs accessed: 1915
508: stonewalling pairs accessed: 1897
63: stonewalling pairs accessed: 1991
305: stonewalling pairs accessed: 1981
173: stonewalling pairs accessed: 2782
416: stonewalling pairs accessed: 1896
25: stonewalling pairs accessed: 2059
470: stonewalling pairs accessed: 2834
543: stonewalling pairs accessed: 1955
532: stonewalling pairs accessed: 1939
97: stonewalling pairs accessed: 1855
322: stonewalling pairs accessed: 2150
165: stonewalling pairs accessed: 2607
391: stonewalling pairs accessed: 2208
39: stonewalling pairs accessed: 2226
229: stonewalling pairs accessed: 1985
476: stonewalling pairs accessed: 2111
565: stonewalling pairs accessed: 1860
522: stonewalling pairs accessed: 1865
83: stonewalling pairs accessed: 1777
320: stonewalling pairs accessed: 2087
384: stonewalling pairs accessed: 2028
19: stonewalling pairs accessed: 1957
433: stonewalling pairs accessed: 2031
552: stonewalling pairs accessed: 1966
535: stonewalling pairs accessed: 2157
65: stonewalling pairs accessed: 1981
321: stonewalling pairs accessed: 1973
154: stonewalling pairs accessed: 3183
375: stonewalling pairs accessed: 2080
34: stonewalling pairs accessed: 2044
442: stonewalling pairs accessed: 2139
559: stonewalling pairs accessed: 1816
537: stonewalling pairs accessed: 2037
94: stonewalling pairs accessed: 2007
353: stonewalling pairs accessed: 1989
127: stonewalling pairs accessed: 2568
408: stonewalling pairs accessed: 2082
45: stonewalling pairs accessed: 1970
454: stonewalling pairs accessed: 1935
584: stonewalling pairs accessed: 2150
506: stonewalling pairs accessed: 2033
60: stonewalling pairs accessed: 1858
326: stonewalling pairs accessed: 2065
126: stonewalling pairs accessed: 2692
411: stonewalling pairs accessed: 2043
31: stonewalling pairs accessed: 1928
473: stonewalling pairs accessed: 2013
588: stonewalling pairs accessed: 2896
482: stonewalling pairs accessed: 2017
87: stonewalling pairs accessed: 1802
318: stonewalling pairs accessed: 2035
150: stonewalling pairs accessed: 2721
369: stonewalling pairs accessed: 1946
33: stonewalling pairs accessed: 2182
453: stonewalling pairs accessed: 2677
550: stonewalling pairs accessed: 2010
533: stonewalling pairs accessed: 1963
119: stonewalling pairs accessed: 2064
312: stonewalling pairs accessed: 2017
141: stonewalling pairs accessed: 1936
414: stonewalling pairs accessed: 2039
20: stonewalling pairs accessed: 1681
474: stonewalling pairs accessed: 1897
561: stonewalling pairs accessed: 1913
517: stonewalling pairs accessed: 2153
113: stonewalling pairs accessed: 2045
323: stonewalling pairs accessed: 2117
171: stonewalling pairs accessed: 1917
399: stonewalling pairs accessed: 2279
10: stonewalling pairs accessed: 2018
435: stonewalling pairs accessed: 2809
540: stonewalling pairs accessed: 1890
483: stonewalling pairs accessed: 2096
74: stonewalling pairs accessed: 1862
332: stonewalling pairs accessed: 1959
167: stonewalling pairs accessed: 1945
368: stonewalling pairs accessed: 2046
1: stonewalling pairs accessed: 1908
466: stonewalling pairs accessed: 1866
557: stonewalling pairs accessed: 1947
536: stonewalling pairs accessed: 1935
112: stonewalling pairs accessed: 2206
330: stonewalling pairs accessed: 1811
148: stonewalling pairs accessed: 1851
263: stonewalling pairs accessed: 3265
362: stonewalling pairs accessed: 1964
55: stonewalling pairs accessed: 2214
182: stonewalling pairs accessed: 3039
423: stonewalling pairs accessed: 2042
566: stonewalling pairs accessed: 1838
531: stonewalling pairs accessed: 1936
116: stonewalling pairs accessed: 1949
300: stonewalling pairs accessed: 2068
387: stonewalling pairs accessed: 2277
17: stonewalling pairs accessed: 1920
188: stonewalling pairs accessed: 2111
472: stonewalling pairs accessed: 2014
578: stonewalling pairs accessed: 2007
512: stonewalling pairs accessed: 1889
75: stonewalling pairs accessed: 1942
302: stonewalling pairs accessed: 2029
124: stonewalling pairs accessed: 2551
372: stonewalling pairs accessed: 1811
42: stonewalling pairs accessed: 1915
438: stonewalling pairs accessed: 1816
548: stonewalling pairs accessed: 1907
519: stonewalling pairs accessed: 2063
70: stonewalling pairs accessed: 1816
335: stonewalling pairs accessed: 1869
400: stonewalling pairs accessed: 1966
3: stonewalling pairs accessed: 1821
458: stonewalling pairs accessed: 1834
542: stonewalling pairs accessed: 1994
514: stonewalling pairs accessed: 2038
85: stonewalling pairs accessed: 1770
314: stonewalling pairs accessed: 1933
418: stonewalling pairs accessed: 2039
54: stonewalling pairs accessed: 1711
226: stonewalling pairs accessed: 3251
430: stonewalling pairs accessed: 1924
556: stonewalling pairs accessed: 1874
500: stonewalling pairs accessed: 1851
82: stonewalling pairs accessed: 1859
303: stonewalling pairs accessed: 1913
158: stonewalling pairs accessed: 2738
274: stonewalling pairs accessed: 1952
376: stonewalling pairs accessed: 2146
37: stonewalling pairs accessed: 2165
460: stonewalling pairs accessed: 1844
554: stonewalling pairs accessed: 1930
530: stonewalling pairs accessed: 1866
76: stonewalling pairs accessed: 2074
315: stonewalling pairs accessed: 2229
168: stonewalling pairs accessed: 2858
412: stonewalling pairs accessed: 2205
4: stonewalling pairs accessed: 1867
455: stonewalling pairs accessed: 1779
558: stonewalling pairs accessed: 1879
515: stonewalling pairs accessed: 2017
105: stonewalling pairs accessed: 2115
316: stonewalling pairs accessed: 1984
175: stonewalling pairs accessed: 2889
377: stonewalling pairs accessed: 1976
46: stonewalling pairs accessed: 2089
451: stonewalling pairs accessed: 1926
594: stonewalling pairs accessed: 1871
501: stonewalling pairs accessed: 2081
101: stonewalling pairs accessed: 2046
351: stonewalling pairs accessed: 1873
121: stonewalling pairs accessed: 1889
363: stonewalling pairs accessed: 2083
29: stonewalling pairs accessed: 1741
216: stonewalling pairs accessed: 1874
439: stonewalling pairs accessed: 1959
591: stonewalling pairs accessed: 1942
528: stonewalling pairs accessed: 1975
96: stonewalling pairs accessed: 1837
329: stonewalling pairs accessed: 2186
143: stonewalling pairs accessed: 1799
259: stonewalling pairs accessed: 2182
388: stonewalling pairs accessed: 2078
32: stonewalling pairs accessed: 2218
184: stonewalling pairs accessed: 1831
465: stonewalling pairs accessed: 1849
560: stonewalling pairs accessed: 1775
496: stonewalling pairs accessed: 1863
72: stonewalling pairs accessed: 2060
345: stonewalling pairs accessed: 2039
140: stonewalling pairs accessed: 1860
386: stonewalling pairs accessed: 2193
30: stonewalling pairs accessed: 1787
421: stonewalling pairs accessed: 1874
546: stonewalling pairs accessed: 2173
505: stonewalling pairs accessed: 1828
117: stonewalling pairs accessed: 2042
346: stonewalling pairs accessed: 1959
166: stonewalling pairs accessed: 1852
24: stonewalling pairs accessed: 2106
464: stonewalling pairs accessed: 1727
481: stonewalling pairs accessed: 2071
179: stonewalling pairs accessed: 1873
495: stonewalling pairs accessed: 1857
172: stonewalling pairs accessed: 2796
170: stonewalling pairs accessed: 2841
156: stonewalling pairs accessed: 2836
177: stonewalling pairs accessed: 2767
133: stonewalling pairs accessed: 1681
120: stonewalling pairs accessed: 1891
135: stonewalling pairs accessed: 1888
153: stonewalling pairs accessed: 1906
149: stonewalling pairs accessed: 1865
122: stonewalling pairs accessed: 2051
231: stonewalling pairs accessed: 2885
201: stonewalling pairs accessed: 1978
292: stonewalling pairs accessed: 2102
123: stonewalling pairs accessed: 1755
144: stonewalling pairs accessed: 1951
161: stonewalling pairs accessed: 1884
160: stonewalling pairs accessed: 1847
138: stonewalling pairs accessed: 1829
163: stonewalling pairs accessed: 1992
131: stonewalling pairs accessed: 1825
169: stonewalling pairs accessed: 1868
129: stonewalling pairs accessed: 2001
176: stonewalling pairs accessed: 1958
202: stonewalling pairs accessed: 3001
174: stonewalling pairs accessed: 1895
146: stonewalling pairs accessed: 1757
151: stonewalling pairs accessed: 1897
157: stonewalling pairs accessed: 1954
134: stonewalling pairs accessed: 1939
147: stonewalling pairs accessed: 1933
219: stonewalling pairs accessed: 1894
152: stonewalling pairs accessed: 1736
132: stonewalling pairs accessed: 1935
164: stonewalling pairs accessed: 1767
162: stonewalling pairs accessed: 1791
142: stonewalling pairs accessed: 1810
130: stonewalling pairs accessed: 2009
128: stonewalling pairs accessed: 1887
139: stonewalling pairs accessed: 1840
145: stonewalling pairs accessed: 1877
187: stonewalling pairs accessed: 2009
186: stonewalling pairs accessed: 2004
253: stonewalling pairs accessed: 1980
223: stonewalling pairs accessed: 1797
136: stonewalling pairs accessed: 2049
218: stonewalling pairs accessed: 2165
239: stonewalling pairs accessed: 1755
125: stonewalling pairs accessed: 1916
210: stonewalling pairs accessed: 1960
137: stonewalling pairs accessed: 1908
199: stonewalling pairs accessed: 1958
197: stonewalling pairs accessed: 1935
272: stonewalling pairs accessed: 2078
237: stonewalling pairs accessed: 1827
190: stonewalling pairs accessed: 2701
208: stonewalling pairs accessed: 1955
198: stonewalling pairs accessed: 2919
234: stonewalling pairs accessed: 2769
273: stonewalling pairs accessed: 2061
252: stonewalling pairs accessed: 2055
185: stonewalling pairs accessed: 2928
209: stonewalling pairs accessed: 1892
196: stonewalling pairs accessed: 1932
224: stonewalling pairs accessed: 2027
213: stonewalling pairs accessed: 1869
235: stonewalling pairs accessed: 2741
287: stonewalling pairs accessed: 2053
206: stonewalling pairs accessed: 2189
221: stonewalling pairs accessed: 1948
200: stonewalling pairs accessed: 1929
217: stonewalling pairs accessed: 3037
222: stonewalling pairs accessed: 1927
203: stonewalling pairs accessed: 2751
205: stonewalling pairs accessed: 2734
212: stonewalling pairs accessed: 2796
194: stonewalling pairs accessed: 2014
289: stonewalling pairs accessed: 1995
193: stonewalling pairs accessed: 1973
183: stonewalling pairs accessed: 1873
291: stonewalling pairs accessed: 2020
288: stonewalling pairs accessed: 3124
251: stonewalling pairs accessed: 3355
256: stonewalling pairs accessed: 2146
260: stonewalling pairs accessed: 2109
282: stonewalling pairs accessed: 2037
285: stonewalling pairs accessed: 2032
264: stonewalling pairs accessed: 2194
243: stonewalling pairs accessed: 3159
286: stonewalling pairs accessed: 2141
293: stonewalling pairs accessed: 2032
290: stonewalling pairs accessed: 3108
275: stonewalling pairs accessed: 2088
284: stonewalling pairs accessed: 1979
294: stonewalling pairs accessed: 3293
255: stonewalling pairs accessed: 1953
248: stonewalling pairs accessed: 1990
278: stonewalling pairs accessed: 1897
295: stonewalling pairs accessed: 2103
271: stonewalling pairs accessed: 2157
267: stonewalling pairs accessed: 2155
281: stonewalling pairs accessed: 2159
262: stonewalling pairs accessed: 3312
240: stonewalling pairs accessed: 2113
296: stonewalling pairs accessed: 3379
265: stonewalling pairs accessed: 1987
244: stonewalling pairs accessed: 3413
279: stonewalling pairs accessed: 2091
276: stonewalling pairs accessed: 2199
280: stonewalling pairs accessed: 2055
269: stonewalling pairs accessed: 1945
298: stonewalling pairs accessed: 1942
266: stonewalling pairs accessed: 2001
249: stonewalling pairs accessed: 3232
247: stonewalling pairs accessed: 3400
277: stonewalling pairs accessed: 2018
299: stonewalling pairs accessed: 3263
245: stonewalling pairs accessed: 3474
270: stonewalling pairs accessed: 3215
246: stonewalling pairs accessed: 2172
297: stonewalling pairs accessed: 2032
stonewalling pairs accessed min: 1681 max: 3858 -- min data: 6.6 GiB mean data: 8.9 GiB time: 300.6s
WARNING: Expected aggregate file size       = 6241124352000000.
WARNING: Stat() of aggregate file size      = 9708974899200.
WARNING: Using actual aggregate bytes moved = 9708974899200.
WARNING: maybe caused by deadlineForStonewalling
write     14548      3638       0.000255    10158080000 4096       0.079455   636.35     0.019625   636.45     0   
Max Write: 14548.18 MiB/sec (15254.87 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       14548.18   14548.18   14548.18       0.00    3637.05    3637.05    3637.05       0.00  636.45070     300.63      18248.68     0    600  60    1   1     1        1         0    0      1 10401873920000  4194304 9259200.0 POSIX      0
Finished            : Tue Jun  2 09:07:21 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 09:47:19 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -r -R -s 1900000 -a POSIX --posix.odirect -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datadir/2020.06.02-08.56.31-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datadir/2020.06.02-08.56.31-scr/ior_hard/stonewall
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 09:47:19 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr/ior_hard
FS                  : 50.0 TiB   Used FS: 44.0%   Inodes: 858.3 Mi   Used Inodes: 5.0%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-08.56.31-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 48.74 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Tue Jun  2 09:47:19 2020

WARNING: Expected aggregate file size       = 53589120000000.
WARNING: Stat() of aggregate file size      = 4698976089600.
WARNING: Using actual aggregate bytes moved = 4698976089600.
read      9946       221865     450.55      45.91      45.91      0.002965   450.55     0.000405   450.55     0   
Max Read:  9946.22 MiB/sec (10429.37 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         9946.22    9946.22    9946.22       0.00  221863.68  221863.68  221863.68       0.00  450.55234         NA            NA     0    600  60    1   0     1        1         0    0 1900000    47008    47008 4481293.0 POSIX      0
Finished            : Tue Jun  2 09:54:49 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 09:15:49 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -w -s 1900000 -a POSIX --posix.odirect -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datadir/2020.06.02-08.56.31-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datadir/2020.06.02-08.56.31-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 09:15:49 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr/ior_hard
FS                  : 50.0 TiB   Used FS: 35.3%   Inodes: 858.3 Mi   Used Inodes: 4.0%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-08.56.31-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 48.74 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Tue Jun  2 09:15:49 2020
359: stonewalling pairs accessed: 166601
419: stonewalling pairs accessed: 166601
539: stonewalling pairs accessed: 166601
479: stonewalling pairs accessed: 166601
599: stonewalling pairs accessed: 166601
59: stonewalling pairs accessed: 166602
179: stonewalling pairs accessed: 166602
119: stonewalling pairs accessed: 166602
239: stonewalling pairs accessed: 166602
418: stonewalling pairs accessed: 36
598: stonewalling pairs accessed: 38
178: stonewalling pairs accessed: 41
358: stonewalling pairs accessed: 40
299: stonewalling pairs accessed: 166602
118: stonewalling pairs accessed: 40
538: stonewalling pairs accessed: 47
478: stonewalling pairs accessed: 47
298: stonewalling pairs accessed: 38
160: stonewalling pairs accessed: 46
332: stonewalling pairs accessed: 55
394: stonewalling pairs accessed: 36
459: stonewalling pairs accessed: 45
591: stonewalling pairs accessed: 66
177: stonewalling pairs accessed: 35
99: stonewalling pairs accessed: 52
190: stonewalling pairs accessed: 34
335: stonewalling pairs accessed: 32
407: stonewalling pairs accessed: 30
494: stonewalling pairs accessed: 34
461: stonewalling pairs accessed: 49
282: stonewalling pairs accessed: 37
24: stonewalling pairs accessed: 50
584: stonewalling pairs accessed: 58
170: stonewalling pairs accessed: 46
98: stonewalling pairs accessed: 50
224: stonewalling pairs accessed: 41
347: stonewalling pairs accessed: 34
398: stonewalling pairs accessed: 28
500: stonewalling pairs accessed: 36
469: stonewalling pairs accessed: 35
259: stonewalling pairs accessed: 36
26: stonewalling pairs accessed: 38
590: stonewalling pairs accessed: 65
138: stonewalling pairs accessed: 50
92: stonewalling pairs accessed: 40
184: stonewalling pairs accessed: 33
349: stonewalling pairs accessed: 31
379: stonewalling pairs accessed: 38
496: stonewalling pairs accessed: 39
476: stonewalling pairs accessed: 60
276: stonewalling pairs accessed: 32
11: stonewalling pairs accessed: 40
595: stonewalling pairs accessed: 64
157: stonewalling pairs accessed: 54
112: stonewalling pairs accessed: 36
201: stonewalling pairs accessed: 40
318: stonewalling pairs accessed: 61
372: stonewalling pairs accessed: 42
510: stonewalling pairs accessed: 40
425: stonewalling pairs accessed: 61
257: stonewalling pairs accessed: 34
37: stonewalling pairs accessed: 57
548: stonewalling pairs accessed: 55
129: stonewalling pairs accessed: 61
114: stonewalling pairs accessed: 32
208: stonewalling pairs accessed: 44
325: stonewalling pairs accessed: 56
392: stonewalling pairs accessed: 39
481: stonewalling pairs accessed: 32
427: stonewalling pairs accessed: 35
284: stonewalling pairs accessed: 36
20: stonewalling pairs accessed: 34
571: stonewalling pairs accessed: 51
137: stonewalling pairs accessed: 58
64: stonewalling pairs accessed: 51
215: stonewalling pairs accessed: 43
326: stonewalling pairs accessed: 43
411: stonewalling pairs accessed: 30
488: stonewalling pairs accessed: 28
434: stonewalling pairs accessed: 57
242: stonewalling pairs accessed: 63
15: stonewalling pairs accessed: 41
580: stonewalling pairs accessed: 50
143: stonewalling pairs accessed: 58
74: stonewalling pairs accessed: 47
236: stonewalling pairs accessed: 72
357: stonewalling pairs accessed: 53
413: stonewalling pairs accessed: 46
495: stonewalling pairs accessed: 49
472: stonewalling pairs accessed: 33
243: stonewalling pairs accessed: 62
31: stonewalling pairs accessed: 42
557: stonewalling pairs accessed: 61
168: stonewalling pairs accessed: 59
85: stonewalling pairs accessed: 32
188: stonewalling pairs accessed: 51
308: stonewalling pairs accessed: 64
404: stonewalling pairs accessed: 37
503: stonewalling pairs accessed: 35
477: stonewalling pairs accessed: 39
254: stonewalling pairs accessed: 42
14: stonewalling pairs accessed: 36
545: stonewalling pairs accessed: 41
164: stonewalling pairs accessed: 42
68: stonewalling pairs accessed: 42
209: stonewalling pairs accessed: 38
314: stonewalling pairs accessed: 62
361: stonewalling pairs accessed: 31
505: stonewalling pairs accessed: 36
435: stonewalling pairs accessed: 53
255: stonewalling pairs accessed: 35
40: stonewalling pairs accessed: 32
553: stonewalling pairs accessed: 62
142: stonewalling pairs accessed: 39
69: stonewalling pairs accessed: 30
230: stonewalling pairs accessed: 40
315: stonewalling pairs accessed: 35
366: stonewalling pairs accessed: 33
507: stonewalling pairs accessed: 34
464: stonewalling pairs accessed: 31
278: stonewalling pairs accessed: 49
2: stonewalling pairs accessed: 35
569: stonewalling pairs accessed: 42
152: stonewalling pairs accessed: 60
93: stonewalling pairs accessed: 28
218: stonewalling pairs accessed: 35
342: stonewalling pairs accessed: 40
387: stonewalling pairs accessed: 36
518: stonewalling pairs accessed: 39
475: stonewalling pairs accessed: 36
247: stonewalling pairs accessed: 56
9: stonewalling pairs accessed: 55
570: stonewalling pairs accessed: 61
128: stonewalling pairs accessed: 56
75: stonewalling pairs accessed: 43
232: stonewalling pairs accessed: 44
346: stonewalling pairs accessed: 37
397: stonewalling pairs accessed: 34
519: stonewalling pairs accessed: 28
437: stonewalling pairs accessed: 36
258: stonewalling pairs accessed: 32
25: stonewalling pairs accessed: 54
549: stonewalling pairs accessed: 36
136: stonewalling pairs accessed: 34
95: stonewalling pairs accessed: 50
182: stonewalling pairs accessed: 36
322: stonewalling pairs accessed: 61
360: stonewalling pairs accessed: 47
523: stonewalling pairs accessed: 37
424: stonewalling pairs accessed: 47
279: stonewalling pairs accessed: 52
36: stonewalling pairs accessed: 50
559: stonewalling pairs accessed: 46
153: stonewalling pairs accessed: 42
107: stonewalling pairs accessed: 41
198: stonewalling pairs accessed: 31
356: stonewalling pairs accessed: 48
363: stonewalling pairs accessed: 45
526: stonewalling pairs accessed: 34
442: stonewalling pairs accessed: 31
291: stonewalling pairs accessed: 45
51: stonewalling pairs accessed: 31
586: stonewalling pairs accessed: 47
156: stonewalling pairs accessed: 59
103: stonewalling pairs accessed: 34
229: stonewalling pairs accessed: 33
330: stonewalling pairs accessed: 56
368: stonewalling pairs accessed: 37
529: stonewalling pairs accessed: 30
474: stonewalling pairs accessed: 59
244: stonewalling pairs accessed: 60
57: stonewalling pairs accessed: 59
594: stonewalling pairs accessed: 50
161: stonewalling pairs accessed: 59
77: stonewalling pairs accessed: 42
220: stonewalling pairs accessed: 35
351: stonewalling pairs accessed: 42
369: stonewalling pairs accessed: 35
536: stonewalling pairs accessed: 31
432: stonewalling pairs accessed: 47
245: stonewalling pairs accessed: 35
8: stonewalling pairs accessed: 32
561: stonewalling pairs accessed: 57
176: stonewalling pairs accessed: 59
106: stonewalling pairs accessed: 53
237: stonewalling pairs accessed: 37
324: stonewalling pairs accessed: 60
378: stonewalling pairs accessed: 39
480: stonewalling pairs accessed: 33
441: stonewalling pairs accessed: 36
277: stonewalling pairs accessed: 45
33: stonewalling pairs accessed: 54
540: stonewalling pairs accessed: 47
120: stonewalling pairs accessed: 32
82: stonewalling pairs accessed: 49
191: stonewalling pairs accessed: 45
333: stonewalling pairs accessed: 52
380: stonewalling pairs accessed: 35
515: stonewalling pairs accessed: 33
426: stonewalling pairs accessed: 41
249: stonewalling pairs accessed: 39
30: stonewalling pairs accessed: 57
572: stonewalling pairs accessed: 59
124: stonewalling pairs accessed: 55
90: stonewalling pairs accessed: 36
225: stonewalling pairs accessed: 32
348: stonewalling pairs accessed: 57
382: stonewalling pairs accessed: 30
483: stonewalling pairs accessed: 31
448: stonewalling pairs accessed: 40
240: stonewalling pairs accessed: 39
16: stonewalling pairs accessed: 57
542: stonewalling pairs accessed: 68
125: stonewalling pairs accessed: 52
108: stonewalling pairs accessed: 28
226: stonewalling pairs accessed: 31
327: stonewalling pairs accessed: 57
383: stonewalling pairs accessed: 29
532: stonewalling pairs accessed: 50
471: stonewalling pairs accessed: 52
285: stonewalling pairs accessed: 32
21: stonewalling pairs accessed: 54
585: stonewalling pairs accessed: 35
126: stonewalling pairs accessed: 42
63: stonewalling pairs accessed: 34
207: stonewalling pairs accessed: 31
317: stonewalling pairs accessed: 63
385: stonewalling pairs accessed: 31
522: stonewalling pairs accessed: 31
451: stonewalling pairs accessed: 39
287: stonewalling pairs accessed: 31
27: stonewalling pairs accessed: 34
593: stonewalling pairs accessed: 56
130: stonewalling pairs accessed: 60
78: stonewalling pairs accessed: 34
192: stonewalling pairs accessed: 32
300: stonewalling pairs accessed: 39
396: stonewalling pairs accessed: 29
501: stonewalling pairs accessed: 28
430: stonewalling pairs accessed: 61
297: stonewalling pairs accessed: 56
38: stonewalling pairs accessed: 57
563: stonewalling pairs accessed: 35
131: stonewalling pairs accessed: 35
104: stonewalling pairs accessed: 54
200: stonewalling pairs accessed: 45
302: stonewalling pairs accessed: 74
408: stonewalling pairs accessed: 41
509: stonewalling pairs accessed: 31
429: stonewalling pairs accessed: 36
264: stonewalling pairs accessed: 46
56: stonewalling pairs accessed: 60
551: stonewalling pairs accessed: 43
132: stonewalling pairs accessed: 61
65: stonewalling pairs accessed: 28
206: stonewalling pairs accessed: 45
305: stonewalling pairs accessed: 72
409: stonewalling pairs accessed: 28
530: stonewalling pairs accessed: 33
450: stonewalling pairs accessed: 31
272: stonewalling pairs accessed: 46
0: stonewalling pairs accessed: 36
581: stonewalling pairs accessed: 34
133: stonewalling pairs accessed: 60
87: stonewalling pairs accessed: 32
217: stonewalling pairs accessed: 44
350: stonewalling pairs accessed: 40
416: stonewalling pairs accessed: 31
517: stonewalling pairs accessed: 28
446: stonewalling pairs accessed: 31
274: stonewalling pairs accessed: 31
17: stonewalling pairs accessed: 54
547: stonewalling pairs accessed: 50
134: stonewalling pairs accessed: 60
111: stonewalling pairs accessed: 32
180: stonewalling pairs accessed: 28
303: stonewalling pairs accessed: 34
417: stonewalling pairs accessed: 33
537: stonewalling pairs accessed: 39
447: stonewalling pairs accessed: 33
251: stonewalling pairs accessed: 48
1: stonewalling pairs accessed: 34
558: stonewalling pairs accessed: 34
135: stonewalling pairs accessed: 42
101: stonewalling pairs accessed: 55
189: stonewalling pairs accessed: 53
323: stonewalling pairs accessed: 55
384: stonewalling pairs accessed: 31
485: stonewalling pairs accessed: 33
470: stonewalling pairs accessed: 39
271: stonewalling pairs accessed: 39
7: stonewalling pairs accessed: 36
550: stonewalling pairs accessed: 65
139: stonewalling pairs accessed: 34
88: stonewalling pairs accessed: 51
227: stonewalling pairs accessed: 31
329: stonewalling pairs accessed: 57
403: stonewalling pairs accessed: 36
497: stonewalling pairs accessed: 39
458: stonewalling pairs accessed: 33
275: stonewalling pairs accessed: 31
13: stonewalling pairs accessed: 54
566: stonewalling pairs accessed: 39
140: stonewalling pairs accessed: 39
86: stonewalling pairs accessed: 28
210: stonewalling pairs accessed: 31
334: stonewalling pairs accessed: 31
364: stonewalling pairs accessed: 31
521: stonewalling pairs accessed: 33
462: stonewalling pairs accessed: 50
263: stonewalling pairs accessed: 31
58: stonewalling pairs accessed: 28
582: stonewalling pairs accessed: 63
146: stonewalling pairs accessed: 60
61: stonewalling pairs accessed: 28
211: stonewalling pairs accessed: 28
353: stonewalling pairs accessed: 38
365: stonewalling pairs accessed: 45
535: stonewalling pairs accessed: 31
421: stonewalling pairs accessed: 39
265: stonewalling pairs accessed: 32
3: stonewalling pairs accessed: 32
560: stonewalling pairs accessed: 65
147: stonewalling pairs accessed: 32
109: stonewalling pairs accessed: 28
203: stonewalling pairs accessed: 32
336: stonewalling pairs accessed: 56
388: stonewalling pairs accessed: 41
487: stonewalling pairs accessed: 31
423: stonewalling pairs accessed: 70
270: stonewalling pairs accessed: 31
55: stonewalling pairs accessed: 32
577: stonewalling pairs accessed: 63
162: stonewalling pairs accessed: 59
91: stonewalling pairs accessed: 32
221: stonewalling pairs accessed: 32
316: stonewalling pairs accessed: 39
402: stonewalling pairs accessed: 43
504: stonewalling pairs accessed: 31
443: stonewalling pairs accessed: 31
266: stonewalling pairs accessed: 31
32: stonewalling pairs accessed: 32
552: stonewalling pairs accessed: 59
163: stonewalling pairs accessed: 34
94: stonewalling pairs accessed: 32
231: stonewalling pairs accessed: 32
309: stonewalling pairs accessed: 68
414: stonewalling pairs accessed: 39
531: stonewalling pairs accessed: 39
467: stonewalling pairs accessed: 33
288: stonewalling pairs accessed: 31
43: stonewalling pairs accessed: 37
576: stonewalling pairs accessed: 63
167: stonewalling pairs accessed: 60
67: stonewalling pairs accessed: 32
195: stonewalling pairs accessed: 28
345: stonewalling pairs accessed: 32
370: stonewalling pairs accessed: 31
508: stonewalling pairs accessed: 28
436: stonewalling pairs accessed: 41
283: stonewalling pairs accessed: 32
50: stonewalling pairs accessed: 32
574: stonewalling pairs accessed: 46
169: stonewalling pairs accessed: 32
113: stonewalling pairs accessed: 32
181: stonewalling pairs accessed: 28
307: stonewalling pairs accessed: 59
362: stonewalling pairs accessed: 41
511: stonewalling pairs accessed: 33
456: stonewalling pairs accessed: 31
295: stonewalling pairs accessed: 41
54: stonewalling pairs accessed: 60
583: stonewalling pairs accessed: 63
175: stonewalling pairs accessed: 53
79: stonewalling pairs accessed: 28
216: stonewalling pairs accessed: 32
339: stonewalling pairs accessed: 45
373: stonewalling pairs accessed: 31
527: stonewalling pairs accessed: 39
463: stonewalling pairs accessed: 39
273: stonewalling pairs accessed: 31
46: stonewalling pairs accessed: 32
579: stonewalling pairs accessed: 33
150: stonewalling pairs accessed: 61
102: stonewalling pairs accessed: 28
185: stonewalling pairs accessed: 32
311: stonewalling pairs accessed: 68
389: stonewalling pairs accessed: 31
484: stonewalling pairs accessed: 33
420: stonewalling pairs accessed: 50
269: stonewalling pairs accessed: 32
10: stonewalling pairs accessed: 32
589: stonewalling pairs accessed: 45
172: stonewalling pairs accessed: 37
110: stonewalling pairs accessed: 32
205: stonewalling pairs accessed: 32
355: stonewalling pairs accessed: 60
415: stonewalling pairs accessed: 28
493: stonewalling pairs accessed: 28
438: stonewalling pairs accessed: 33
267: stonewalling pairs accessed: 32
44: stonewalling pairs accessed: 60
564: stonewalling pairs accessed: 59
121: stonewalling pairs accessed: 32
70: stonewalling pairs accessed: 28
233: stonewalling pairs accessed: 32
338: stonewalling pairs accessed: 31
393: stonewalling pairs accessed: 39
489: stonewalling pairs accessed: 31
422: stonewalling pairs accessed: 41
241: stonewalling pairs accessed: 32
42: stonewalling pairs accessed: 60
588: stonewalling pairs accessed: 43
144: stonewalling pairs accessed: 32
117: stonewalling pairs accessed: 32
199: stonewalling pairs accessed: 28
344: stonewalling pairs accessed: 58
377: stonewalling pairs accessed: 31
516: stonewalling pairs accessed: 31
444: stonewalling pairs accessed: 41
280: stonewalling pairs accessed: 32
49: stonewalling pairs accessed: 28
541: stonewalling pairs accessed: 33
123: stonewalling pairs accessed: 32
96: stonewalling pairs accessed: 28
183: stonewalling pairs accessed: 32
306: stonewalling pairs accessed: 32
367: stonewalling pairs accessed: 45
514: stonewalling pairs accessed: 28
473: stonewalling pairs accessed: 33
248: stonewalling pairs accessed: 32
6: stonewalling pairs accessed: 32
597: stonewalling pairs accessed: 59
171: stonewalling pairs accessed: 32
115: stonewalling pairs accessed: 32
212: stonewalling pairs accessed: 32
331: stonewalling pairs accessed: 31
371: stonewalling pairs accessed: 31
533: stonewalling pairs accessed: 33
433: stonewalling pairs accessed: 31
292: stonewalling pairs accessed: 32
35: stonewalling pairs accessed: 32
546: stonewalling pairs accessed: 39
122: stonewalling pairs accessed: 32
116: stonewalling pairs accessed: 32
214: stonewalling pairs accessed: 28
340: stonewalling pairs accessed: 58
386: stonewalling pairs accessed: 41
498: stonewalling pairs accessed: 31
468: stonewalling pairs accessed: 31
293: stonewalling pairs accessed: 32
18: stonewalling pairs accessed: 32
578: stonewalling pairs accessed: 50
141: stonewalling pairs accessed: 32
97: stonewalling pairs accessed: 28
219: stonewalling pairs accessed: 32
354: stonewalling pairs accessed: 45
376: stonewalling pairs accessed: 41
534: stonewalling pairs accessed: 31
454: stonewalling pairs accessed: 31
256: stonewalling pairs accessed: 32
34: stonewalling pairs accessed: 32
568: stonewalling pairs accessed: 59
127: stonewalling pairs accessed: 32
76: stonewalling pairs accessed: 28
187: stonewalling pairs accessed: 32
328: stonewalling pairs accessed: 35
374: stonewalling pairs accessed: 31
482: stonewalling pairs accessed: 31
460: stonewalling pairs accessed: 33
246: stonewalling pairs accessed: 32
52: stonewalling pairs accessed: 28
587: stonewalling pairs accessed: 68
165: stonewalling pairs accessed: 32
83: stonewalling pairs accessed: 28
238: stonewalling pairs accessed: 32
343: stonewalling pairs accessed: 39
391: stonewalling pairs accessed: 31
492: stonewalling pairs accessed: 50
455: stonewalling pairs accessed: 31
252: stonewalling pairs accessed: 32
19: stonewalling pairs accessed: 32
573: stonewalling pairs accessed: 33
174: stonewalling pairs accessed: 32
66: stonewalling pairs accessed: 28
197: stonewalling pairs accessed: 28
319: stonewalling pairs accessed: 41
395: stonewalling pairs accessed: 28
486: stonewalling pairs accessed: 28
457: stonewalling pairs accessed: 33
286: stonewalling pairs accessed: 32
48: stonewalling pairs accessed: 32
596: stonewalling pairs accessed: 68
149: stonewalling pairs accessed: 32
84: stonewalling pairs accessed: 28
193: stonewalling pairs accessed: 32
320: stonewalling pairs accessed: 39
375: stonewalling pairs accessed: 31
524: stonewalling pairs accessed: 28
440: stonewalling pairs accessed: 31
289: stonewalling pairs accessed: 32
28: stonewalling pairs accessed: 32
556: stonewalling pairs accessed: 59
145: stonewalling pairs accessed: 32
62: stonewalling pairs accessed: 28
222: stonewalling pairs accessed: 32
337: stonewalling pairs accessed: 41
406: stonewalling pairs accessed: 28
490: stonewalling pairs accessed: 31
466: stonewalling pairs accessed: 31
294: stonewalling pairs accessed: 32
4: stonewalling pairs accessed: 32
562: stonewalling pairs accessed: 33
166: stonewalling pairs accessed: 32
60: stonewalling pairs accessed: 28
213: stonewalling pairs accessed: 28
301: stonewalling pairs accessed: 39
401: stonewalling pairs accessed: 31
528: stonewalling pairs accessed: 31
439: stonewalling pairs accessed: 31
268: stonewalling pairs accessed: 32
12: stonewalling pairs accessed: 32
592: stonewalling pairs accessed: 68
173: stonewalling pairs accessed: 32
71: stonewalling pairs accessed: 28
202: stonewalling pairs accessed: 32
321: stonewalling pairs accessed: 39
410: stonewalling pairs accessed: 28
520: stonewalling pairs accessed: 28
428: stonewalling pairs accessed: 31
253: stonewalling pairs accessed: 32
29: stonewalling pairs accessed: 32
554: stonewalling pairs accessed: 33
159: stonewalling pairs accessed: 32
105: stonewalling pairs accessed: 28
223: stonewalling pairs accessed: 32
310: stonewalling pairs accessed: 32
390: stonewalling pairs accessed: 31
491: stonewalling pairs accessed: 31
465: stonewalling pairs accessed: 31
281: stonewalling pairs accessed: 32
5: stonewalling pairs accessed: 32
565: stonewalling pairs accessed: 33
151: stonewalling pairs accessed: 32
72: stonewalling pairs accessed: 28
235: stonewalling pairs accessed: 32
352: stonewalling pairs accessed: 41
381: stonewalling pairs accessed: 31
502: stonewalling pairs accessed: 28
453: stonewalling pairs accessed: 31
296: stonewalling pairs accessed: 32
41: stonewalling pairs accessed: 32
555: stonewalling pairs accessed: 33
158: stonewalling pairs accessed: 32
100: stonewalling pairs accessed: 28
186: stonewalling pairs accessed: 32
341: stonewalling pairs accessed: 41
399: stonewalling pairs accessed: 28
499: stonewalling pairs accessed: 28
431: stonewalling pairs accessed: 31
250: stonewalling pairs accessed: 32
47: stonewalling pairs accessed: 32
575: stonewalling pairs accessed: 33
148: stonewalling pairs accessed: 32
80: stonewalling pairs accessed: 28
234: stonewalling pairs accessed: 32
304: stonewalling pairs accessed: 32
412: stonewalling pairs accessed: 28
512: stonewalling pairs accessed: 28
449: stonewalling pairs accessed: 31
290: stonewalling pairs accessed: 32
22: stonewalling pairs accessed: 32
543: stonewalling pairs accessed: 33
154: stonewalling pairs accessed: 32
89: stonewalling pairs accessed: 28
204: stonewalling pairs accessed: 32
312: stonewalling pairs accessed: 32
405: stonewalling pairs accessed: 28
525: stonewalling pairs accessed: 28
445: stonewalling pairs accessed: 31
260: stonewalling pairs accessed: 32
39: stonewalling pairs accessed: 32
567: stonewalling pairs accessed: 33
155: stonewalling pairs accessed: 32
73: stonewalling pairs accessed: 28
228: stonewalling pairs accessed: 32
313: stonewalling pairs accessed: 32
400: stonewalling pairs accessed: 28
506: stonewalling pairs accessed: 28
452: stonewalling pairs accessed: 31
262: stonewalling pairs accessed: 32
53: stonewalling pairs accessed: 28
544: stonewalling pairs accessed: 33
81: stonewalling pairs accessed: 28
196: stonewalling pairs accessed: 32
513: stonewalling pairs accessed: 28
261: stonewalling pairs accessed: 32
23: stonewalling pairs accessed: 32
194: stonewalling pairs accessed: 32
45: stonewalling pairs accessed: 28
stonewalling pairs accessed min: 28 max: 166602 -- min data: 0.0 GiB mean data: 0.1 GiB time: 300.0s
WARNING: Expected aggregate file size       = 53589120000000.
WARNING: Stat() of aggregate file size      = 4698976089600.
WARNING: Using actual aggregate bytes moved = 4698976089600.
WARNING: maybe caused by deadlineForStonewalling
write     9337       208268     475.30      45.91      45.91      0.002308   479.96     0.000553   479.97     0   
Max Write: 9336.68 MiB/sec (9790.21 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        9336.68    9336.68    9336.68       0.00  208266.98  208266.98  208266.98       0.00  479.96662     300.01        252.41     0    600  60    1   0     1        1         0    0 1900000    47008    47008 4481293.0 POSIX      0
Finished            : Tue Jun  2 09:23:49 2020
mdtest_easy_delete
-- started at 06/02/2020 10:00:45 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-r' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_easy' '-n' '29000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 44.0%   Inodes: 858.3 Mi   Used Inodes: 5.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 17400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      33220.564      33220.534      33220.543          0.008
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.061          2.061          2.061          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        523.772        523.772        523.772          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.485          0.485          0.485          0.000
-- finished at 06/02/2020 10:13:05 --

mdtest_easy_stat
-- started at 06/02/2020 09:42:28 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-T' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_easy' '-n' '29000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 44.0%   Inodes: 858.3 Mi   Used Inodes: 5.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 17400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     201857.104     201856.898     201857.019          0.045
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         86.200         86.200         86.200          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 09:47:17 --

mdtest_easy_write
-- started at 06/02/2020 09:07:23 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-C' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_easy' '-n' '29000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 35.2%   Inodes: 858.3 Mi   Used Inodes: 2.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 17400000 files
Continue stonewall hit min: 424 max: 29000 avg: 19386.4 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      37494.209      37494.189      37494.202          0.005
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      38650.713             NA
   Tree creation             :         18.330         18.330         18.330          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        464.072        464.072        464.072          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.947             NA
   Tree creation             :          0.055          0.055          0.055          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 09:15:47 --

mdtest_hard_delete
-- started at 06/02/2020 10:20:20 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-r' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 43.8%   Inodes: 858.3 Mi   Used Inodes: 3.1%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      14791.778      14791.761      14791.767          0.004
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.322          0.322          0.322          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        315.703        315.702        315.703          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.104          3.104          3.104          0.000
-- finished at 06/02/2020 10:29:58 --

mdtest_hard_read
-- started at 06/02/2020 10:14:05 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 43.8%   Inodes: 858.3 Mi   Used Inodes: 3.1%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      38342.094      38342.058      38342.075          0.009
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        121.793        121.793        121.793          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 10:20:18 --

mdtest_hard_stat
-- started at 06/02/2020 09:55:50 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-T' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 44.0%   Inodes: 858.3 Mi   Used Inodes: 5.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      60207.388      60207.257      60207.287          0.035
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         77.562         77.562         77.562          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 10:00:43 --

mdtest_hard_write
-- started at 06/02/2020 09:23:51 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-C' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-08.56.31-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-08.56.31-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-08.56.31-scr
FS: 50.0 TiB   Used FS: 43.9%   Inodes: 858.3 Mi   Used Inodes: 4.5%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files
Continue stonewall hit min: 7605 max: 7783 avg: 7710.0 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      14285.209      14285.198      14285.204          0.002
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      15417.669             NA
   Tree creation             :        974.513        974.513        974.513          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        326.898        326.898        326.898          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.047             NA
   Tree creation             :          0.001          0.001          0.001          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 09:30:32 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               14.207 GiB/s : time 636.35 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               37.494 kiops : time 464.07 seconds
[RESULT] BW   phase 2            ior_hard_write                9.118 GiB/s : time 479.96 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               14.285 kiops : time 326.90 seconds
[RESULT] IOPS phase 3                      find              288.630 kiops : time  76.47 seconds
[RESULT] BW   phase 3             ior_easy_read               15.661 GiB/s : time 577.35 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              201.857 kiops : time  86.20 seconds
[RESULT] BW   phase 4             ior_hard_read                9.713 GiB/s : time 450.55 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               60.207 kiops : time  77.56 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               33.221 kiops : time 523.77 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               38.342 kiops : time 121.79 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               14.792 kiops : time 579.92 seconds
[SCORE] Bandwidth 11.848 GiB/s : IOPS 49.3883 kiops : TOTAL 24.1899