SharedScratch

Institution Janelia Research Campus, HHMI
Client Procs Per Node
Client Operating System Scientific-Linux
Client Operating System Version 7.7
Client Kernel Version 3.10.0-1062.9.1.el7.x86_64

DATA SERVER

Storage Type Flash
Volatile Memory 192GB
Storage Interface NVMe
Network 100GbE
Software Version 3.7.0
OS Version 7.7

INFORMATION

Client Nodes 10
Client Total Procs 600

METADATA

Easy Write 35.53 kIOP/s
Easy Stat 126.33 kIOP/s
Easy Delete 29.59 kIOP/s
Hard Write 12.84 kIOP/s
Hard Read 29.96 kIOP/s
Hard Stat 39.70 kIOP/s
Hard Delete 13.69 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 600 -ppn 60 -f $PWD/hostfile-10-h09 --bootstrap ssh"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 20:17:51 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -r -R -a POSIX -t 32m --posix.odirect -v -b 6000000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datadir/2020.06.02-19.41.54-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datadir/2020.06.02-19.41.54-scr/ior_easy/stonewall
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 20:17:51 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr/ior_easy
FS                  : 50.0 TiB   Used FS: 64.5%   Inodes: 858.3 Mi   Used Inodes: 5.9%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-19.41.54-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 32 MiB
blocksize           : 5.72 TiB
aggregate filesize  : 3433.23 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Tue Jun  2 20:17:51 2020

WARNING: Expected aggregate file size       = 3774873600000000.
WARNING: Stat() of aggregate file size      = 6321654988800.
WARNING: Using actual aggregate bytes moved = 6321654988800.
read      13267      414.62     0.002423    6144000000 32768      0.008634   454.39     0.002786   454.40     0   
Max Read:  13267.48 MiB/sec (13911.96 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        13267.48   13267.48   13267.48       0.00     414.61     414.61     414.61       0.00  454.40435         NA            NA     0    600  60    1   1     1        1         0    0      1 6291456000000 33554432 6028800.0 POSIX      0
Finished            : Tue Jun  2 20:25:26 2020
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 19:42:09 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -w -a POSIX -t 32m --posix.odirect -v -b 6000000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datadir/2020.06.02-19.41.54-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datadir/2020.06.02-19.41.54-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 19:42:09 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr/ior_easy
FS                  : 50.0 TiB   Used FS: 43.8%   Inodes: 858.3 Mi   Used Inodes: 2.5%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-19.41.54-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 32 MiB
blocksize           : 5.72 TiB
aggregate filesize  : 3433.23 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Tue Jun  2 19:42:09 2020
483: stonewalling pairs accessed: 290
409: stonewalling pairs accessed: 172
329: stonewalling pairs accessed: 185
183: stonewalling pairs accessed: 272
269: stonewalling pairs accessed: 212
164: stonewalling pairs accessed: 212
499: stonewalling pairs accessed: 211
60: stonewalling pairs accessed: 182
56: stonewalling pairs accessed: 231
441: stonewalling pairs accessed: 188
444: stonewalling pairs accessed: 196
16: stonewalling pairs accessed: 215
403: stonewalling pairs accessed: 226
217: stonewalling pairs accessed: 243
157: stonewalling pairs accessed: 247
27: stonewalling pairs accessed: 183
340: stonewalling pairs accessed: 200
271: stonewalling pairs accessed: 240
562: stonewalling pairs accessed: 236
312: stonewalling pairs accessed: 176
47: stonewalling pairs accessed: 219
345: stonewalling pairs accessed: 175
580: stonewalling pairs accessed: 224
360: stonewalling pairs accessed: 244
196: stonewalling pairs accessed: 203
414: stonewalling pairs accessed: 242
48: stonewalling pairs accessed: 193
304: stonewalling pairs accessed: 221
39: stonewalling pairs accessed: 241
289: stonewalling pairs accessed: 229
265: stonewalling pairs accessed: 241
263: stonewalling pairs accessed: 249
428: stonewalling pairs accessed: 227
480: stonewalling pairs accessed: 223
49: stonewalling pairs accessed: 213
195: stonewalling pairs accessed: 201
284: stonewalling pairs accessed: 228
201: stonewalling pairs accessed: 228
98: stonewalling pairs accessed: 208
330: stonewalling pairs accessed: 227
137: stonewalling pairs accessed: 214
437: stonewalling pairs accessed: 218
315: stonewalling pairs accessed: 212
350: stonewalling pairs accessed: 248
358: stonewalling pairs accessed: 230
521: stonewalling pairs accessed: 214
23: stonewalling pairs accessed: 197
143: stonewalling pairs accessed: 232
158: stonewalling pairs accessed: 196
188: stonewalling pairs accessed: 276
185: stonewalling pairs accessed: 237
449: stonewalling pairs accessed: 226
84: stonewalling pairs accessed: 219
243: stonewalling pairs accessed: 254
283: stonewalling pairs accessed: 239
54: stonewalling pairs accessed: 218
576: stonewalling pairs accessed: 259
376: stonewalling pairs accessed: 250
69: stonewalling pairs accessed: 205
165: stonewalling pairs accessed: 233
281: stonewalling pairs accessed: 187
205: stonewalling pairs accessed: 223
502: stonewalling pairs accessed: 233
180: stonewalling pairs accessed: 261
287: stonewalling pairs accessed: 205
259: stonewalling pairs accessed: 208
126: stonewalling pairs accessed: 229
554: stonewalling pairs accessed: 241
89: stonewalling pairs accessed: 214
560: stonewalling pairs accessed: 219
211: stonewalling pairs accessed: 188
139: stonewalling pairs accessed: 233
44: stonewalling pairs accessed: 216
460: stonewalling pairs accessed: 222
333: stonewalling pairs accessed: 183
127: stonewalling pairs accessed: 269
229: stonewalling pairs accessed: 202
584: stonewalling pairs accessed: 206
285: stonewalling pairs accessed: 174
491: stonewalling pairs accessed: 276
34: stonewalling pairs accessed: 188
101: stonewalling pairs accessed: 266
451: stonewalling pairs accessed: 191
191: stonewalling pairs accessed: 223
430: stonewalling pairs accessed: 238
18: stonewalling pairs accessed: 255
130: stonewalling pairs accessed: 223
547: stonewalling pairs accessed: 272
402: stonewalling pairs accessed: 192
328: stonewalling pairs accessed: 230
19: stonewalling pairs accessed: 215
427: stonewalling pairs accessed: 176
38: stonewalling pairs accessed: 239
52: stonewalling pairs accessed: 199
36: stonewalling pairs accessed: 185
418: stonewalling pairs accessed: 230
559: stonewalling pairs accessed: 236
534: stonewalling pairs accessed: 246
369: stonewalling pairs accessed: 225
523: stonewalling pairs accessed: 211
391: stonewalling pairs accessed: 238
214: stonewalling pairs accessed: 178
545: stonewalling pairs accessed: 221
15: stonewalling pairs accessed: 238
518: stonewalling pairs accessed: 228
351: stonewalling pairs accessed: 166
375: stonewalling pairs accessed: 257
522: stonewalling pairs accessed: 185
539: stonewalling pairs accessed: 217
372: stonewalling pairs accessed: 209
121: stonewalling pairs accessed: 226
371: stonewalling pairs accessed: 228
589: stonewalling pairs accessed: 203
221: stonewalling pairs accessed: 235
581: stonewalling pairs accessed: 201
401: stonewalling pairs accessed: 206
543: stonewalling pairs accessed: 245
198: stonewalling pairs accessed: 266
594: stonewalling pairs accessed: 231
564: stonewalling pairs accessed: 236
344: stonewalling pairs accessed: 220
239: stonewalling pairs accessed: 230
339: stonewalling pairs accessed: 222
68: stonewalling pairs accessed: 213
447: stonewalling pairs accessed: 237
218: stonewalling pairs accessed: 202
565: stonewalling pairs accessed: 210
513: stonewalling pairs accessed: 247
270: stonewalling pairs accessed: 230
453: stonewalling pairs accessed: 259
250: stonewalling pairs accessed: 227
2: stonewalling pairs accessed: 252
442: stonewalling pairs accessed: 214
25: stonewalling pairs accessed: 277
43: stonewalling pairs accessed: 195
29: stonewalling pairs accessed: 194
407: stonewalling pairs accessed: 206
509: stonewalling pairs accessed: 235
544: stonewalling pairs accessed: 223
242: stonewalling pairs accessed: 292
17: stonewalling pairs accessed: 223
431: stonewalling pairs accessed: 220
55: stonewalling pairs accessed: 215
395: stonewalling pairs accessed: 173
423: stonewalling pairs accessed: 244
108: stonewalling pairs accessed: 241
425: stonewalling pairs accessed: 236
455: stonewalling pairs accessed: 213
12: stonewalling pairs accessed: 192
8: stonewalling pairs accessed: 290
456: stonewalling pairs accessed: 195
10: stonewalling pairs accessed: 213
532: stonewalling pairs accessed: 225
45: stonewalling pairs accessed: 166
498: stonewalling pairs accessed: 274
177: stonewalling pairs accessed: 206
14: stonewalling pairs accessed: 239
26: stonewalling pairs accessed: 238
6: stonewalling pairs accessed: 254
110: stonewalling pairs accessed: 201
379: stonewalling pairs accessed: 265
587: stonewalling pairs accessed: 233
77: stonewalling pairs accessed: 240
294: stonewalling pairs accessed: 243
458: stonewalling pairs accessed: 267
599: stonewalling pairs accessed: 226
100: stonewalling pairs accessed: 229
478: stonewalling pairs accessed: 200
112: stonewalling pairs accessed: 218
168: stonewalling pairs accessed: 210
397: stonewalling pairs accessed: 209
154: stonewalling pairs accessed: 222
11: stonewalling pairs accessed: 281
79: stonewalling pairs accessed: 227
525: stonewalling pairs accessed: 218
189: stonewalling pairs accessed: 262
471: stonewalling pairs accessed: 183
278: stonewalling pairs accessed: 218
541: stonewalling pairs accessed: 227
58: stonewalling pairs accessed: 197
415: stonewalling pairs accessed: 167
440: stonewalling pairs accessed: 239
332: stonewalling pairs accessed: 232
5: stonewalling pairs accessed: 240
435: stonewalling pairs accessed: 204
325: stonewalling pairs accessed: 240
3: stonewalling pairs accessed: 242
531: stonewalling pairs accessed: 199
207: stonewalling pairs accessed: 217
394: stonewalling pairs accessed: 192
500: stonewalling pairs accessed: 261
450: stonewalling pairs accessed: 203
28: stonewalling pairs accessed: 253
46: stonewalling pairs accessed: 238
459: stonewalling pairs accessed: 213
13: stonewalling pairs accessed: 203
215: stonewalling pairs accessed: 200
475: stonewalling pairs accessed: 238
597: stonewalling pairs accessed: 193
274: stonewalling pairs accessed: 257
51: stonewalling pairs accessed: 228
392: stonewalling pairs accessed: 221
224: stonewalling pairs accessed: 241
493: stonewalling pairs accessed: 207
93: stonewalling pairs accessed: 212
138: stonewalling pairs accessed: 241
467: stonewalling pairs accessed: 203
323: stonewalling pairs accessed: 193
579: stonewalling pairs accessed: 174
253: stonewalling pairs accessed: 238
20: stonewalling pairs accessed: 260
361: stonewalling pairs accessed: 242
233: stonewalling pairs accessed: 191
512: stonewalling pairs accessed: 221
62: stonewalling pairs accessed: 180
125: stonewalling pairs accessed: 221
445: stonewalling pairs accessed: 171
349: stonewalling pairs accessed: 255
578: stonewalling pairs accessed: 213
268: stonewalling pairs accessed: 220
30: stonewalling pairs accessed: 229
373: stonewalling pairs accessed: 205
223: stonewalling pairs accessed: 226
530: stonewalling pairs accessed: 244
65: stonewalling pairs accessed: 221
152: stonewalling pairs accessed: 237
421: stonewalling pairs accessed: 208
318: stonewalling pairs accessed: 239
551: stonewalling pairs accessed: 271
244: stonewalling pairs accessed: 217
22: stonewalling pairs accessed: 272
412: stonewalling pairs accessed: 186
228: stonewalling pairs accessed: 221
516: stonewalling pairs accessed: 229
78: stonewalling pairs accessed: 252
140: stonewalling pairs accessed: 275
465: stonewalling pairs accessed: 232
320: stonewalling pairs accessed: 258
593: stonewalling pairs accessed: 218
254: stonewalling pairs accessed: 274
1: stonewalling pairs accessed: 293
413: stonewalling pairs accessed: 205
194: stonewalling pairs accessed: 233
484: stonewalling pairs accessed: 244
82: stonewalling pairs accessed: 243
160: stonewalling pairs accessed: 203
473: stonewalling pairs accessed: 194
353: stonewalling pairs accessed: 241
583: stonewalling pairs accessed: 247
255: stonewalling pairs accessed: 225
42: stonewalling pairs accessed: 183
381: stonewalling pairs accessed: 233
238: stonewalling pairs accessed: 235
527: stonewalling pairs accessed: 200
104: stonewalling pairs accessed: 204
178: stonewalling pairs accessed: 186
335: stonewalling pairs accessed: 218
577: stonewalling pairs accessed: 214
286: stonewalling pairs accessed: 190
4: stonewalling pairs accessed: 253
378: stonewalling pairs accessed: 211
237: stonewalling pairs accessed: 226
508: stonewalling pairs accessed: 267
80: stonewalling pairs accessed: 261
149: stonewalling pairs accessed: 231
343: stonewalling pairs accessed: 216
566: stonewalling pairs accessed: 206
59: stonewalling pairs accessed: 231
386: stonewalling pairs accessed: 233
236: stonewalling pairs accessed: 199
481: stonewalling pairs accessed: 219
92: stonewalling pairs accessed: 191
303: stonewalling pairs accessed: 283
574: stonewalling pairs accessed: 256
41: stonewalling pairs accessed: 221
410: stonewalling pairs accessed: 223
192: stonewalling pairs accessed: 265
528: stonewalling pairs accessed: 199
117: stonewalling pairs accessed: 195
347: stonewalling pairs accessed: 172
548: stonewalling pairs accessed: 238
387: stonewalling pairs accessed: 245
222: stonewalling pairs accessed: 236
95: stonewalling pairs accessed: 191
338: stonewalling pairs accessed: 202
569: stonewalling pairs accessed: 217
363: stonewalling pairs accessed: 239
202: stonewalling pairs accessed: 196
91: stonewalling pairs accessed: 196
346: stonewalling pairs accessed: 208
570: stonewalling pairs accessed: 247
362: stonewalling pairs accessed: 234
203: stonewalling pairs accessed: 201
515: stonewalling pairs accessed: 200
307: stonewalling pairs accessed: 223
588: stonewalling pairs accessed: 267
247: stonewalling pairs accessed: 215
393: stonewalling pairs accessed: 209
186: stonewalling pairs accessed: 278
316: stonewalling pairs accessed: 262
563: stonewalling pairs accessed: 273
260: stonewalling pairs accessed: 261
417: stonewalling pairs accessed: 176
206: stonewalling pairs accessed: 293
520: stonewalling pairs accessed: 199
568: stonewalling pairs accessed: 232
241: stonewalling pairs accessed: 294
396: stonewalling pairs accessed: 225
204: stonewalling pairs accessed: 201
439: stonewalling pairs accessed: 215
549: stonewalling pairs accessed: 246
384: stonewalling pairs accessed: 212
197: stonewalling pairs accessed: 302
128: stonewalling pairs accessed: 172
479: stonewalling pairs accessed: 194
542: stonewalling pairs accessed: 232
388: stonewalling pairs accessed: 235
216: stonewalling pairs accessed: 215
573: stonewalling pairs accessed: 189
383: stonewalling pairs accessed: 232
230: stonewalling pairs accessed: 229
556: stonewalling pairs accessed: 231
405: stonewalling pairs accessed: 244
232: stonewalling pairs accessed: 197
553: stonewalling pairs accessed: 227
367: stonewalling pairs accessed: 178
182: stonewalling pairs accessed: 218
420: stonewalling pairs accessed: 258
585: stonewalling pairs accessed: 231
365: stonewalling pairs accessed: 260
231: stonewalling pairs accessed: 225
595: stonewalling pairs accessed: 212
399: stonewalling pairs accessed: 230
144: stonewalling pairs accessed: 180
567: stonewalling pairs accessed: 219
374: stonewalling pairs accessed: 241
334: stonewalling pairs accessed: 204
552: stonewalling pairs accessed: 237
190: stonewalling pairs accessed: 224
119: stonewalling pairs accessed: 253
590: stonewalling pairs accessed: 219
557: stonewalling pairs accessed: 229
591: stonewalling pairs accessed: 219
213: stonewalling pairs accessed: 226
153: stonewalling pairs accessed: 194
142: stonewalling pairs accessed: 220
299: stonewalling pairs accessed: 201
135: stonewalling pairs accessed: 244
586: stonewalling pairs accessed: 233
122: stonewalling pairs accessed: 261
504: stonewalling pairs accessed: 248
187: stonewalling pairs accessed: 182
31: stonewalling pairs accessed: 225
572: stonewalling pairs accessed: 160
592: stonewalling pairs accessed: 191
327: stonewalling pairs accessed: 198
212: stonewalling pairs accessed: 225
21: stonewalling pairs accessed: 252
357: stonewalling pairs accessed: 187
461: stonewalling pairs accessed: 184
161: stonewalling pairs accessed: 175
61: stonewalling pairs accessed: 209
496: stonewalling pairs accessed: 232
446: stonewalling pairs accessed: 233
131: stonewalling pairs accessed: 233
234: stonewalling pairs accessed: 196
193: stonewalling pairs accessed: 228
454: stonewalling pairs accessed: 209
159: stonewalling pairs accessed: 189
390: stonewalling pairs accessed: 205
297: stonewalling pairs accessed: 226
466: stonewalling pairs accessed: 179
225: stonewalling pairs accessed: 200
477: stonewalling pairs accessed: 215
322: stonewalling pairs accessed: 209
24: stonewalling pairs accessed: 214
301: stonewalling pairs accessed: 224
219: stonewalling pairs accessed: 200
313: stonewalling pairs accessed: 238
354: stonewalling pairs accessed: 267
33: stonewalling pairs accessed: 232
37: stonewalling pairs accessed: 244
40: stonewalling pairs accessed: 230
74: stonewalling pairs accessed: 219
124: stonewalling pairs accessed: 249
123: stonewalling pairs accessed: 260
317: stonewalling pairs accessed: 248
306: stonewalling pairs accessed: 224
174: stonewalling pairs accessed: 191
50: stonewalling pairs accessed: 240
181: stonewalling pairs accessed: 205
497: stonewalling pairs accessed: 276
264: stonewalling pairs accessed: 213
524: stonewalling pairs accessed: 219
494: stonewalling pairs accessed: 183
300: stonewalling pairs accessed: 221
35: stonewalling pairs accessed: 201
558: stonewalling pairs accessed: 224
277: stonewalling pairs accessed: 197
57: stonewalling pairs accessed: 194
116: stonewalling pairs accessed: 181
296: stonewalling pairs accessed: 223
514: stonewalling pairs accessed: 234
352: stonewalling pairs accessed: 214
291: stonewalling pairs accessed: 250
429: stonewalling pairs accessed: 270
102: stonewalling pairs accessed: 203
485: stonewalling pairs accessed: 224
175: stonewalling pairs accessed: 204
0: stonewalling pairs accessed: 232
538: stonewalling pairs accessed: 314
310: stonewalling pairs accessed: 194
526: stonewalling pairs accessed: 212
155: stonewalling pairs accessed: 196
331: stonewalling pairs accessed: 226
505: stonewalling pairs accessed: 231
510: stonewalling pairs accessed: 196
151: stonewalling pairs accessed: 223
319: stonewalling pairs accessed: 226
293: stonewalling pairs accessed: 207
32: stonewalling pairs accessed: 212
487: stonewalling pairs accessed: 195
464: stonewalling pairs accessed: 180
106: stonewalling pairs accessed: 213
96: stonewalling pairs accessed: 193
424: stonewalling pairs accessed: 241
292: stonewalling pairs accessed: 253
470: stonewalling pairs accessed: 199
53: stonewalling pairs accessed: 220
501: stonewalling pairs accessed: 274
90: stonewalling pairs accessed: 209
426: stonewalling pairs accessed: 253
506: stonewalling pairs accessed: 223
148: stonewalling pairs accessed: 227
503: stonewalling pairs accessed: 236
535: stonewalling pairs accessed: 213
141: stonewalling pairs accessed: 243
288: stonewalling pairs accessed: 242
120: stonewalling pairs accessed: 226
257: stonewalling pairs accessed: 238
438: stonewalling pairs accessed: 222
474: stonewalling pairs accessed: 213
476: stonewalling pairs accessed: 241
305: stonewalling pairs accessed: 224
336: stonewalling pairs accessed: 214
462: stonewalling pairs accessed: 186
276: stonewalling pairs accessed: 222
150: stonewalling pairs accessed: 190
342: stonewalling pairs accessed: 234
83: stonewalling pairs accessed: 210
469: stonewalling pairs accessed: 196
452: stonewalling pairs accessed: 219
266: stonewalling pairs accessed: 216
519: stonewalling pairs accessed: 240
537: stonewalling pairs accessed: 180
302: stonewalling pairs accessed: 213
517: stonewalling pairs accessed: 234
314: stonewalling pairs accessed: 251
448: stonewalling pairs accessed: 260
356: stonewalling pairs accessed: 223
272: stonewalling pairs accessed: 192
482: stonewalling pairs accessed: 248
596: stonewalling pairs accessed: 216
598: stonewalling pairs accessed: 190
507: stonewalling pairs accessed: 206
64: stonewalling pairs accessed: 251
118: stonewalling pairs accessed: 151
184: stonewalling pairs accessed: 200
179: stonewalling pairs accessed: 245
282: stonewalling pairs accessed: 215
172: stonewalling pairs accessed: 214
489: stonewalling pairs accessed: 226
226: stonewalling pairs accessed: 183
147: stonewalling pairs accessed: 240
7: stonewalling pairs accessed: 205
167: stonewalling pairs accessed: 238
540: stonewalling pairs accessed: 223
348: stonewalling pairs accessed: 215
275: stonewalling pairs accessed: 222
324: stonewalling pairs accessed: 212
261: stonewalling pairs accessed: 219
103: stonewalling pairs accessed: 210
406: stonewalling pairs accessed: 182
311: stonewalling pairs accessed: 217
170: stonewalling pairs accessed: 203
251: stonewalling pairs accessed: 193
280: stonewalling pairs accessed: 191
533: stonewalling pairs accessed: 225
321: stonewalling pairs accessed: 280
71: stonewalling pairs accessed: 187
433: stonewalling pairs accessed: 254
398: stonewalling pairs accessed: 258
94: stonewalling pairs accessed: 227
113: stonewalling pairs accessed: 221
267: stonewalling pairs accessed: 213
166: stonewalling pairs accessed: 251
109: stonewalling pairs accessed: 261
359: stonewalling pairs accessed: 225
262: stonewalling pairs accessed: 189
86: stonewalling pairs accessed: 190
245: stonewalling pairs accessed: 242
145: stonewalling pairs accessed: 232
99: stonewalling pairs accessed: 202
422: stonewalling pairs accessed: 188
97: stonewalling pairs accessed: 192
434: stonewalling pairs accessed: 226
76: stonewalling pairs accessed: 232
575: stonewalling pairs accessed: 180
546: stonewalling pairs accessed: 195
561: stonewalling pairs accessed: 232
81: stonewalling pairs accessed: 256
571: stonewalling pairs accessed: 167
582: stonewalling pairs accessed: 236
249: stonewalling pairs accessed: 231
256: stonewalling pairs accessed: 235
298: stonewalling pairs accessed: 225
419: stonewalling pairs accessed: 214
85: stonewalling pairs accessed: 203
246: stonewalling pairs accessed: 263
9: stonewalling pairs accessed: 223
66: stonewalling pairs accessed: 206
210: stonewalling pairs accessed: 201
173: stonewalling pairs accessed: 277
258: stonewalling pairs accessed: 226
129: stonewalling pairs accessed: 244
105: stonewalling pairs accessed: 209
341: stonewalling pairs accessed: 222
443: stonewalling pairs accessed: 234
555: stonewalling pairs accessed: 211
67: stonewalling pairs accessed: 249
472: stonewalling pairs accessed: 176
111: stonewalling pairs accessed: 252
308: stonewalling pairs accessed: 200
240: stonewalling pairs accessed: 237
550: stonewalling pairs accessed: 237
227: stonewalling pairs accessed: 200
146: stonewalling pairs accessed: 263
200: stonewalling pairs accessed: 197
163: stonewalling pairs accessed: 215
457: stonewalling pairs accessed: 219
463: stonewalling pairs accessed: 269
208: stonewalling pairs accessed: 196
235: stonewalling pairs accessed: 274
220: stonewalling pairs accessed: 229
368: stonewalling pairs accessed: 210
382: stonewalling pairs accessed: 185
377: stonewalling pairs accessed: 236
199: stonewalling pairs accessed: 239
408: stonewalling pairs accessed: 219
326: stonewalling pairs accessed: 230
486: stonewalling pairs accessed: 224
87: stonewalling pairs accessed: 241
209: stonewalling pairs accessed: 174
63: stonewalling pairs accessed: 245
416: stonewalling pairs accessed: 213
364: stonewalling pairs accessed: 239
370: stonewalling pairs accessed: 253
366: stonewalling pairs accessed: 228
385: stonewalling pairs accessed: 230
400: stonewalling pairs accessed: 234
176: stonewalling pairs accessed: 195
380: stonewalling pairs accessed: 236
404: stonewalling pairs accessed: 205
488: stonewalling pairs accessed: 259
536: stonewalling pairs accessed: 217
389: stonewalling pairs accessed: 220
248: stonewalling pairs accessed: 231
432: stonewalling pairs accessed: 221
411: stonewalling pairs accessed: 167
355: stonewalling pairs accessed: 221
468: stonewalling pairs accessed: 215
511: stonewalling pairs accessed: 211
88: stonewalling pairs accessed: 252
337: stonewalling pairs accessed: 200
492: stonewalling pairs accessed: 206
490: stonewalling pairs accessed: 286
495: stonewalling pairs accessed: 213
436: stonewalling pairs accessed: 241
309: stonewalling pairs accessed: 271
114: stonewalling pairs accessed: 271
75: stonewalling pairs accessed: 249
529: stonewalling pairs accessed: 202
273: stonewalling pairs accessed: 198
133: stonewalling pairs accessed: 194
169: stonewalling pairs accessed: 148
171: stonewalling pairs accessed: 179
252: stonewalling pairs accessed: 241
162: stonewalling pairs accessed: 181
295: stonewalling pairs accessed: 239
290: stonewalling pairs accessed: 228
70: stonewalling pairs accessed: 187
136: stonewalling pairs accessed: 254
279: stonewalling pairs accessed: 212
132: stonewalling pairs accessed: 244
73: stonewalling pairs accessed: 233
107: stonewalling pairs accessed: 230
156: stonewalling pairs accessed: 239
134: stonewalling pairs accessed: 199
72: stonewalling pairs accessed: 270
115: stonewalling pairs accessed: 197
stonewalling pairs accessed min: 148 max: 314 -- min data: 4.6 GiB mean data: 7.0 GiB time: 300.8s
WARNING: Expected aggregate file size       = 3774873600000000.
WARNING: Stat() of aggregate file size      = 6321654988800.
WARNING: Using actual aggregate bytes moved = 6321654988800.
WARNING: maybe caused by deadlineForStonewalling
write     14620      456.98     0.001997    6144000000 32768      0.079677   412.27     0.020230   412.37     0   
Max Write: 14619.77 MiB/sec (15329.94 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       14619.77   14619.77   14619.77       0.00     456.87     456.87     456.87       0.00  412.37305     300.82      14208.71     0    600  60    1   1     1        1         0    0      1 6291456000000 33554432 6028800.0 POSIX      0
Finished            : Tue Jun  2 19:49:01 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 20:32:17 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -r -R -s 1800000 -a POSIX --posix.odirect -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datadir/2020.06.02-19.41.54-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datadir/2020.06.02-19.41.54-scr/ior_hard/stonewall
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 20:32:17 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr/ior_hard
FS                  : 50.0 TiB   Used FS: 64.5%   Inodes: 858.3 Mi   Used Inodes: 5.9%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-19.41.54-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1800000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 46.17 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Tue Jun  2 20:32:17 2020

WARNING: Expected aggregate file size       = 50768640000000.
WARNING: Stat() of aggregate file size      = 4913840256000.
WARNING: Using actual aggregate bytes moved = 4913840256000.
read      9693       216210     483.46      45.91      45.91      0.002984   483.48     0.000464   483.48     0   
Max Read:  9692.68 MiB/sec (10163.52 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         9692.68    9692.68    9692.68       0.00  216208.22  216208.22  216208.22       0.00  483.47838         NA            NA     0    600  60    1   0     1        1         0    0 1800000    47008    47008 4686203.0 POSIX      0
Finished            : Tue Jun  2 20:40:20 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 19:58:40 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -w -s 1800000 -a POSIX --posix.odirect -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datadir/2020.06.02-19.41.54-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datadir/2020.06.02-19.41.54-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 19:58:40 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr/ior_hard
FS                  : 50.0 TiB   Used FS: 55.4%   Inodes: 858.3 Mi   Used Inodes: 4.8%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-19.41.54-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1800000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 46.17 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Tue Jun  2 19:58:40 2020
598: stonewalling pairs accessed: 174219
119: stonewalling pairs accessed: 174220
599: stonewalling pairs accessed: 25
59: stonewalling pairs accessed: 174220
179: stonewalling pairs accessed: 174220
239: stonewalling pairs accessed: 174220
299: stonewalling pairs accessed: 174220
58: stonewalling pairs accessed: 33
298: stonewalling pairs accessed: 35
359: stonewalling pairs accessed: 174220
178: stonewalling pairs accessed: 36
118: stonewalling pairs accessed: 39
419: stonewalling pairs accessed: 174220
418: stonewalling pairs accessed: 36
538: stonewalling pairs accessed: 174220
479: stonewalling pairs accessed: 174220
238: stonewalling pairs accessed: 37
358: stonewalling pairs accessed: 33
539: stonewalling pairs accessed: 26
478: stonewalling pairs accessed: 36
458: stonewalling pairs accessed: 48
580: stonewalling pairs accessed: 33
109: stonewalling pairs accessed: 39
394: stonewalling pairs accessed: 48
187: stonewalling pairs accessed: 57
268: stonewalling pairs accessed: 42
342: stonewalling pairs accessed: 58
36: stonewalling pairs accessed: 39
504: stonewalling pairs accessed: 25
473: stonewalling pairs accessed: 42
156: stonewalling pairs accessed: 32
581: stonewalling pairs accessed: 30
76: stonewalling pairs accessed: 51
401: stonewalling pairs accessed: 42
229: stonewalling pairs accessed: 33
270: stonewalling pairs accessed: 52
344: stonewalling pairs accessed: 50
46: stonewalling pairs accessed: 53
482: stonewalling pairs accessed: 21
459: stonewalling pairs accessed: 36
126: stonewalling pairs accessed: 57
552: stonewalling pairs accessed: 41
112: stonewalling pairs accessed: 49
405: stonewalling pairs accessed: 50
186: stonewalling pairs accessed: 48
246: stonewalling pairs accessed: 43
301: stonewalling pairs accessed: 35
2: stonewalling pairs accessed: 40
487: stonewalling pairs accessed: 25
470: stonewalling pairs accessed: 53
163: stonewalling pairs accessed: 41
555: stonewalling pairs accessed: 28
65: stonewalling pairs accessed: 56
371: stonewalling pairs accessed: 57
232: stonewalling pairs accessed: 48
250: stonewalling pairs accessed: 41
336: stonewalling pairs accessed: 58
48: stonewalling pairs accessed: 56
531: stonewalling pairs accessed: 22
474: stonewalling pairs accessed: 58
177: stonewalling pairs accessed: 31
540: stonewalling pairs accessed: 32
67: stonewalling pairs accessed: 43
372: stonewalling pairs accessed: 46
190: stonewalling pairs accessed: 56
240: stonewalling pairs accessed: 37
303: stonewalling pairs accessed: 46
52: stonewalling pairs accessed: 62
501: stonewalling pairs accessed: 21
462: stonewalling pairs accessed: 31
136: stonewalling pairs accessed: 57
546: stonewalling pairs accessed: 42
71: stonewalling pairs accessed: 42
396: stonewalling pairs accessed: 46
209: stonewalling pairs accessed: 32
247: stonewalling pairs accessed: 64
324: stonewalling pairs accessed: 36
10: stonewalling pairs accessed: 44
532: stonewalling pairs accessed: 27
467: stonewalling pairs accessed: 54
154: stonewalling pairs accessed: 36
570: stonewalling pairs accessed: 27
86: stonewalling pairs accessed: 30
366: stonewalling pairs accessed: 60
183: stonewalling pairs accessed: 34
256: stonewalling pairs accessed: 58
343: stonewalling pairs accessed: 52
11: stonewalling pairs accessed: 47
537: stonewalling pairs accessed: 26
472: stonewalling pairs accessed: 58
120: stonewalling pairs accessed: 41
576: stonewalling pairs accessed: 33
115: stonewalling pairs accessed: 30
387: stonewalling pairs accessed: 45
205: stonewalling pairs accessed: 31
260: stonewalling pairs accessed: 53
345: stonewalling pairs accessed: 45
37: stonewalling pairs accessed: 46
516: stonewalling pairs accessed: 21
435: stonewalling pairs accessed: 32
134: stonewalling pairs accessed: 42
589: stonewalling pairs accessed: 36
84: stonewalling pairs accessed: 37
403: stonewalling pairs accessed: 51
230: stonewalling pairs accessed: 54
263: stonewalling pairs accessed: 51
306: stonewalling pairs accessed: 53
39: stonewalling pairs accessed: 44
518: stonewalling pairs accessed: 23
451: stonewalling pairs accessed: 49
137: stonewalling pairs accessed: 40
591: stonewalling pairs accessed: 28
87: stonewalling pairs accessed: 43
408: stonewalling pairs accessed: 49
194: stonewalling pairs accessed: 32
276: stonewalling pairs accessed: 50
327: stonewalling pairs accessed: 38
45: stonewalling pairs accessed: 40
519: stonewalling pairs accessed: 20
469: stonewalling pairs accessed: 32
151: stonewalling pairs accessed: 41
579: stonewalling pairs accessed: 43
100: stonewalling pairs accessed: 30
414: stonewalling pairs accessed: 60
217: stonewalling pairs accessed: 42
284: stonewalling pairs accessed: 40
330: stonewalling pairs accessed: 33
49: stonewalling pairs accessed: 37
536: stonewalling pairs accessed: 25
465: stonewalling pairs accessed: 41
125: stonewalling pairs accessed: 34
593: stonewalling pairs accessed: 42
60: stonewalling pairs accessed: 33
395: stonewalling pairs accessed: 32
193: stonewalling pairs accessed: 36
286: stonewalling pairs accessed: 57
350: stonewalling pairs accessed: 58
50: stonewalling pairs accessed: 59
530: stonewalling pairs accessed: 27
428: stonewalling pairs accessed: 31
135: stonewalling pairs accessed: 55
595: stonewalling pairs accessed: 35
74: stonewalling pairs accessed: 43
363: stonewalling pairs accessed: 30
204: stonewalling pairs accessed: 40
290: stonewalling pairs accessed: 55
349: stonewalling pairs accessed: 31
51: stonewalling pairs accessed: 42
494: stonewalling pairs accessed: 30
447: stonewalling pairs accessed: 31
165: stonewalling pairs accessed: 36
542: stonewalling pairs accessed: 33
82: stonewalling pairs accessed: 31
376: stonewalling pairs accessed: 56
237: stonewalling pairs accessed: 31
257: stonewalling pairs accessed: 33
357: stonewalling pairs accessed: 42
1: stonewalling pairs accessed: 54
506: stonewalling pairs accessed: 30
446: stonewalling pairs accessed: 38
142: stonewalling pairs accessed: 32
543: stonewalling pairs accessed: 36
103: stonewalling pairs accessed: 41
380: stonewalling pairs accessed: 53
223: stonewalling pairs accessed: 55
259: stonewalling pairs accessed: 52
318: stonewalling pairs accessed: 30
15: stonewalling pairs accessed: 42
521: stonewalling pairs accessed: 20
466: stonewalling pairs accessed: 46
122: stonewalling pairs accessed: 65
553: stonewalling pairs accessed: 31
90: stonewalling pairs accessed: 42
377: stonewalling pairs accessed: 35
225: stonewalling pairs accessed: 36
279: stonewalling pairs accessed: 47
332: stonewalling pairs accessed: 41
8: stonewalling pairs accessed: 52
533: stonewalling pairs accessed: 30
461: stonewalling pairs accessed: 32
123: stonewalling pairs accessed: 32
556: stonewalling pairs accessed: 36
98: stonewalling pairs accessed: 31
416: stonewalling pairs accessed: 62
236: stonewalling pairs accessed: 31
282: stonewalling pairs accessed: 48
337: stonewalling pairs accessed: 35
9: stonewalling pairs accessed: 48
486: stonewalling pairs accessed: 23
421: stonewalling pairs accessed: 32
130: stonewalling pairs accessed: 63
571: stonewalling pairs accessed: 44
113: stonewalling pairs accessed: 53
360: stonewalling pairs accessed: 31
207: stonewalling pairs accessed: 31
287: stonewalling pairs accessed: 34
311: stonewalling pairs accessed: 36
23: stonewalling pairs accessed: 33
502: stonewalling pairs accessed: 28
443: stonewalling pairs accessed: 32
131: stonewalling pairs accessed: 39
584: stonewalling pairs accessed: 45
94: stonewalling pairs accessed: 41
397: stonewalling pairs accessed: 30
195: stonewalling pairs accessed: 46
292: stonewalling pairs accessed: 64
353: stonewalling pairs accessed: 36
42: stonewalling pairs accessed: 49
527: stonewalling pairs accessed: 28
444: stonewalling pairs accessed: 31
148: stonewalling pairs accessed: 37
585: stonewalling pairs accessed: 38
95: stonewalling pairs accessed: 30
411: stonewalling pairs accessed: 62
197: stonewalling pairs accessed: 31
294: stonewalling pairs accessed: 47
314: stonewalling pairs accessed: 32
55: stonewalling pairs accessed: 35
505: stonewalling pairs accessed: 20
420: stonewalling pairs accessed: 30
153: stonewalling pairs accessed: 35
588: stonewalling pairs accessed: 45
106: stonewalling pairs accessed: 35
370: stonewalling pairs accessed: 42
210: stonewalling pairs accessed: 31
261: stonewalling pairs accessed: 38
356: stonewalling pairs accessed: 38
3: stonewalling pairs accessed: 37
499: stonewalling pairs accessed: 23
429: stonewalling pairs accessed: 30
157: stonewalling pairs accessed: 40
592: stonewalling pairs accessed: 48
114: stonewalling pairs accessed: 37
407: stonewalling pairs accessed: 35
227: stonewalling pairs accessed: 32
295: stonewalling pairs accessed: 70
338: stonewalling pairs accessed: 53
25: stonewalling pairs accessed: 44
508: stonewalling pairs accessed: 20
440: stonewalling pairs accessed: 38
168: stonewalling pairs accessed: 31
596: stonewalling pairs accessed: 34
70: stonewalling pairs accessed: 37
417: stonewalling pairs accessed: 35
213: stonewalling pairs accessed: 31
254: stonewalling pairs accessed: 36
322: stonewalling pairs accessed: 37
20: stonewalling pairs accessed: 37
534: stonewalling pairs accessed: 39
441: stonewalling pairs accessed: 33
175: stonewalling pairs accessed: 69
597: stonewalling pairs accessed: 31
75: stonewalling pairs accessed: 32
368: stonewalling pairs accessed: 41
206: stonewalling pairs accessed: 40
241: stonewalling pairs accessed: 68
326: stonewalling pairs accessed: 40
33: stonewalling pairs accessed: 32
488: stonewalling pairs accessed: 28
454: stonewalling pairs accessed: 31
176: stonewalling pairs accessed: 45
582: stonewalling pairs accessed: 30
83: stonewalling pairs accessed: 31
415: stonewalling pairs accessed: 42
231: stonewalling pairs accessed: 63
266: stonewalling pairs accessed: 36
334: stonewalling pairs accessed: 35
13: stonewalling pairs accessed: 52
517: stonewalling pairs accessed: 28
455: stonewalling pairs accessed: 31
128: stonewalling pairs accessed: 63
541: stonewalling pairs accessed: 31
89: stonewalling pairs accessed: 30
413: stonewalling pairs accessed: 34
212: stonewalling pairs accessed: 31
293: stonewalling pairs accessed: 36
341: stonewalling pairs accessed: 54
21: stonewalling pairs accessed: 31
525: stonewalling pairs accessed: 22
464: stonewalling pairs accessed: 53
155: stonewalling pairs accessed: 40
544: stonewalling pairs accessed: 53
102: stonewalling pairs accessed: 32
390: stonewalling pairs accessed: 35
216: stonewalling pairs accessed: 32
242: stonewalling pairs accessed: 36
317: stonewalling pairs accessed: 41
35: stonewalling pairs accessed: 45
493: stonewalling pairs accessed: 20
475: stonewalling pairs accessed: 35
132: stonewalling pairs accessed: 37
551: stonewalling pairs accessed: 28
108: stonewalling pairs accessed: 45
412: stonewalling pairs accessed: 60
228: stonewalling pairs accessed: 63
269: stonewalling pairs accessed: 31
321: stonewalling pairs accessed: 35
54: stonewalling pairs accessed: 65
526: stonewalling pairs accessed: 23
430: stonewalling pairs accessed: 41
146: stonewalling pairs accessed: 45
566: stonewalling pairs accessed: 39
62: stonewalling pairs accessed: 32
381: stonewalling pairs accessed: 38
215: stonewalling pairs accessed: 37
280: stonewalling pairs accessed: 41
328: stonewalling pairs accessed: 38
31: stonewalling pairs accessed: 47
529: stonewalling pairs accessed: 20
453: stonewalling pairs accessed: 53
139: stonewalling pairs accessed: 32
568: stonewalling pairs accessed: 43
69: stonewalling pairs accessed: 30
400: stonewalling pairs accessed: 30
233: stonewalling pairs accessed: 37
281: stonewalling pairs accessed: 53
340: stonewalling pairs accessed: 30
22: stonewalling pairs accessed: 32
481: stonewalling pairs accessed: 38
463: stonewalling pairs accessed: 31
124: stonewalling pairs accessed: 34
574: stonewalling pairs accessed: 27
96: stonewalling pairs accessed: 31
404: stonewalling pairs accessed: 38
202: stonewalling pairs accessed: 31
291: stonewalling pairs accessed: 32
315: stonewalling pairs accessed: 38
41: stonewalling pairs accessed: 30
500: stonewalling pairs accessed: 28
471: stonewalling pairs accessed: 42
160: stonewalling pairs accessed: 34
577: stonewalling pairs accessed: 27
97: stonewalling pairs accessed: 37
409: stonewalling pairs accessed: 41
200: stonewalling pairs accessed: 32
273: stonewalling pairs accessed: 40
351: stonewalling pairs accessed: 41
57: stonewalling pairs accessed: 59
524: stonewalling pairs accessed: 20
436: stonewalling pairs accessed: 41
166: stonewalling pairs accessed: 37
578: stonewalling pairs accessed: 31
101: stonewalling pairs accessed: 31
383: stonewalling pairs accessed: 46
185: stonewalling pairs accessed: 39
243: stonewalling pairs accessed: 37
316: stonewalling pairs accessed: 40
17: stonewalling pairs accessed: 45
495: stonewalling pairs accessed: 23
448: stonewalling pairs accessed: 31
129: stonewalling pairs accessed: 45
547: stonewalling pairs accessed: 53
111: stonewalling pairs accessed: 31
393: stonewalling pairs accessed: 37
219: stonewalling pairs accessed: 40
248: stonewalling pairs accessed: 40
302: stonewalling pairs accessed: 32
47: stonewalling pairs accessed: 37
498: stonewalling pairs accessed: 32
438: stonewalling pairs accessed: 41
172: stonewalling pairs accessed: 37
554: stonewalling pairs accessed: 31
116: stonewalling pairs accessed: 31
402: stonewalling pairs accessed: 42
224: stonewalling pairs accessed: 37
251: stonewalling pairs accessed: 32
320: stonewalling pairs accessed: 38
5: stonewalling pairs accessed: 48
503: stonewalling pairs accessed: 20
449: stonewalling pairs accessed: 31
159: stonewalling pairs accessed: 32
558: stonewalling pairs accessed: 36
79: stonewalling pairs accessed: 45
406: stonewalling pairs accessed: 54
182: stonewalling pairs accessed: 37
289: stonewalling pairs accessed: 38
354: stonewalling pairs accessed: 41
28: stonewalling pairs accessed: 34
511: stonewalling pairs accessed: 28
456: stonewalling pairs accessed: 42
171: stonewalling pairs accessed: 45
560: stonewalling pairs accessed: 27
99: stonewalling pairs accessed: 31
374: stonewalling pairs accessed: 38
199: stonewalling pairs accessed: 40
253: stonewalling pairs accessed: 57
323: stonewalling pairs accessed: 30
34: stonewalling pairs accessed: 32
513: stonewalling pairs accessed: 20
468: stonewalling pairs accessed: 35
173: stonewalling pairs accessed: 32
561: stonewalling pairs accessed: 27
68: stonewalling pairs accessed: 49
388: stonewalling pairs accessed: 35
218: stonewalling pairs accessed: 32
274: stonewalling pairs accessed: 32
313: stonewalling pairs accessed: 35
30: stonewalling pairs accessed: 37
480: stonewalling pairs accessed: 28
434: stonewalling pairs accessed: 41
161: stonewalling pairs accessed: 40
562: stonewalling pairs accessed: 27
80: stonewalling pairs accessed: 41
382: stonewalling pairs accessed: 55
181: stonewalling pairs accessed: 32
288: stonewalling pairs accessed: 37
325: stonewalling pairs accessed: 35
12: stonewalling pairs accessed: 37
520: stonewalling pairs accessed: 20
427: stonewalling pairs accessed: 41
149: stonewalling pairs accessed: 45
563: stonewalling pairs accessed: 27
85: stonewalling pairs accessed: 30
398: stonewalling pairs accessed: 35
201: stonewalling pairs accessed: 40
265: stonewalling pairs accessed: 56
329: stonewalling pairs accessed: 38
38: stonewalling pairs accessed: 45
490: stonewalling pairs accessed: 28
450: stonewalling pairs accessed: 42
164: stonewalling pairs accessed: 37
567: stonewalling pairs accessed: 46
91: stonewalling pairs accessed: 34
410: stonewalling pairs accessed: 54
234: stonewalling pairs accessed: 32
244: stonewalling pairs accessed: 32
346: stonewalling pairs accessed: 41
18: stonewalling pairs accessed: 45
507: stonewalling pairs accessed: 20
452: stonewalling pairs accessed: 31
121: stonewalling pairs accessed: 37
575: stonewalling pairs accessed: 29
93: stonewalling pairs accessed: 31
364: stonewalling pairs accessed: 30
180: stonewalling pairs accessed: 32
296: stonewalling pairs accessed: 37
310: stonewalling pairs accessed: 30
32: stonewalling pairs accessed: 37
496: stonewalling pairs accessed: 28
457: stonewalling pairs accessed: 35
145: stonewalling pairs accessed: 37
583: stonewalling pairs accessed: 29
105: stonewalling pairs accessed: 45
367: stonewalling pairs accessed: 35
211: stonewalling pairs accessed: 40
249: stonewalling pairs accessed: 40
352: stonewalling pairs accessed: 30
43: stonewalling pairs accessed: 37
515: stonewalling pairs accessed: 20
477: stonewalling pairs accessed: 35
167: stonewalling pairs accessed: 45
569: stonewalling pairs accessed: 27
64: stonewalling pairs accessed: 45
391: stonewalling pairs accessed: 30
214: stonewalling pairs accessed: 40
297: stonewalling pairs accessed: 37
312: stonewalling pairs accessed: 40
6: stonewalling pairs accessed: 37
484: stonewalling pairs accessed: 23
445: stonewalling pairs accessed: 30
144: stonewalling pairs accessed: 32
548: stonewalling pairs accessed: 42
81: stonewalling pairs accessed: 30
386: stonewalling pairs accessed: 35
222: stonewalling pairs accessed: 32
285: stonewalling pairs accessed: 32
355: stonewalling pairs accessed: 30
26: stonewalling pairs accessed: 37
509: stonewalling pairs accessed: 20
476: stonewalling pairs accessed: 35
170: stonewalling pairs accessed: 37
557: stonewalling pairs accessed: 31
92: stonewalling pairs accessed: 31
378: stonewalling pairs accessed: 30
188: stonewalling pairs accessed: 45
278: stonewalling pairs accessed: 30
305: stonewalling pairs accessed: 32
14: stonewalling pairs accessed: 45
514: stonewalling pairs accessed: 20
460: stonewalling pairs accessed: 35
133: stonewalling pairs accessed: 37
572: stonewalling pairs accessed: 28
63: stonewalling pairs accessed: 32
379: stonewalling pairs accessed: 30
221: stonewalling pairs accessed: 40
255: stonewalling pairs accessed: 32
319: stonewalling pairs accessed: 30
44: stonewalling pairs accessed: 37
528: stonewalling pairs accessed: 23
437: stonewalling pairs accessed: 30
150: stonewalling pairs accessed: 45
590: stonewalling pairs accessed: 29
110: stonewalling pairs accessed: 32
365: stonewalling pairs accessed: 30
192: stonewalling pairs accessed: 32
271: stonewalling pairs accessed: 32
339: stonewalling pairs accessed: 30
24: stonewalling pairs accessed: 37
492: stonewalling pairs accessed: 23
442: stonewalling pairs accessed: 30
162: stonewalling pairs accessed: 37
564: stonewalling pairs accessed: 31
117: stonewalling pairs accessed: 32
384: stonewalling pairs accessed: 30
208: stonewalling pairs accessed: 40
272: stonewalling pairs accessed: 30
308: stonewalling pairs accessed: 30
27: stonewalling pairs accessed: 45
497: stonewalling pairs accessed: 28
439: stonewalling pairs accessed: 30
138: stonewalling pairs accessed: 37
587: stonewalling pairs accessed: 31
73: stonewalling pairs accessed: 30
362: stonewalling pairs accessed: 30
220: stonewalling pairs accessed: 32
277: stonewalling pairs accessed: 30
348: stonewalling pairs accessed: 30
19: stonewalling pairs accessed: 45
512: stonewalling pairs accessed: 20
425: stonewalling pairs accessed: 30
143: stonewalling pairs accessed: 32
586: stonewalling pairs accessed: 49
104: stonewalling pairs accessed: 37
399: stonewalling pairs accessed: 30
203: stonewalling pairs accessed: 40
245: stonewalling pairs accessed: 32
333: stonewalling pairs accessed: 30
40: stonewalling pairs accessed: 37
510: stonewalling pairs accessed: 23
423: stonewalling pairs accessed: 30
174: stonewalling pairs accessed: 32
549: stonewalling pairs accessed: 28
72: stonewalling pairs accessed: 32
373: stonewalling pairs accessed: 30
235: stonewalling pairs accessed: 32
262: stonewalling pairs accessed: 32
309: stonewalling pairs accessed: 30
29: stonewalling pairs accessed: 45
535: stonewalling pairs accessed: 28
431: stonewalling pairs accessed: 30
127: stonewalling pairs accessed: 37
565: stonewalling pairs accessed: 28
78: stonewalling pairs accessed: 37
389: stonewalling pairs accessed: 30
191: stonewalling pairs accessed: 32
283: stonewalling pairs accessed: 32
307: stonewalling pairs accessed: 32
16: stonewalling pairs accessed: 37
522: stonewalling pairs accessed: 23
433: stonewalling pairs accessed: 30
158: stonewalling pairs accessed: 32
545: stonewalling pairs accessed: 42
61: stonewalling pairs accessed: 37
361: stonewalling pairs accessed: 30
226: stonewalling pairs accessed: 32
267: stonewalling pairs accessed: 32
331: stonewalling pairs accessed: 30
4: stonewalling pairs accessed: 37
491: stonewalling pairs accessed: 23
424: stonewalling pairs accessed: 30
147: stonewalling pairs accessed: 37
550: stonewalling pairs accessed: 28
107: stonewalling pairs accessed: 37
385: stonewalling pairs accessed: 30
189: stonewalling pairs accessed: 32
275: stonewalling pairs accessed: 32
335: stonewalling pairs accessed: 30
53: stonewalling pairs accessed: 37
483: stonewalling pairs accessed: 23
422: stonewalling pairs accessed: 30
169: stonewalling pairs accessed: 32
559: stonewalling pairs accessed: 28
66: stonewalling pairs accessed: 37
369: stonewalling pairs accessed: 30
196: stonewalling pairs accessed: 32
258: stonewalling pairs accessed: 32
347: stonewalling pairs accessed: 30
0: stonewalling pairs accessed: 37
523: stonewalling pairs accessed: 23
432: stonewalling pairs accessed: 30
141: stonewalling pairs accessed: 32
573: stonewalling pairs accessed: 28
88: stonewalling pairs accessed: 30
392: stonewalling pairs accessed: 30
184: stonewalling pairs accessed: 32
252: stonewalling pairs accessed: 32
304: stonewalling pairs accessed: 32
56: stonewalling pairs accessed: 37
485: stonewalling pairs accessed: 23
426: stonewalling pairs accessed: 30
140: stonewalling pairs accessed: 32
594: stonewalling pairs accessed: 36
77: stonewalling pairs accessed: 37
375: stonewalling pairs accessed: 30
198: stonewalling pairs accessed: 32
264: stonewalling pairs accessed: 32
300: stonewalling pairs accessed: 32
7: stonewalling pairs accessed: 37
489: stonewalling pairs accessed: 23
152: stonewalling pairs accessed: 32
stonewalling pairs accessed min: 20 max: 174220 -- min data: 0.0 GiB mean data: 0.1 GiB time: 300.0s
WARNING: Expected aggregate file size       = 50768640000000.
WARNING: Stat() of aggregate file size      = 4913840256000.
WARNING: Using actual aggregate bytes moved = 4913840256000.
WARNING: maybe caused by deadlineForStonewalling
write     9439       210541     485.29      45.91      45.91      0.002436   496.49     0.000478   496.49     0   
Max Write: 9438.58 MiB/sec (9897.06 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        9438.58    9438.58    9438.58       0.00  210539.98  210539.98  210539.98       0.00  496.49477     300.01        263.60     0    600  60    1   0     1        1         0    0 1800000    47008    47008 4686203.0 POSIX      0
Finished            : Tue Jun  2 20:06:56 2020
mdtest_easy_delete
-- started at 06/02/2020 20:47:29 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-r' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_easy' '-n' '24000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.5%   Inodes: 858.3 Mi   Used Inodes: 5.9%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 14400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      29588.997      29588.989      29588.994          0.002
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.069          2.069          2.069          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        486.668        486.667        486.667          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.483          0.483          0.483          0.000
-- finished at 06/02/2020 21:00:41 --

mdtest_easy_stat
-- started at 06/02/2020 20:25:28 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-T' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_easy' '-n' '24000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.5%   Inodes: 858.3 Mi   Used Inodes: 5.9%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 14400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     126328.137     126327.979     126328.025          0.039
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        113.989        113.989        113.989          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 20:32:15 --

mdtest_easy_write
-- started at 06/02/2020 19:49:04 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-C' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_easy' '-n' '24000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 55.3%   Inodes: 858.3 Mi   Used Inodes: 3.2%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 14400000 files
Continue stonewall hit min: 536 max: 24000 avg: 17732.5 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      35526.266      35526.257      35526.262          0.002
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      35358.436             NA
   Tree creation             :         18.391         18.391         18.391          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        405.334        405.334        405.334          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.903             NA
   Tree creation             :          0.054          0.054          0.054          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 19:58:38 --

mdtest_hard_delete
-- started at 06/02/2020 21:09:03 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-r' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.3%   Inodes: 858.3 Mi   Used Inodes: 4.3%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      13684.747      13684.743      13684.745          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.467          0.467          0.467          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        347.599        347.599        347.599          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.141          2.141          2.141          0.000
-- finished at 06/02/2020 21:20:40 --

mdtest_hard_read
-- started at 06/02/2020 21:00:43 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.3%   Inodes: 858.3 Mi   Used Inodes: 4.3%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      29964.351      29964.336      29964.345          0.004
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        158.749        158.749        158.749          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 21:09:00 --

mdtest_hard_stat
-- started at 06/02/2020 20:40:23 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-T' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.5%   Inodes: 858.3 Mi   Used Inodes: 5.9%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      39700.589      39700.566      39700.578          0.005
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        119.817        119.817        119.817          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 20:47:27 --

mdtest_hard_write
-- started at 06/02/2020 20:06:59 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-C' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.4%   Inodes: 858.3 Mi   Used Inodes: 5.4%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files
Continue stonewall hit min: 7755 max: 7928 avg: 7854.9 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      12843.527      12843.524      12843.526          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      15707.899             NA
   Tree creation             :        791.229        791.229        791.229          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        370.366        370.366        370.366          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.037             NA
   Tree creation             :          0.001          0.001          0.001          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 20:16:33 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               14.277 GiB/s : time 412.27 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               35.526 kiops : time 405.33 seconds
[RESULT] BW   phase 2            ior_hard_write                9.218 GiB/s : time 496.49 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               12.843 kiops : time 370.37 seconds
[RESULT] IOPS phase 3                      find              251.560 kiops : time  76.15 seconds
[RESULT] BW   phase 3             ior_easy_read               12.956 GiB/s : time 454.39 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              126.328 kiops : time 113.99 seconds
[RESULT] BW   phase 4             ior_hard_read                9.466 GiB/s : time 483.48 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               39.701 kiops : time 119.82 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               29.589 kiops : time 486.67 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               29.964 kiops : time 158.75 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               13.685 kiops : time 699.21 seconds
[SCORE] Bandwidth 11.2714 GiB/s : IOPS 40.3198 kiops : TOTAL 21.3181