SharedScratch

Institution Janelia Research Campus, HHMI
Client Procs Per Node
Client Operating System Scientific-Linux
Client Operating System Version 7.7
Client Kernel Version 3.10.0-1062.9.1.el7.x86_64

DATA SERVER

Storage Type Flash
Volatile Memory 192GB
Storage Interface NVMe
Network 100GbE
Software Version 3.7.0
OS Version 7.7

INFORMATION

Client Nodes 10
Client Total Procs 600
Metadata Nodes 0
Metadata Storage Devices 0
Data Nodes 20
Data Storage Devices 80

METADATA

Easy Write 35.53 kIOP/s
Easy Stat 126.33 kIOP/s
Easy Delete 29.59 kIOP/s
Hard Write 12.84 kIOP/s
Hard Read 29.96 kIOP/s
Hard Stat 39.70 kIOP/s
Hard Delete 13.69 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 600 -ppn 60 -f $PWD/hostfile-10-h09 --bootstrap ssh"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 20:17:51 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -r -R -a POSIX -t 32m --posix.odirect -v -b 6000000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datadir/2020.06.02-19.41.54-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datadir/2020.06.02-19.41.54-scr/ior_easy/stonewall
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 20:17:51 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr/ior_easy
FS                  : 50.0 TiB   Used FS: 64.5%   Inodes: 858.3 Mi   Used Inodes: 5.9%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-19.41.54-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 32 MiB
blocksize           : 5.72 TiB
aggregate filesize  : 3433.23 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Tue Jun  2 20:17:51 2020

WARNING: Expected aggregate file size       = 3774873600000000.
WARNING: Stat() of aggregate file size      = 6321654988800.
WARNING: Using actual aggregate bytes moved = 6321654988800.
read      13267      414.62     0.002423    6144000000 32768      0.008634   454.39     0.002786   454.40     0   
Max Read:  13267.48 MiB/sec (13911.96 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        13267.48   13267.48   13267.48       0.00     414.61     414.61     414.61       0.00  454.40435         NA            NA     0    600  60    1   1     1        1         0    0      1 6291456000000 33554432 6028800.0 POSIX      0
Finished            : Tue Jun  2 20:25:26 2020
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 19:42:09 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -w -a POSIX -t 32m --posix.odirect -v -b 6000000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datadir/2020.06.02-19.41.54-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datadir/2020.06.02-19.41.54-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 19:42:09 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr/ior_easy
FS                  : 50.0 TiB   Used FS: 43.8%   Inodes: 858.3 Mi   Used Inodes: 2.5%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-19.41.54-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 32 MiB
blocksize           : 5.72 TiB
aggregate filesize  : 3433.23 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Tue Jun  2 19:42:09 2020
483: stonewalling pairs accessed: 290
409: stonewalling pairs accessed: 172
329: stonewalling pairs accessed: 185
183: stonewalling pairs accessed: 272
269: stonewalling pairs accessed: 212
164: stonewalling pairs accessed: 212
499: stonewalling pairs accessed: 211
60: stonewalling pairs accessed: 182
56: stonewalling pairs accessed: 231
441: stonewalling pairs accessed: 188
444: stonewalling pairs accessed: 196
16: stonewalling pairs accessed: 215
403: stonewalling pairs accessed: 226
217: stonewalling pairs accessed: 243
157: stonewalling pairs accessed: 247
27: stonewalling pairs accessed: 183
340: stonewalling pairs accessed: 200
271: stonewalling pairs accessed: 240
562: stonewalling pairs accessed: 236
312: stonewalling pairs accessed: 176
47: stonewalling pairs accessed: 219
345: stonewalling pairs accessed: 175
580: stonewalling pairs accessed: 224
360: stonewalling pairs accessed: 244
196: stonewalling pairs accessed: 203
414: stonewalling pairs accessed: 242
48: stonewalling pairs accessed: 193
304: stonewalling pairs accessed: 221
39: stonewalling pairs accessed: 241
289: stonewalling pairs accessed: 229
265: stonewalling pairs accessed: 241
263: stonewalling pairs accessed: 249
428: stonewalling pairs accessed: 227
480: stonewalling pairs accessed: 223
49: stonewalling pairs accessed: 213
195: stonewalling pairs accessed: 201
284: stonewalling pairs accessed: 228
201: stonewalling pairs accessed: 228
98: stonewalling pairs accessed: 208
330: stonewalling pairs accessed: 227
137: stonewalling pairs accessed: 214
437: stonewalling pairs accessed: 218
315: stonewalling pairs accessed: 212
350: stonewalling pairs accessed: 248
358: stonewalling pairs accessed: 230
521: stonewalling pairs accessed: 214
23: stonewalling pairs accessed: 197
143: stonewalling pairs accessed: 232
158: stonewalling pairs accessed: 196
188: stonewalling pairs accessed: 276
185: stonewalling pairs accessed: 237
449: stonewalling pairs accessed: 226
84: stonewalling pairs accessed: 219
243: stonewalling pairs accessed: 254
283: stonewalling pairs accessed: 239
54: stonewalling pairs accessed: 218
576: stonewalling pairs accessed: 259
376: stonewalling pairs accessed: 250
69: stonewalling pairs accessed: 205
165: stonewalling pairs accessed: 233
281: stonewalling pairs accessed: 187
205: stonewalling pairs accessed: 223
502: stonewalling pairs accessed: 233
180: stonewalling pairs accessed: 261
287: stonewalling pairs accessed: 205
259: stonewalling pairs accessed: 208
126: stonewalling pairs accessed: 229
554: stonewalling pairs accessed: 241
89: stonewalling pairs accessed: 214
560: stonewalling pairs accessed: 219
211: stonewalling pairs accessed: 188
139: stonewalling pairs accessed: 233
44: stonewalling pairs accessed: 216
460: stonewalling pairs accessed: 222
333: stonewalling pairs accessed: 183
127: stonewalling pairs accessed: 269
229: stonewalling pairs accessed: 202
584: stonewalling pairs accessed: 206
285: stonewalling pairs accessed: 174
491: stonewalling pairs accessed: 276
34: stonewalling pairs accessed: 188
101: stonewalling pairs accessed: 266
451: stonewalling pairs accessed: 191
191: stonewalling pairs accessed: 223
430: stonewalling pairs accessed: 238
18: stonewalling pairs accessed: 255
130: stonewalling pairs accessed: 223
547: stonewalling pairs accessed: 272
402: stonewalling pairs accessed: 192
328: stonewalling pairs accessed: 230
19: stonewalling pairs accessed: 215
427: stonewalling pairs accessed: 176
38: stonewalling pairs accessed: 239
52: stonewalling pairs accessed: 199
36: stonewalling pairs accessed: 185
418: stonewalling pairs accessed: 230
559: stonewalling pairs accessed: 236
534: stonewalling pairs accessed: 246
369: stonewalling pairs accessed: 225
523: stonewalling pairs accessed: 211
391: stonewalling pairs accessed: 238
214: stonewalling pairs accessed: 178
545: stonewalling pairs accessed: 221
15: stonewalling pairs accessed: 238
518: stonewalling pairs accessed: 228
351: stonewalling pairs accessed: 166
375: stonewalling pairs accessed: 257
522: stonewalling pairs accessed: 185
539: stonewalling pairs accessed: 217
372: stonewalling pairs accessed: 209
121: stonewalling pairs accessed: 226
371: stonewalling pairs accessed: 228
589: stonewalling pairs accessed: 203
221: stonewalling pairs accessed: 235
581: stonewalling pairs accessed: 201
401: stonewalling pairs accessed: 206
543: stonewalling pairs accessed: 245
198: stonewalling pairs accessed: 266
594: stonewalling pairs accessed: 231
564: stonewalling pairs accessed: 236
344: stonewalling pairs accessed: 220
239: stonewalling pairs accessed: 230
339: stonewalling pairs accessed: 222
68: stonewalling pairs accessed: 213
447: stonewalling pairs accessed: 237
218: stonewalling pairs accessed: 202
565: stonewalling pairs accessed: 210
513: stonewalling pairs accessed: 247
270: stonewalling pairs accessed: 230
453: stonewalling pairs accessed: 259
250: stonewalling pairs accessed: 227
2: stonewalling pairs accessed: 252
442: stonewalling pairs accessed: 214
25: stonewalling pairs accessed: 277
43: stonewalling pairs accessed: 195
29: stonewalling pairs accessed: 194
407: stonewalling pairs accessed: 206
509: stonewalling pairs accessed: 235
544: stonewalling pairs accessed: 223
242: stonewalling pairs accessed: 292
17: stonewalling pairs accessed: 223
431: stonewalling pairs accessed: 220
55: stonewalling pairs accessed: 215
395: stonewalling pairs accessed: 173
423: stonewalling pairs accessed: 244
108: stonewalling pairs accessed: 241
425: stonewalling pairs accessed: 236
455: stonewalling pairs accessed: 213
12: stonewalling pairs accessed: 192
8: stonewalling pairs accessed: 290
456: stonewalling pairs accessed: 195
10: stonewalling pairs accessed: 213
532: stonewalling pairs accessed: 225
45: stonewalling pairs accessed: 166
498: stonewalling pairs accessed: 274
177: stonewalling pairs accessed: 206
14: stonewalling pairs accessed: 239
26: stonewalling pairs accessed: 238
6: stonewalling pairs accessed: 254
110: stonewalling pairs accessed: 201
379: stonewalling pairs accessed: 265
587: stonewalling pairs accessed: 233
77: stonewalling pairs accessed: 240
294: stonewalling pairs accessed: 243
458: stonewalling pairs accessed: 267
599: stonewalling pairs accessed: 226
100: stonewalling pairs accessed: 229
478: stonewalling pairs accessed: 200
112: stonewalling pairs accessed: 218
168: stonewalling pairs accessed: 210
397: stonewalling pairs accessed: 209
154: stonewalling pairs accessed: 222
11: stonewalling pairs accessed: 281
79: stonewalling pairs accessed: 227
525: stonewalling pairs accessed: 218
189: stonewalling pairs accessed: 262
471: stonewalling pairs accessed: 183
278: stonewalling pairs accessed: 218
541: stonewalling pairs accessed: 227
58: stonewalling pairs accessed: 197
415: stonewalling pairs accessed: 167
440: stonewalling pairs accessed: 239
332: stonewalling pairs accessed: 232
5: stonewalling pairs accessed: 240
435: stonewalling pairs accessed: 204
325: stonewalling pairs accessed: 240
3: stonewalling pairs accessed: 242
531: stonewalling pairs accessed: 199
207: stonewalling pairs accessed: 217
394: stonewalling pairs accessed: 192
500: stonewalling pairs accessed: 261
450: stonewalling pairs accessed: 203
28: stonewalling pairs accessed: 253
46: stonewalling pairs accessed: 238
459: stonewalling pairs accessed: 213
13: stonewalling pairs accessed: 203
215: stonewalling pairs accessed: 200
475: stonewalling pairs accessed: 238
597: stonewalling pairs accessed: 193
274: stonewalling pairs accessed: 257
51: stonewalling pairs accessed: 228
392: stonewalling pairs accessed: 221
224: stonewalling pairs accessed: 241
493: stonewalling pairs accessed: 207
93: stonewalling pairs accessed: 212
138: stonewalling pairs accessed: 241
467: stonewalling pairs accessed: 203
323: stonewalling pairs accessed: 193
579: stonewalling pairs accessed: 174
253: stonewalling pairs accessed: 238
20: stonewalling pairs accessed: 260
361: stonewalling pairs accessed: 242
233: stonewalling pairs accessed: 191
512: stonewalling pairs accessed: 221
62: stonewalling pairs accessed: 180
125: stonewalling pairs accessed: 221
445: stonewalling pairs accessed: 171
349: stonewalling pairs accessed: 255
578: stonewalling pairs accessed: 213
268: stonewalling pairs accessed: 220
30: stonewalling pairs accessed: 229
373: stonewalling pairs accessed: 205
223: stonewalling pairs accessed: 226
530: stonewalling pairs accessed: 244
65: stonewalling pairs accessed: 221
152: stonewalling pairs accessed: 237
421: stonewalling pairs accessed: 208
318: stonewalling pairs accessed: 239
551: stonewalling pairs accessed: 271
244: stonewalling pairs accessed: 217
22: stonewalling pairs accessed: 272
412: stonewalling pairs accessed: 186
228: stonewalling pairs accessed: 221
516: stonewalling pairs accessed: 229
78: stonewalling pairs accessed: 252
140: stonewalling pairs accessed: 275
465: stonewalling pairs accessed: 232
320: stonewalling pairs accessed: 258
593: stonewalling pairs accessed: 218
254: stonewalling pairs accessed: 274
1: stonewalling pairs accessed: 293
413: stonewalling pairs accessed: 205
194: stonewalling pairs accessed: 233
484: stonewalling pairs accessed: 244
82: stonewalling pairs accessed: 243
160: stonewalling pairs accessed: 203
473: stonewalling pairs accessed: 194
353: stonewalling pairs accessed: 241
583: stonewalling pairs accessed: 247
255: stonewalling pairs accessed: 225
42: stonewalling pairs accessed: 183
381: stonewalling pairs accessed: 233
238: stonewalling pairs accessed: 235
527: stonewalling pairs accessed: 200
104: stonewalling pairs accessed: 204
178: stonewalling pairs accessed: 186
335: stonewalling pairs accessed: 218
577: stonewalling pairs accessed: 214
286: stonewalling pairs accessed: 190
4: stonewalling pairs accessed: 253
378: stonewalling pairs accessed: 211
237: stonewalling pairs accessed: 226
508: stonewalling pairs accessed: 267
80: stonewalling pairs accessed: 261
149: stonewalling pairs accessed: 231
343: stonewalling pairs accessed: 216
566: stonewalling pairs accessed: 206
59: stonewalling pairs accessed: 231
386: stonewalling pairs accessed: 233
236: stonewalling pairs accessed: 199
481: stonewalling pairs accessed: 219
92: stonewalling pairs accessed: 191
303: stonewalling pairs accessed: 283
574: stonewalling pairs accessed: 256
41: stonewalling pairs accessed: 221
410: stonewalling pairs accessed: 223
192: stonewalling pairs accessed: 265
528: stonewalling pairs accessed: 199
117: stonewalling pairs accessed: 195
347: stonewalling pairs accessed: 172
548: stonewalling pairs accessed: 238
387: stonewalling pairs accessed: 245
222: stonewalling pairs accessed: 236
95: stonewalling pairs accessed: 191
338: stonewalling pairs accessed: 202
569: stonewalling pairs accessed: 217
363: stonewalling pairs accessed: 239
202: stonewalling pairs accessed: 196
91: stonewalling pairs accessed: 196
346: stonewalling pairs accessed: 208
570: stonewalling pairs accessed: 247
362: stonewalling pairs accessed: 234
203: stonewalling pairs accessed: 201
515: stonewalling pairs accessed: 200
307: stonewalling pairs accessed: 223
588: stonewalling pairs accessed: 267
247: stonewalling pairs accessed: 215
393: stonewalling pairs accessed: 209
186: stonewalling pairs accessed: 278
316: stonewalling pairs accessed: 262
563: stonewalling pairs accessed: 273
260: stonewalling pairs accessed: 261
417: stonewalling pairs accessed: 176
206: stonewalling pairs accessed: 293
520: stonewalling pairs accessed: 199
568: stonewalling pairs accessed: 232
241: stonewalling pairs accessed: 294
396: stonewalling pairs accessed: 225
204: stonewalling pairs accessed: 201
439: stonewalling pairs accessed: 215
549: stonewalling pairs accessed: 246
384: stonewalling pairs accessed: 212
197: stonewalling pairs accessed: 302
128: stonewalling pairs accessed: 172
479: stonewalling pairs accessed: 194
542: stonewalling pairs accessed: 232
388: stonewalling pairs accessed: 235
216: stonewalling pairs accessed: 215
573: stonewalling pairs accessed: 189
383: stonewalling pairs accessed: 232
230: stonewalling pairs accessed: 229
556: stonewalling pairs accessed: 231
405: stonewalling pairs accessed: 244
232: stonewalling pairs accessed: 197
553: stonewalling pairs accessed: 227
367: stonewalling pairs accessed: 178
182: stonewalling pairs accessed: 218
420: stonewalling pairs accessed: 258
585: stonewalling pairs accessed: 231
365: stonewalling pairs accessed: 260
231: stonewalling pairs accessed: 225
595: stonewalling pairs accessed: 212
399: stonewalling pairs accessed: 230
144: stonewalling pairs accessed: 180
567: stonewalling pairs accessed: 219
374: stonewalling pairs accessed: 241
334: stonewalling pairs accessed: 204
552: stonewalling pairs accessed: 237
190: stonewalling pairs accessed: 224
119: stonewalling pairs accessed: 253
590: stonewalling pairs accessed: 219
557: stonewalling pairs accessed: 229
591: stonewalling pairs accessed: 219
213: stonewalling pairs accessed: 226
153: stonewalling pairs accessed: 194
142: stonewalling pairs accessed: 220
299: stonewalling pairs accessed: 201
135: stonewalling pairs accessed: 244
586: stonewalling pairs accessed: 233
122: stonewalling pairs accessed: 261
504: stonewalling pairs accessed: 248
187: stonewalling pairs accessed: 182
31: stonewalling pairs accessed: 225
572: stonewalling pairs accessed: 160
592: stonewalling pairs accessed: 191
327: stonewalling pairs accessed: 198
212: stonewalling pairs accessed: 225
21: stonewalling pairs accessed: 252
357: stonewalling pairs accessed: 187
461: stonewalling pairs accessed: 184
161: stonewalling pairs accessed: 175
61: stonewalling pairs accessed: 209
496: stonewalling pairs accessed: 232
446: stonewalling pairs accessed: 233
131: stonewalling pairs accessed: 233
234: stonewalling pairs accessed: 196
193: stonewalling pairs accessed: 228
454: stonewalling pairs accessed: 209
159: stonewalling pairs accessed: 189
390: stonewalling pairs accessed: 205
297: stonewalling pairs accessed: 226
466: stonewalling pairs accessed: 179
225: stonewalling pairs accessed: 200
477: stonewalling pairs accessed: 215
322: stonewalling pairs accessed: 209
24: stonewalling pairs accessed: 214
301: stonewalling pairs accessed: 224
219: stonewalling pairs accessed: 200
313: stonewalling pairs accessed: 238
354: stonewalling pairs accessed: 267
33: stonewalling pairs accessed: 232
37: stonewalling pairs accessed: 244
40: stonewalling pairs accessed: 230
74: stonewalling pairs accessed: 219
124: stonewalling pairs accessed: 249
123: stonewalling pairs accessed: 260
317: stonewalling pairs accessed: 248
306: stonewalling pairs accessed: 224
174: stonewalling pairs accessed: 191
50: stonewalling pairs accessed: 240
181: stonewalling pairs accessed: 205
497: stonewalling pairs accessed: 276
264: stonewalling pairs accessed: 213
524: stonewalling pairs accessed: 219
494: stonewalling pairs accessed: 183
300: stonewalling pairs accessed: 221
35: stonewalling pairs accessed: 201
558: stonewalling pairs accessed: 224
277: stonewalling pairs accessed: 197
57: stonewalling pairs accessed: 194
116: stonewalling pairs accessed: 181
296: stonewalling pairs accessed: 223
514: stonewalling pairs accessed: 234
352: stonewalling pairs accessed: 214
291: stonewalling pairs accessed: 250
429: stonewalling pairs accessed: 270
102: stonewalling pairs accessed: 203
485: stonewalling pairs accessed: 224
175: stonewalling pairs accessed: 204
0: stonewalling pairs accessed: 232
538: stonewalling pairs accessed: 314
310: stonewalling pairs accessed: 194
526: stonewalling pairs accessed: 212
155: stonewalling pairs accessed: 196
331: stonewalling pairs accessed: 226
505: stonewalling pairs accessed: 231
510: stonewalling pairs accessed: 196
151: stonewalling pairs accessed: 223
319: stonewalling pairs accessed: 226
293: stonewalling pairs accessed: 207
32: stonewalling pairs accessed: 212
487: stonewalling pairs accessed: 195
464: stonewalling pairs accessed: 180
106: stonewalling pairs accessed: 213
96: stonewalling pairs accessed: 193
424: stonewalling pairs accessed: 241
292: stonewalling pairs accessed: 253
470: stonewalling pairs accessed: 199
53: stonewalling pairs accessed: 220
501: stonewalling pairs accessed: 274
90: stonewalling pairs accessed: 209
426: stonewalling pairs accessed: 253
506: stonewalling pairs accessed: 223
148: stonewalling pairs accessed: 227
503: stonewalling pairs accessed: 236
535: stonewalling pairs accessed: 213
141: stonewalling pairs accessed: 243
288: stonewalling pairs accessed: 242
120: stonewalling pairs accessed: 226
257: stonewalling pairs accessed: 238
438: stonewalling pairs accessed: 222
474: stonewalling pairs accessed: 213
476: stonewalling pairs accessed: 241
305: stonewalling pairs accessed: 224
336: stonewalling pairs accessed: 214
462: stonewalling pairs accessed: 186
276: stonewalling pairs accessed: 222
150: stonewalling pairs accessed: 190
342: stonewalling pairs accessed: 234
83: stonewalling pairs accessed: 210
469: stonewalling pairs accessed: 196
452: stonewalling pairs accessed: 219
266: stonewalling pairs accessed: 216
519: stonewalling pairs accessed: 240
537: stonewalling pairs accessed: 180
302: stonewalling pairs accessed: 213
517: stonewalling pairs accessed: 234
314: stonewalling pairs accessed: 251
448: stonewalling pairs accessed: 260
356: stonewalling pairs accessed: 223
272: stonewalling pairs accessed: 192
482: stonewalling pairs accessed: 248
596: stonewalling pairs accessed: 216
598: stonewalling pairs accessed: 190
507: stonewalling pairs accessed: 206
64: stonewalling pairs accessed: 251
118: stonewalling pairs accessed: 151
184: stonewalling pairs accessed: 200
179: stonewalling pairs accessed: 245
282: stonewalling pairs accessed: 215
172: stonewalling pairs accessed: 214
489: stonewalling pairs accessed: 226
226: stonewalling pairs accessed: 183
147: stonewalling pairs accessed: 240
7: stonewalling pairs accessed: 205
167: stonewalling pairs accessed: 238
540: stonewalling pairs accessed: 223
348: stonewalling pairs accessed: 215
275: stonewalling pairs accessed: 222
324: stonewalling pairs accessed: 212
261: stonewalling pairs accessed: 219
103: stonewalling pairs accessed: 210
406: stonewalling pairs accessed: 182
311: stonewalling pairs accessed: 217
170: stonewalling pairs accessed: 203
251: stonewalling pairs accessed: 193
280: stonewalling pairs accessed: 191
533: stonewalling pairs accessed: 225
321: stonewalling pairs accessed: 280
71: stonewalling pairs accessed: 187
433: stonewalling pairs accessed: 254
398: stonewalling pairs accessed: 258
94: stonewalling pairs accessed: 227
113: stonewalling pairs accessed: 221
267: stonewalling pairs accessed: 213
166: stonewalling pairs accessed: 251
109: stonewalling pairs accessed: 261
359: stonewalling pairs accessed: 225
262: stonewalling pairs accessed: 189
86: stonewalling pairs accessed: 190
245: stonewalling pairs accessed: 242
145: stonewalling pairs accessed: 232
99: stonewalling pairs accessed: 202
422: stonewalling pairs accessed: 188
97: stonewalling pairs accessed: 192
434: stonewalling pairs accessed: 226
76: stonewalling pairs accessed: 232
575: stonewalling pairs accessed: 180
546: stonewalling pairs accessed: 195
561: stonewalling pairs accessed: 232
81: stonewalling pairs accessed: 256
571: stonewalling pairs accessed: 167
582: stonewalling pairs accessed: 236
249: stonewalling pairs accessed: 231
256: stonewalling pairs accessed: 235
298: stonewalling pairs accessed: 225
419: stonewalling pairs accessed: 214
85: stonewalling pairs accessed: 203
246: stonewalling pairs accessed: 263
9: stonewalling pairs accessed: 223
66: stonewalling pairs accessed: 206
210: stonewalling pairs accessed: 201
173: stonewalling pairs accessed: 277
258: stonewalling pairs accessed: 226
129: stonewalling pairs accessed: 244
105: stonewalling pairs accessed: 209
341: stonewalling pairs accessed: 222
443: stonewalling pairs accessed: 234
555: stonewalling pairs accessed: 211
67: stonewalling pairs accessed: 249
472: stonewalling pairs accessed: 176
111: stonewalling pairs accessed: 252
308: stonewalling pairs accessed: 200
240: stonewalling pairs accessed: 237
550: stonewalling pairs accessed: 237
227: stonewalling pairs accessed: 200
146: stonewalling pairs accessed: 263
200: stonewalling pairs accessed: 197
163: stonewalling pairs accessed: 215
457: stonewalling pairs accessed: 219
463: stonewalling pairs accessed: 269
208: stonewalling pairs accessed: 196
235: stonewalling pairs accessed: 274
220: stonewalling pairs accessed: 229
368: stonewalling pairs accessed: 210
382: stonewalling pairs accessed: 185
377: stonewalling pairs accessed: 236
199: stonewalling pairs accessed: 239
408: stonewalling pairs accessed: 219
326: stonewalling pairs accessed: 230
486: stonewalling pairs accessed: 224
87: stonewalling pairs accessed: 241
209: stonewalling pairs accessed: 174
63: stonewalling pairs accessed: 245
416: stonewalling pairs accessed: 213
364: stonewalling pairs accessed: 239
370: stonewalling pairs accessed: 253
366: stonewalling pairs accessed: 228
385: stonewalling pairs accessed: 230
400: stonewalling pairs accessed: 234
176: stonewalling pairs accessed: 195
380: stonewalling pairs accessed: 236
404: stonewalling pairs accessed: 205
488: stonewalling pairs accessed: 259
536: stonewalling pairs accessed: 217
389: stonewalling pairs accessed: 220
248: stonewalling pairs accessed: 231
432: stonewalling pairs accessed: 221
411: stonewalling pairs accessed: 167
355: stonewalling pairs accessed: 221
468: stonewalling pairs accessed: 215
511: stonewalling pairs accessed: 211
88: stonewalling pairs accessed: 252
337: stonewalling pairs accessed: 200
492: stonewalling pairs accessed: 206
490: stonewalling pairs accessed: 286
495: stonewalling pairs accessed: 213
436: stonewalling pairs accessed: 241
309: stonewalling pairs accessed: 271
114: stonewalling pairs accessed: 271
75: stonewalling pairs accessed: 249
529: stonewalling pairs accessed: 202
273: stonewalling pairs accessed: 198
133: stonewalling pairs accessed: 194
169: stonewalling pairs accessed: 148
171: stonewalling pairs accessed: 179
252: stonewalling pairs accessed: 241
162: stonewalling pairs accessed: 181
295: stonewalling pairs accessed: 239
290: stonewalling pairs accessed: 228
70: stonewalling pairs accessed: 187
136: stonewalling pairs accessed: 254
279: stonewalling pairs accessed: 212
132: stonewalling pairs accessed: 244
73: stonewalling pairs accessed: 233
107: stonewalling pairs accessed: 230
156: stonewalling pairs accessed: 239
134: stonewalling pairs accessed: 199
72: stonewalling pairs accessed: 270
115: stonewalling pairs accessed: 197
stonewalling pairs accessed min: 148 max: 314 -- min data: 4.6 GiB mean data: 7.0 GiB time: 300.8s
WARNING: Expected aggregate file size       = 3774873600000000.
WARNING: Stat() of aggregate file size      = 6321654988800.
WARNING: Using actual aggregate bytes moved = 6321654988800.
WARNING: maybe caused by deadlineForStonewalling
write     14620      456.98     0.001997    6144000000 32768      0.079677   412.27     0.020230   412.37     0   
Max Write: 14619.77 MiB/sec (15329.94 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       14619.77   14619.77   14619.77       0.00     456.87     456.87     456.87       0.00  412.37305     300.82      14208.71     0    600  60    1   1     1        1         0    0      1 6291456000000 33554432 6028800.0 POSIX      0
Finished            : Tue Jun  2 19:49:01 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 20:32:17 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -r -R -s 1800000 -a POSIX --posix.odirect -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datadir/2020.06.02-19.41.54-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datadir/2020.06.02-19.41.54-scr/ior_hard/stonewall
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 20:32:17 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr/ior_hard
FS                  : 50.0 TiB   Used FS: 64.5%   Inodes: 858.3 Mi   Used Inodes: 5.9%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-19.41.54-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1800000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 46.17 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Tue Jun  2 20:32:17 2020

WARNING: Expected aggregate file size       = 50768640000000.
WARNING: Stat() of aggregate file size      = 4913840256000.
WARNING: Using actual aggregate bytes moved = 4913840256000.
read      9693       216210     483.46      45.91      45.91      0.002984   483.48     0.000464   483.48     0   
Max Read:  9692.68 MiB/sec (10163.52 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         9692.68    9692.68    9692.68       0.00  216208.22  216208.22  216208.22       0.00  483.47838         NA            NA     0    600  60    1   0     1        1         0    0 1800000    47008    47008 4686203.0 POSIX      0
Finished            : Tue Jun  2 20:40:20 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jun  2 19:58:40 2020
Command line        : /io500/io500-isc20/io500-app/bin/ior -w -s 1800000 -a POSIX --posix.odirect -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datadir/2020.06.02-19.41.54-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datadir/2020.06.02-19.41.54-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux h09u02.int.janelia.org
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jun  2 19:58:40 2020
Path                : /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr/ior_hard
FS                  : 50.0 TiB   Used FS: 55.4%   Inodes: 858.3 Mi   Used Inodes: 4.8%
Participating tasks: 600
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datadir/2020.06.02-19.41.54-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1800000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 600
clients per node    : 60
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 46.17 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Tue Jun  2 19:58:40 2020
598: stonewalling pairs accessed: 174219
119: stonewalling pairs accessed: 174220
599: stonewalling pairs accessed: 25
59: stonewalling pairs accessed: 174220
179: stonewalling pairs accessed: 174220
239: stonewalling pairs accessed: 174220
299: stonewalling pairs accessed: 174220
58: stonewalling pairs accessed: 33
298: stonewalling pairs accessed: 35
359: stonewalling pairs accessed: 174220
178: stonewalling pairs accessed: 36
118: stonewalling pairs accessed: 39
419: stonewalling pairs accessed: 174220
418: stonewalling pairs accessed: 36
538: stonewalling pairs accessed: 174220
479: stonewalling pairs accessed: 174220
238: stonewalling pairs accessed: 37
358: stonewalling pairs accessed: 33
539: stonewalling pairs accessed: 26
478: stonewalling pairs accessed: 36
458: stonewalling pairs accessed: 48
580: stonewalling pairs accessed: 33
109: stonewalling pairs accessed: 39
394: stonewalling pairs accessed: 48
187: stonewalling pairs accessed: 57
268: stonewalling pairs accessed: 42
342: stonewalling pairs accessed: 58
36: stonewalling pairs accessed: 39
504: stonewalling pairs accessed: 25
473: stonewalling pairs accessed: 42
156: stonewalling pairs accessed: 32
581: stonewalling pairs accessed: 30
76: stonewalling pairs accessed: 51
401: stonewalling pairs accessed: 42
229: stonewalling pairs accessed: 33
270: stonewalling pairs accessed: 52
344: stonewalling pairs accessed: 50
46: stonewalling pairs accessed: 53
482: stonewalling pairs accessed: 21
459: stonewalling pairs accessed: 36
126: stonewalling pairs accessed: 57
552: stonewalling pairs accessed: 41
112: stonewalling pairs accessed: 49
405: stonewalling pairs accessed: 50
186: stonewalling pairs accessed: 48
246: stonewalling pairs accessed: 43
301: stonewalling pairs accessed: 35
2: stonewalling pairs accessed: 40
487: stonewalling pairs accessed: 25
470: stonewalling pairs accessed: 53
163: stonewalling pairs accessed: 41
555: stonewalling pairs accessed: 28
65: stonewalling pairs accessed: 56
371: stonewalling pairs accessed: 57
232: stonewalling pairs accessed: 48
250: stonewalling pairs accessed: 41
336: stonewalling pairs accessed: 58
48: stonewalling pairs accessed: 56
531: stonewalling pairs accessed: 22
474: stonewalling pairs accessed: 58
177: stonewalling pairs accessed: 31
540: stonewalling pairs accessed: 32
67: stonewalling pairs accessed: 43
372: stonewalling pairs accessed: 46
190: stonewalling pairs accessed: 56
240: stonewalling pairs accessed: 37
303: stonewalling pairs accessed: 46
52: stonewalling pairs accessed: 62
501: stonewalling pairs accessed: 21
462: stonewalling pairs accessed: 31
136: stonewalling pairs accessed: 57
546: stonewalling pairs accessed: 42
71: stonewalling pairs accessed: 42
396: stonewalling pairs accessed: 46
209: stonewalling pairs accessed: 32
247: stonewalling pairs accessed: 64
324: stonewalling pairs accessed: 36
10: stonewalling pairs accessed: 44
532: stonewalling pairs accessed: 27
467: stonewalling pairs accessed: 54
154: stonewalling pairs accessed: 36
570: stonewalling pairs accessed: 27
86: stonewalling pairs accessed: 30
366: stonewalling pairs accessed: 60
183: stonewalling pairs accessed: 34
256: stonewalling pairs accessed: 58
343: stonewalling pairs accessed: 52
11: stonewalling pairs accessed: 47
537: stonewalling pairs accessed: 26
472: stonewalling pairs accessed: 58
120: stonewalling pairs accessed: 41
576: stonewalling pairs accessed: 33
115: stonewalling pairs accessed: 30
387: stonewalling pairs accessed: 45
205: stonewalling pairs accessed: 31
260: stonewalling pairs accessed: 53
345: stonewalling pairs accessed: 45
37: stonewalling pairs accessed: 46
516: stonewalling pairs accessed: 21
435: stonewalling pairs accessed: 32
134: stonewalling pairs accessed: 42
589: stonewalling pairs accessed: 36
84: stonewalling pairs accessed: 37
403: stonewalling pairs accessed: 51
230: stonewalling pairs accessed: 54
263: stonewalling pairs accessed: 51
306: stonewalling pairs accessed: 53
39: stonewalling pairs accessed: 44
518: stonewalling pairs accessed: 23
451: stonewalling pairs accessed: 49
137: stonewalling pairs accessed: 40
591: stonewalling pairs accessed: 28
87: stonewalling pairs accessed: 43
408: stonewalling pairs accessed: 49
194: stonewalling pairs accessed: 32
276: stonewalling pairs accessed: 50
327: stonewalling pairs accessed: 38
45: stonewalling pairs accessed: 40
519: stonewalling pairs accessed: 20
469: stonewalling pairs accessed: 32
151: stonewalling pairs accessed: 41
579: stonewalling pairs accessed: 43
100: stonewalling pairs accessed: 30
414: stonewalling pairs accessed: 60
217: stonewalling pairs accessed: 42
284: stonewalling pairs accessed: 40
330: stonewalling pairs accessed: 33
49: stonewalling pairs accessed: 37
536: stonewalling pairs accessed: 25
465: stonewalling pairs accessed: 41
125: stonewalling pairs accessed: 34
593: stonewalling pairs accessed: 42
60: stonewalling pairs accessed: 33
395: stonewalling pairs accessed: 32
193: stonewalling pairs accessed: 36
286: stonewalling pairs accessed: 57
350: stonewalling pairs accessed: 58
50: stonewalling pairs accessed: 59
530: stonewalling pairs accessed: 27
428: stonewalling pairs accessed: 31
135: stonewalling pairs accessed: 55
595: stonewalling pairs accessed: 35
74: stonewalling pairs accessed: 43
363: stonewalling pairs accessed: 30
204: stonewalling pairs accessed: 40
290: stonewalling pairs accessed: 55
349: stonewalling pairs accessed: 31
51: stonewalling pairs accessed: 42
494: stonewalling pairs accessed: 30
447: stonewalling pairs accessed: 31
165: stonewalling pairs accessed: 36
542: stonewalling pairs accessed: 33
82: stonewalling pairs accessed: 31
376: stonewalling pairs accessed: 56
237: stonewalling pairs accessed: 31
257: stonewalling pairs accessed: 33
357: stonewalling pairs accessed: 42
1: stonewalling pairs accessed: 54
506: stonewalling pairs accessed: 30
446: stonewalling pairs accessed: 38
142: stonewalling pairs accessed: 32
543: stonewalling pairs accessed: 36
103: stonewalling pairs accessed: 41
380: stonewalling pairs accessed: 53
223: stonewalling pairs accessed: 55
259: stonewalling pairs accessed: 52
318: stonewalling pairs accessed: 30
15: stonewalling pairs accessed: 42
521: stonewalling pairs accessed: 20
466: stonewalling pairs accessed: 46
122: stonewalling pairs accessed: 65
553: stonewalling pairs accessed: 31
90: stonewalling pairs accessed: 42
377: stonewalling pairs accessed: 35
225: stonewalling pairs accessed: 36
279: stonewalling pairs accessed: 47
332: stonewalling pairs accessed: 41
8: stonewalling pairs accessed: 52
533: stonewalling pairs accessed: 30
461: stonewalling pairs accessed: 32
123: stonewalling pairs accessed: 32
556: stonewalling pairs accessed: 36
98: stonewalling pairs accessed: 31
416: stonewalling pairs accessed: 62
236: stonewalling pairs accessed: 31
282: stonewalling pairs accessed: 48
337: stonewalling pairs accessed: 35
9: stonewalling pairs accessed: 48
486: stonewalling pairs accessed: 23
421: stonewalling pairs accessed: 32
130: stonewalling pairs accessed: 63
571: stonewalling pairs accessed: 44
113: stonewalling pairs accessed: 53
360: stonewalling pairs accessed: 31
207: stonewalling pairs accessed: 31
287: stonewalling pairs accessed: 34
311: stonewalling pairs accessed: 36
23: stonewalling pairs accessed: 33
502: stonewalling pairs accessed: 28
443: stonewalling pairs accessed: 32
131: stonewalling pairs accessed: 39
584: stonewalling pairs accessed: 45
94: stonewalling pairs accessed: 41
397: stonewalling pairs accessed: 30
195: stonewalling pairs accessed: 46
292: stonewalling pairs accessed: 64
353: stonewalling pairs accessed: 36
42: stonewalling pairs accessed: 49
527: stonewalling pairs accessed: 28
444: stonewalling pairs accessed: 31
148: stonewalling pairs accessed: 37
585: stonewalling pairs accessed: 38
95: stonewalling pairs accessed: 30
411: stonewalling pairs accessed: 62
197: stonewalling pairs accessed: 31
294: stonewalling pairs accessed: 47
314: stonewalling pairs accessed: 32
55: stonewalling pairs accessed: 35
505: stonewalling pairs accessed: 20
420: stonewalling pairs accessed: 30
153: stonewalling pairs accessed: 35
588: stonewalling pairs accessed: 45
106: stonewalling pairs accessed: 35
370: stonewalling pairs accessed: 42
210: stonewalling pairs accessed: 31
261: stonewalling pairs accessed: 38
356: stonewalling pairs accessed: 38
3: stonewalling pairs accessed: 37
499: stonewalling pairs accessed: 23
429: stonewalling pairs accessed: 30
157: stonewalling pairs accessed: 40
592: stonewalling pairs accessed: 48
114: stonewalling pairs accessed: 37
407: stonewalling pairs accessed: 35
227: stonewalling pairs accessed: 32
295: stonewalling pairs accessed: 70
338: stonewalling pairs accessed: 53
25: stonewalling pairs accessed: 44
508: stonewalling pairs accessed: 20
440: stonewalling pairs accessed: 38
168: stonewalling pairs accessed: 31
596: stonewalling pairs accessed: 34
70: stonewalling pairs accessed: 37
417: stonewalling pairs accessed: 35
213: stonewalling pairs accessed: 31
254: stonewalling pairs accessed: 36
322: stonewalling pairs accessed: 37
20: stonewalling pairs accessed: 37
534: stonewalling pairs accessed: 39
441: stonewalling pairs accessed: 33
175: stonewalling pairs accessed: 69
597: stonewalling pairs accessed: 31
75: stonewalling pairs accessed: 32
368: stonewalling pairs accessed: 41
206: stonewalling pairs accessed: 40
241: stonewalling pairs accessed: 68
326: stonewalling pairs accessed: 40
33: stonewalling pairs accessed: 32
488: stonewalling pairs accessed: 28
454: stonewalling pairs accessed: 31
176: stonewalling pairs accessed: 45
582: stonewalling pairs accessed: 30
83: stonewalling pairs accessed: 31
415: stonewalling pairs accessed: 42
231: stonewalling pairs accessed: 63
266: stonewalling pairs accessed: 36
334: stonewalling pairs accessed: 35
13: stonewalling pairs accessed: 52
517: stonewalling pairs accessed: 28
455: stonewalling pairs accessed: 31
128: stonewalling pairs accessed: 63
541: stonewalling pairs accessed: 31
89: stonewalling pairs accessed: 30
413: stonewalling pairs accessed: 34
212: stonewalling pairs accessed: 31
293: stonewalling pairs accessed: 36
341: stonewalling pairs accessed: 54
21: stonewalling pairs accessed: 31
525: stonewalling pairs accessed: 22
464: stonewalling pairs accessed: 53
155: stonewalling pairs accessed: 40
544: stonewalling pairs accessed: 53
102: stonewalling pairs accessed: 32
390: stonewalling pairs accessed: 35
216: stonewalling pairs accessed: 32
242: stonewalling pairs accessed: 36
317: stonewalling pairs accessed: 41
35: stonewalling pairs accessed: 45
493: stonewalling pairs accessed: 20
475: stonewalling pairs accessed: 35
132: stonewalling pairs accessed: 37
551: stonewalling pairs accessed: 28
108: stonewalling pairs accessed: 45
412: stonewalling pairs accessed: 60
228: stonewalling pairs accessed: 63
269: stonewalling pairs accessed: 31
321: stonewalling pairs accessed: 35
54: stonewalling pairs accessed: 65
526: stonewalling pairs accessed: 23
430: stonewalling pairs accessed: 41
146: stonewalling pairs accessed: 45
566: stonewalling pairs accessed: 39
62: stonewalling pairs accessed: 32
381: stonewalling pairs accessed: 38
215: stonewalling pairs accessed: 37
280: stonewalling pairs accessed: 41
328: stonewalling pairs accessed: 38
31: stonewalling pairs accessed: 47
529: stonewalling pairs accessed: 20
453: stonewalling pairs accessed: 53
139: stonewalling pairs accessed: 32
568: stonewalling pairs accessed: 43
69: stonewalling pairs accessed: 30
400: stonewalling pairs accessed: 30
233: stonewalling pairs accessed: 37
281: stonewalling pairs accessed: 53
340: stonewalling pairs accessed: 30
22: stonewalling pairs accessed: 32
481: stonewalling pairs accessed: 38
463: stonewalling pairs accessed: 31
124: stonewalling pairs accessed: 34
574: stonewalling pairs accessed: 27
96: stonewalling pairs accessed: 31
404: stonewalling pairs accessed: 38
202: stonewalling pairs accessed: 31
291: stonewalling pairs accessed: 32
315: stonewalling pairs accessed: 38
41: stonewalling pairs accessed: 30
500: stonewalling pairs accessed: 28
471: stonewalling pairs accessed: 42
160: stonewalling pairs accessed: 34
577: stonewalling pairs accessed: 27
97: stonewalling pairs accessed: 37
409: stonewalling pairs accessed: 41
200: stonewalling pairs accessed: 32
273: stonewalling pairs accessed: 40
351: stonewalling pairs accessed: 41
57: stonewalling pairs accessed: 59
524: stonewalling pairs accessed: 20
436: stonewalling pairs accessed: 41
166: stonewalling pairs accessed: 37
578: stonewalling pairs accessed: 31
101: stonewalling pairs accessed: 31
383: stonewalling pairs accessed: 46
185: stonewalling pairs accessed: 39
243: stonewalling pairs accessed: 37
316: stonewalling pairs accessed: 40
17: stonewalling pairs accessed: 45
495: stonewalling pairs accessed: 23
448: stonewalling pairs accessed: 31
129: stonewalling pairs accessed: 45
547: stonewalling pairs accessed: 53
111: stonewalling pairs accessed: 31
393: stonewalling pairs accessed: 37
219: stonewalling pairs accessed: 40
248: stonewalling pairs accessed: 40
302: stonewalling pairs accessed: 32
47: stonewalling pairs accessed: 37
498: stonewalling pairs accessed: 32
438: stonewalling pairs accessed: 41
172: stonewalling pairs accessed: 37
554: stonewalling pairs accessed: 31
116: stonewalling pairs accessed: 31
402: stonewalling pairs accessed: 42
224: stonewalling pairs accessed: 37
251: stonewalling pairs accessed: 32
320: stonewalling pairs accessed: 38
5: stonewalling pairs accessed: 48
503: stonewalling pairs accessed: 20
449: stonewalling pairs accessed: 31
159: stonewalling pairs accessed: 32
558: stonewalling pairs accessed: 36
79: stonewalling pairs accessed: 45
406: stonewalling pairs accessed: 54
182: stonewalling pairs accessed: 37
289: stonewalling pairs accessed: 38
354: stonewalling pairs accessed: 41
28: stonewalling pairs accessed: 34
511: stonewalling pairs accessed: 28
456: stonewalling pairs accessed: 42
171: stonewalling pairs accessed: 45
560: stonewalling pairs accessed: 27
99: stonewalling pairs accessed: 31
374: stonewalling pairs accessed: 38
199: stonewalling pairs accessed: 40
253: stonewalling pairs accessed: 57
323: stonewalling pairs accessed: 30
34: stonewalling pairs accessed: 32
513: stonewalling pairs accessed: 20
468: stonewalling pairs accessed: 35
173: stonewalling pairs accessed: 32
561: stonewalling pairs accessed: 27
68: stonewalling pairs accessed: 49
388: stonewalling pairs accessed: 35
218: stonewalling pairs accessed: 32
274: stonewalling pairs accessed: 32
313: stonewalling pairs accessed: 35
30: stonewalling pairs accessed: 37
480: stonewalling pairs accessed: 28
434: stonewalling pairs accessed: 41
161: stonewalling pairs accessed: 40
562: stonewalling pairs accessed: 27
80: stonewalling pairs accessed: 41
382: stonewalling pairs accessed: 55
181: stonewalling pairs accessed: 32
288: stonewalling pairs accessed: 37
325: stonewalling pairs accessed: 35
12: stonewalling pairs accessed: 37
520: stonewalling pairs accessed: 20
427: stonewalling pairs accessed: 41
149: stonewalling pairs accessed: 45
563: stonewalling pairs accessed: 27
85: stonewalling pairs accessed: 30
398: stonewalling pairs accessed: 35
201: stonewalling pairs accessed: 40
265: stonewalling pairs accessed: 56
329: stonewalling pairs accessed: 38
38: stonewalling pairs accessed: 45
490: stonewalling pairs accessed: 28
450: stonewalling pairs accessed: 42
164: stonewalling pairs accessed: 37
567: stonewalling pairs accessed: 46
91: stonewalling pairs accessed: 34
410: stonewalling pairs accessed: 54
234: stonewalling pairs accessed: 32
244: stonewalling pairs accessed: 32
346: stonewalling pairs accessed: 41
18: stonewalling pairs accessed: 45
507: stonewalling pairs accessed: 20
452: stonewalling pairs accessed: 31
121: stonewalling pairs accessed: 37
575: stonewalling pairs accessed: 29
93: stonewalling pairs accessed: 31
364: stonewalling pairs accessed: 30
180: stonewalling pairs accessed: 32
296: stonewalling pairs accessed: 37
310: stonewalling pairs accessed: 30
32: stonewalling pairs accessed: 37
496: stonewalling pairs accessed: 28
457: stonewalling pairs accessed: 35
145: stonewalling pairs accessed: 37
583: stonewalling pairs accessed: 29
105: stonewalling pairs accessed: 45
367: stonewalling pairs accessed: 35
211: stonewalling pairs accessed: 40
249: stonewalling pairs accessed: 40
352: stonewalling pairs accessed: 30
43: stonewalling pairs accessed: 37
515: stonewalling pairs accessed: 20
477: stonewalling pairs accessed: 35
167: stonewalling pairs accessed: 45
569: stonewalling pairs accessed: 27
64: stonewalling pairs accessed: 45
391: stonewalling pairs accessed: 30
214: stonewalling pairs accessed: 40
297: stonewalling pairs accessed: 37
312: stonewalling pairs accessed: 40
6: stonewalling pairs accessed: 37
484: stonewalling pairs accessed: 23
445: stonewalling pairs accessed: 30
144: stonewalling pairs accessed: 32
548: stonewalling pairs accessed: 42
81: stonewalling pairs accessed: 30
386: stonewalling pairs accessed: 35
222: stonewalling pairs accessed: 32
285: stonewalling pairs accessed: 32
355: stonewalling pairs accessed: 30
26: stonewalling pairs accessed: 37
509: stonewalling pairs accessed: 20
476: stonewalling pairs accessed: 35
170: stonewalling pairs accessed: 37
557: stonewalling pairs accessed: 31
92: stonewalling pairs accessed: 31
378: stonewalling pairs accessed: 30
188: stonewalling pairs accessed: 45
278: stonewalling pairs accessed: 30
305: stonewalling pairs accessed: 32
14: stonewalling pairs accessed: 45
514: stonewalling pairs accessed: 20
460: stonewalling pairs accessed: 35
133: stonewalling pairs accessed: 37
572: stonewalling pairs accessed: 28
63: stonewalling pairs accessed: 32
379: stonewalling pairs accessed: 30
221: stonewalling pairs accessed: 40
255: stonewalling pairs accessed: 32
319: stonewalling pairs accessed: 30
44: stonewalling pairs accessed: 37
528: stonewalling pairs accessed: 23
437: stonewalling pairs accessed: 30
150: stonewalling pairs accessed: 45
590: stonewalling pairs accessed: 29
110: stonewalling pairs accessed: 32
365: stonewalling pairs accessed: 30
192: stonewalling pairs accessed: 32
271: stonewalling pairs accessed: 32
339: stonewalling pairs accessed: 30
24: stonewalling pairs accessed: 37
492: stonewalling pairs accessed: 23
442: stonewalling pairs accessed: 30
162: stonewalling pairs accessed: 37
564: stonewalling pairs accessed: 31
117: stonewalling pairs accessed: 32
384: stonewalling pairs accessed: 30
208: stonewalling pairs accessed: 40
272: stonewalling pairs accessed: 30
308: stonewalling pairs accessed: 30
27: stonewalling pairs accessed: 45
497: stonewalling pairs accessed: 28
439: stonewalling pairs accessed: 30
138: stonewalling pairs accessed: 37
587: stonewalling pairs accessed: 31
73: stonewalling pairs accessed: 30
362: stonewalling pairs accessed: 30
220: stonewalling pairs accessed: 32
277: stonewalling pairs accessed: 30
348: stonewalling pairs accessed: 30
19: stonewalling pairs accessed: 45
512: stonewalling pairs accessed: 20
425: stonewalling pairs accessed: 30
143: stonewalling pairs accessed: 32
586: stonewalling pairs accessed: 49
104: stonewalling pairs accessed: 37
399: stonewalling pairs accessed: 30
203: stonewalling pairs accessed: 40
245: stonewalling pairs accessed: 32
333: stonewalling pairs accessed: 30
40: stonewalling pairs accessed: 37
510: stonewalling pairs accessed: 23
423: stonewalling pairs accessed: 30
174: stonewalling pairs accessed: 32
549: stonewalling pairs accessed: 28
72: stonewalling pairs accessed: 32
373: stonewalling pairs accessed: 30
235: stonewalling pairs accessed: 32
262: stonewalling pairs accessed: 32
309: stonewalling pairs accessed: 30
29: stonewalling pairs accessed: 45
535: stonewalling pairs accessed: 28
431: stonewalling pairs accessed: 30
127: stonewalling pairs accessed: 37
565: stonewalling pairs accessed: 28
78: stonewalling pairs accessed: 37
389: stonewalling pairs accessed: 30
191: stonewalling pairs accessed: 32
283: stonewalling pairs accessed: 32
307: stonewalling pairs accessed: 32
16: stonewalling pairs accessed: 37
522: stonewalling pairs accessed: 23
433: stonewalling pairs accessed: 30
158: stonewalling pairs accessed: 32
545: stonewalling pairs accessed: 42
61: stonewalling pairs accessed: 37
361: stonewalling pairs accessed: 30
226: stonewalling pairs accessed: 32
267: stonewalling pairs accessed: 32
331: stonewalling pairs accessed: 30
4: stonewalling pairs accessed: 37
491: stonewalling pairs accessed: 23
424: stonewalling pairs accessed: 30
147: stonewalling pairs accessed: 37
550: stonewalling pairs accessed: 28
107: stonewalling pairs accessed: 37
385: stonewalling pairs accessed: 30
189: stonewalling pairs accessed: 32
275: stonewalling pairs accessed: 32
335: stonewalling pairs accessed: 30
53: stonewalling pairs accessed: 37
483: stonewalling pairs accessed: 23
422: stonewalling pairs accessed: 30
169: stonewalling pairs accessed: 32
559: stonewalling pairs accessed: 28
66: stonewalling pairs accessed: 37
369: stonewalling pairs accessed: 30
196: stonewalling pairs accessed: 32
258: stonewalling pairs accessed: 32
347: stonewalling pairs accessed: 30
0: stonewalling pairs accessed: 37
523: stonewalling pairs accessed: 23
432: stonewalling pairs accessed: 30
141: stonewalling pairs accessed: 32
573: stonewalling pairs accessed: 28
88: stonewalling pairs accessed: 30
392: stonewalling pairs accessed: 30
184: stonewalling pairs accessed: 32
252: stonewalling pairs accessed: 32
304: stonewalling pairs accessed: 32
56: stonewalling pairs accessed: 37
485: stonewalling pairs accessed: 23
426: stonewalling pairs accessed: 30
140: stonewalling pairs accessed: 32
594: stonewalling pairs accessed: 36
77: stonewalling pairs accessed: 37
375: stonewalling pairs accessed: 30
198: stonewalling pairs accessed: 32
264: stonewalling pairs accessed: 32
300: stonewalling pairs accessed: 32
7: stonewalling pairs accessed: 37
489: stonewalling pairs accessed: 23
152: stonewalling pairs accessed: 32
stonewalling pairs accessed min: 20 max: 174220 -- min data: 0.0 GiB mean data: 0.1 GiB time: 300.0s
WARNING: Expected aggregate file size       = 50768640000000.
WARNING: Stat() of aggregate file size      = 4913840256000.
WARNING: Using actual aggregate bytes moved = 4913840256000.
WARNING: maybe caused by deadlineForStonewalling
write     9439       210541     485.29      45.91      45.91      0.002436   496.49     0.000478   496.49     0   
Max Write: 9438.58 MiB/sec (9897.06 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        9438.58    9438.58    9438.58       0.00  210539.98  210539.98  210539.98       0.00  496.49477     300.01        263.60     0    600  60    1   0     1        1         0    0 1800000    47008    47008 4686203.0 POSIX      0
Finished            : Tue Jun  2 20:06:56 2020
mdtest_easy_delete
-- started at 06/02/2020 20:47:29 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-r' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_easy' '-n' '24000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.5%   Inodes: 858.3 Mi   Used Inodes: 5.9%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 14400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      29588.997      29588.989      29588.994          0.002
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.069          2.069          2.069          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        486.668        486.667        486.667          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.483          0.483          0.483          0.000
-- finished at 06/02/2020 21:00:41 --

mdtest_easy_stat
-- started at 06/02/2020 20:25:28 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-T' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_easy' '-n' '24000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.5%   Inodes: 858.3 Mi   Used Inodes: 5.9%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 14400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     126328.137     126327.979     126328.025          0.039
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        113.989        113.989        113.989          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 20:32:15 --

mdtest_easy_write
-- started at 06/02/2020 19:49:04 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-C' '-F' '-P' '-Y' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_easy' '-n' '24000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 55.3%   Inodes: 858.3 Mi   Used Inodes: 3.2%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 14400000 files
Continue stonewall hit min: 536 max: 24000 avg: 17732.5 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      35526.266      35526.257      35526.262          0.002
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      35358.436             NA
   Tree creation             :         18.391         18.391         18.391          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        405.334        405.334        405.334          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.903             NA
   Tree creation             :          0.054          0.054          0.054          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 19:58:38 --

mdtest_hard_delete
-- started at 06/02/2020 21:09:03 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-r' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.3%   Inodes: 858.3 Mi   Used Inodes: 4.3%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      13684.747      13684.743      13684.745          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.467          0.467          0.467          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        347.599        347.599        347.599          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.141          2.141          2.141          0.000
-- finished at 06/02/2020 21:20:40 --

mdtest_hard_read
-- started at 06/02/2020 21:00:43 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.3%   Inodes: 858.3 Mi   Used Inodes: 4.3%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      29964.351      29964.336      29964.345          0.004
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        158.749        158.749        158.749          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 21:09:00 --

mdtest_hard_stat
-- started at 06/02/2020 20:40:23 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-T' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.5%   Inodes: 858.3 Mi   Used Inodes: 5.9%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      39700.589      39700.566      39700.578          0.005
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        119.817        119.817        119.817          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 20:47:27 --

mdtest_hard_write
-- started at 06/02/2020 20:06:59 --

mdtest-3.3.0+dev was launched with 600 total task(s) on 10 node(s)
Command line used: /io500/io500-isc20/io500-app/bin/mdtest '-C' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' './datadir/2020.06.02-19.41.54-scr/mdt_hard' '-n' '1000000' '-x' './datadir/2020.06.02-19.41.54-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /io500/io500-isc20/io500-app/datadir/2020.06.02-19.41.54-scr
FS: 50.0 TiB   Used FS: 64.4%   Inodes: 858.3 Mi   Used Inodes: 5.4%

Nodemap: 111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 60 for each phase.
600 tasks, 600000000 files
Continue stonewall hit min: 7755 max: 7928 avg: 7854.9 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      12843.527      12843.524      12843.526          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      15707.899             NA
   Tree creation             :        791.229        791.229        791.229          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        370.366        370.366        370.366          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.037             NA
   Tree creation             :          0.001          0.001          0.001          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/02/2020 20:16:33 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               14.277 GiB/s : time 412.27 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               35.526 kiops : time 405.33 seconds
[RESULT] BW   phase 2            ior_hard_write                9.218 GiB/s : time 496.49 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               12.843 kiops : time 370.37 seconds
[RESULT] IOPS phase 3                      find              251.560 kiops : time  76.15 seconds
[RESULT] BW   phase 3             ior_easy_read               12.956 GiB/s : time 454.39 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              126.328 kiops : time 113.99 seconds
[RESULT] BW   phase 4             ior_hard_read                9.466 GiB/s : time 483.48 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               39.701 kiops : time 119.82 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               29.589 kiops : time 486.67 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               29.964 kiops : time 158.75 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               13.685 kiops : time 699.21 seconds
[SCORE] Bandwidth 11.2714 GiB/s : IOPS 40.3198 kiops : TOTAL 21.3181