Presque

Institution Argonne National Laboratory
Client Procs Per Node
Client Operating System RHEL
Client Operating System Version 7.7
Client Kernel Version 3.10.0-1062.4.3.el7.x86_64

DATA SERVER

Storage Type NVMe-SSD
Volatile Memory 384
Storage Interface NVMe
Network 100gE
Software Version 1.1
OS Version 7.7

INFORMATION

Client Nodes 16
Client Total Procs 544
Metadata Nodes 10
Metadata Storage Devices 12
Data Nodes 10
Data Storage Devices 12

METADATA

Easy Write 4,386.20 kIOP/s
Easy Stat 4,521.23 kIOP/s
Easy Delete 2,559.27 kIOP/s
Hard Write 924.09 kIOP/s
Hard Read 2,100.11 kIOP/s
Hard Stat 3,730.50 kIOP/s
Hard Delete 2,357.16 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=/home/kalfizah/io500-new/bin/ior
  io500_mdtest_cmd=/home/kalfizah/io500-new/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
#  io500_mpiargs="-envnone --bind-to socket -np 12"
  io500_mpiargs="-envnone --bind-to socket -np 544 -f /home/kalfizah/io500-new/hfile1"
#  io500_mpiargs="-np 420 --hostfile /home/daos/io-500/config/cli_hosts"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

#  if [[ -s "system-information.txt" ]]; then
#    echo "Warning: please create a system-information.txt description by"
#    echo "copying the information from https://vi4io.org/io500-info-creator/"
#  else
#    cp "system-information.txt" $io500_result_dir
#  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val" && io500_find_mpi="True"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
[0] Pool uuid = 9020212b-0f07-4d33-8f84-6ac45cb4d7eb, SVCL = 1

[0] DFS Container namespace uuid = ce610953-295d-464c-af58-bd04279ded70

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 20:25:09 2020
Command line        : /home/kalfizah/io500-new/bin/ior -r -R -a DFS --dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb --dfs.cont=ce610953-295d-464c-af58-bd04279ded70 --dfs.svcl=1 --dfs.prefix=/tmp/dfuse/ -t 256k -v -b 256gm -i 1 -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_easy/stonewall
Machine             : Linux daos08.ftm.alcf.anl.gov
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Thu Jul  9 20:25:09 2020
Participating tasks: 544
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 16
tasks               : 544
clients per node    : 34
repetitions         : 1
xfersize            : 262144 bytes
blocksize           : 256 GiB
aggregate filesize  : 136 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Thu Jul  9 20:25:13 2020

WARNING: Expected aggregate file size       = 149533581377536.
WARNING: Stat() of aggregate file size      = 149364382892032.
WARNING: Using actual aggregate bytes moved = 57489605263360.
read      133356     538721     0.000388    268435456  256.00     4.04       407.09     0.000870   411.13     0   
Max Read:  133355.99 MiB/sec (139833.90 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       133355.99  133355.99  133355.99       0.00  533423.98  533423.98  533423.98       0.00  411.12782         NA            NA     0    544  34    1   0     1        1         0    0      1 274877906944   262144 54826360.0 DFS      0
Finished            : Thu Jul  9 20:32:00 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_easy_write
[0] Pool uuid = 9020212b-0f07-4d33-8f84-6ac45cb4d7eb, SVCL = 1

[0] DFS Container namespace uuid = ce610953-295d-464c-af58-bd04279ded70

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 19:56:11 2020
Command line        : /home/kalfizah/io500-new/bin/ior -w -a DFS --dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb --dfs.cont=ce610953-295d-464c-af58-bd04279ded70 --dfs.svcl=1 --dfs.prefix=/tmp/dfuse/ -t 256k -v -b 256gm -i 1 -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux daos08.ftm.alcf.anl.gov
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Thu Jul  9 19:56:11 2020
Participating tasks: 544
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 16
tasks               : 544
clients per node    : 34
repetitions         : 1
xfersize            : 262144 bytes
blocksize           : 256 GiB
aggregate filesize  : 136 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Thu Jul  9 19:56:18 2020
438: stonewalling pairs accessed: 403065
214: stonewalling pairs accessed: 396412
115: stonewalling pairs accessed: 390661
128: stonewalling pairs accessed: 391680
337: stonewalling pairs accessed: 396510
260: stonewalling pairs accessed: 395335
229: stonewalling pairs accessed: 396277
147: stonewalling pairs accessed: 401374
304: stonewalling pairs accessed: 390234
401: stonewalling pairs accessed: 391787
66: stonewalling pairs accessed: 390561
533: stonewalling pairs accessed: 401178
150: stonewalling pairs accessed: 395694
307: stonewalling pairs accessed: 394787
160: stonewalling pairs accessed: 388916
513: stonewalling pairs accessed: 395428
82: stonewalling pairs accessed: 388003
148: stonewalling pairs accessed: 395948
101: stonewalling pairs accessed: 397620
83: stonewalling pairs accessed: 395989
192: stonewalling pairs accessed: 393638
433: stonewalling pairs accessed: 401558
242: stonewalling pairs accessed: 396992
308: stonewalling pairs accessed: 390488
195: stonewalling pairs accessed: 396031
288: stonewalling pairs accessed: 389851
289: stonewalling pairs accessed: 398012
354: stonewalling pairs accessed: 397995
311: stonewalling pairs accessed: 392057
356: stonewalling pairs accessed: 395958
102: stonewalling pairs accessed: 391248
403: stonewalling pairs accessed: 397505
368: stonewalling pairs accessed: 394575
97: stonewalling pairs accessed: 396715
370: stonewalling pairs accessed: 394835
327: stonewalling pairs accessed: 391997
404: stonewalling pairs accessed: 396226
261: stonewalling pairs accessed: 396124
182: stonewalling pairs accessed: 396929
35: stonewalling pairs accessed: 400624
416: stonewalling pairs accessed: 394649
273: stonewalling pairs accessed: 395314
418: stonewalling pairs accessed: 392025
391: stonewalling pairs accessed: 396742
132: stonewalling pairs accessed: 392426
373: stonewalling pairs accessed: 386971
323: stonewalling pairs accessed: 401056
464: stonewalling pairs accessed: 403135
449: stonewalling pairs accessed: 401217
466: stonewalling pairs accessed: 398606
423: stonewalling pairs accessed: 389492
292: stonewalling pairs accessed: 396573
405: stonewalling pairs accessed: 391072
16: stonewalling pairs accessed: 395641
481: stonewalling pairs accessed: 397379
258: stonewalling pairs accessed: 397369
263: stonewalling pairs accessed: 399130
388: stonewalling pairs accessed: 391087
53: stonewalling pairs accessed: 389179
6: stonewalling pairs accessed: 394832
64: stonewalling pairs accessed: 396769
225: stonewalling pairs accessed: 401162
306: stonewalling pairs accessed: 395725
519: stonewalling pairs accessed: 398373
4: stonewalling pairs accessed: 397244
325: stonewalling pairs accessed: 397427
54: stonewalling pairs accessed: 393163
320: stonewalling pairs accessed: 397864
417: stonewalling pairs accessed: 395489
482: stonewalling pairs accessed: 396717
535: stonewalling pairs accessed: 400709
276: stonewalling pairs accessed: 389503
357: stonewalling pairs accessed: 394986
70: stonewalling pairs accessed: 396597
48: stonewalling pairs accessed: 389537
530: stonewalling pairs accessed: 394427
487: stonewalling pairs accessed: 395461
516: stonewalling pairs accessed: 393754
223: stonewalling pairs accessed: 204060
230: stonewalling pairs accessed: 394181
96: stonewalling pairs accessed: 397757
369: stonewalling pairs accessed: 391953
386: stonewalling pairs accessed: 393154
244: stonewalling pairs accessed: 394056
303: stonewalling pairs accessed: 200791
326: stonewalling pairs accessed: 395565
80: stonewalling pairs accessed: 396927
450: stonewalling pairs accessed: 399242
228: stonewalling pairs accessed: 389245
253: stonewalling pairs accessed: 197454
342: stonewalling pairs accessed: 393626
94: stonewalling pairs accessed: 202684
384: stonewalling pairs accessed: 397472
498: stonewalling pairs accessed: 388632
367: stonewalling pairs accessed: 202128
454: stonewalling pairs accessed: 394209
494: stonewalling pairs accessed: 198743
235: stonewalling pairs accessed: 198507
355: stonewalling pairs accessed: 400059
240: stonewalling pairs accessed: 390333
178: stonewalling pairs accessed: 393252
8: stonewalling pairs accessed: 201632
73: stonewalling pairs accessed: 202923
44: stonewalling pairs accessed: 202913
532: stonewalling pairs accessed: 395392
269: stonewalling pairs accessed: 201223
383: stonewalling pairs accessed: 204728
37: stonewalling pairs accessed: 392608
390: stonewalling pairs accessed: 395794
10: stonewalling pairs accessed: 201908
78: stonewalling pairs accessed: 202623
27: stonewalling pairs accessed: 199866
456: stonewalling pairs accessed: 207051
265: stonewalling pairs accessed: 198435
204: stonewalling pairs accessed: 196356
141: stonewalling pairs accessed: 202629
95: stonewalling pairs accessed: 200044
26: stonewalling pairs accessed: 202293
286: stonewalling pairs accessed: 204336
491: stonewalling pairs accessed: 200949
504: stonewalling pairs accessed: 201141
441: stonewalling pairs accessed: 202278
300: stonewalling pairs accessed: 200048
205: stonewalling pairs accessed: 206915
127: stonewalling pairs accessed: 209222
170: stonewalling pairs accessed: 198438
302: stonewalling pairs accessed: 200003
120: stonewalling pairs accessed: 199049
505: stonewalling pairs accessed: 200633
428: stonewalling pairs accessed: 197798
109: stonewalling pairs accessed: 199460
479: stonewalling pairs accessed: 206296
202: stonewalling pairs accessed: 200772
382: stonewalling pairs accessed: 202476
296: stonewalling pairs accessed: 201744
521: stonewalling pairs accessed: 199843
28: stonewalling pairs accessed: 201940
333: stonewalling pairs accessed: 203582
330: stonewalling pairs accessed: 206053
510: stonewalling pairs accessed: 203523
328: stonewalling pairs accessed: 203882
537: stonewalling pairs accessed: 199153
76: stonewalling pairs accessed: 199247
362: stonewalling pairs accessed: 204261
334: stonewalling pairs accessed: 201659
88: stonewalling pairs accessed: 201460
9: stonewalling pairs accessed: 205442
156: stonewalling pairs accessed: 200042
90: stonewalling pairs accessed: 200305
350: stonewalling pairs accessed: 202839
155: stonewalling pairs accessed: 199736
72: stonewalling pairs accessed: 203853
169: stonewalling pairs accessed: 208200
268: stonewalling pairs accessed: 203590
266: stonewalling pairs accessed: 205837
158: stonewalling pairs accessed: 199981
203: stonewalling pairs accessed: 205025
168: stonewalling pairs accessed: 203137
25: stonewalling pairs accessed: 203452
220: stonewalling pairs accessed: 203265
314: stonewalling pairs accessed: 204771
238: stonewalling pairs accessed: 202485
219: stonewalling pairs accessed: 199464
360: stonewalling pairs accessed: 204767
217: stonewalling pairs accessed: 203106
348: stonewalling pairs accessed: 202517
410: stonewalling pairs accessed: 200631
270: stonewalling pairs accessed: 200539
283: stonewalling pairs accessed: 201876
264: stonewalling pairs accessed: 197729
249: stonewalling pairs accessed: 200309
538: stonewalling pairs accessed: 198787
299: stonewalling pairs accessed: 201193
145: stonewalling pairs accessed: 397721
345: stonewalling pairs accessed: 199952
250: stonewalling pairs accessed: 201934
475: stonewalling pairs accessed: 199849
179: stonewalling pairs accessed: 394427
448: stonewalling pairs accessed: 393619
49: stonewalling pairs accessed: 399270
130: stonewalling pairs accessed: 398445
104: stonewalling pairs accessed: 199048
313: stonewalling pairs accessed: 202527
279: stonewalling pairs accessed: 394195
116: stonewalling pairs accessed: 392388
429: stonewalling pairs accessed: 204875
463: stonewalling pairs accessed: 201886
277: stonewalling pairs accessed: 394932
358: stonewalling pairs accessed: 392044
490: stonewalling pairs accessed: 200694
190: stonewalling pairs accessed: 201575
523: stonewalling pairs accessed: 202136
227: stonewalling pairs accessed: 397340
208: stonewalling pairs accessed: 391969
162: stonewalling pairs accessed: 389968
392: stonewalling pairs accessed: 202231
409: stonewalling pairs accessed: 200305
407: stonewalling pairs accessed: 397472
61: stonewalling pairs accessed: 204743
47: stonewalling pairs accessed: 206563
389: stonewalling pairs accessed: 397809
298: stonewalling pairs accessed: 197447
256: stonewalling pairs accessed: 399362
57: stonewalling pairs accessed: 200185
39: stonewalling pairs accessed: 397476
189: stonewalling pairs accessed: 196907
351: stonewalling pairs accessed: 203372
272: stonewalling pairs accessed: 390013
201: stonewalling pairs accessed: 201942
343: stonewalling pairs accessed: 395069
495: stonewalling pairs accessed: 206213
0: stonewalling pairs accessed: 388726
471: stonewalling pairs accessed: 391272
140: stonewalling pairs accessed: 201100
172: stonewalling pairs accessed: 201982
397: stonewalling pairs accessed: 200164
117: stonewalling pairs accessed: 394445
316: stonewalling pairs accessed: 201224
233: stonewalling pairs accessed: 203191
213: stonewalling pairs accessed: 391255
534: stonewalling pairs accessed: 402412
107: stonewalling pairs accessed: 205459
511: stonewalling pairs accessed: 204989
171: stonewalling pairs accessed: 203896
18: stonewalling pairs accessed: 393952
93: stonewalling pairs accessed: 202167
133: stonewalling pairs accessed: 402191
74: stonewalling pairs accessed: 201312
210: stonewalling pairs accessed: 392661
324: stonewalling pairs accessed: 397733
143: stonewalling pairs accessed: 201291
469: stonewalling pairs accessed: 395657
411: stonewalling pairs accessed: 202497
193: stonewalling pairs accessed: 398163
7: stonewalling pairs accessed: 395068
157: stonewalling pairs accessed: 202995
543: stonewalling pairs accessed: 202129
5: stonewalling pairs accessed: 394337
474: stonewalling pairs accessed: 202391
290: stonewalling pairs accessed: 392163
248: stonewalling pairs accessed: 195356
121: stonewalling pairs accessed: 199396
23: stonewalling pairs accessed: 395914
36: stonewalling pairs accessed: 397476
493: stonewalling pairs accessed: 205546
15: stonewalling pairs accessed: 201217
470: stonewalling pairs accessed: 400675
58: stonewalling pairs accessed: 205747
352: stonewalling pairs accessed: 394966
473: stonewalling pairs accessed: 204802
231: stonewalling pairs accessed: 393487
468: stonewalling pairs accessed: 392281
285: stonewalling pairs accessed: 200933
425: stonewalling pairs accessed: 205317
199: stonewalling pairs accessed: 398442
196: stonewalling pairs accessed: 389506
541: stonewalling pairs accessed: 202136
243: stonewalling pairs accessed: 394839
33: stonewalling pairs accessed: 398737
488: stonewalling pairs accessed: 196093
185: stonewalling pairs accessed: 203633
108: stonewalling pairs accessed: 199741
372: stonewalling pairs accessed: 393365
445: stonewalling pairs accessed: 202362
453: stonewalling pairs accessed: 390318
435: stonewalling pairs accessed: 392782
436: stonewalling pairs accessed: 389518
275: stonewalling pairs accessed: 385269
295: stonewalling pairs accessed: 397611
176: stonewalling pairs accessed: 398859
440: stonewalling pairs accessed: 201892
284: stonewalling pairs accessed: 202977
432: stonewalling pairs accessed: 392086
232: stonewalling pairs accessed: 199178
3: stonewalling pairs accessed: 392671
344: stonewalling pairs accessed: 202664
167: stonewalling pairs accessed: 390055
191: stonewalling pairs accessed: 198261
22: stonewalling pairs accessed: 397003
122: stonewalling pairs accessed: 203526
186: stonewalling pairs accessed: 197911
163: stonewalling pairs accessed: 393037
215: stonewalling pairs accessed: 394001
29: stonewalling pairs accessed: 201820
442: stonewalling pairs accessed: 204016
467: stonewalling pairs accessed: 396984
460: stonewalling pairs accessed: 202132
335: stonewalling pairs accessed: 203584
100: stonewalling pairs accessed: 394078
365: stonewalling pairs accessed: 196332
527: stonewalling pairs accessed: 205801
515: stonewalling pairs accessed: 387966
424: stonewalling pairs accessed: 201759
69: stonewalling pairs accessed: 394452
174: stonewalling pairs accessed: 201254
531: stonewalling pairs accessed: 392793
56: stonewalling pairs accessed: 200275
206: stonewalling pairs accessed: 201240
259: stonewalling pairs accessed: 390560
430: stonewalling pairs accessed: 202904
379: stonewalling pairs accessed: 197050
131: stonewalling pairs accessed: 392162
400: stonewalling pairs accessed: 392168
241: stonewalling pairs accessed: 393601
507: stonewalling pairs accessed: 200392
501: stonewalling pairs accessed: 383593
245: stonewalling pairs accessed: 396361
377: stonewalling pairs accessed: 200299
32: stonewalling pairs accessed: 384825
353: stonewalling pairs accessed: 393433
67: stonewalling pairs accessed: 389704
329: stonewalling pairs accessed: 202749
198: stonewalling pairs accessed: 393309
87: stonewalling pairs accessed: 394965
278: stonewalling pairs accessed: 396993
451: stonewalling pairs accessed: 399507
52: stonewalling pairs accessed: 393646
113: stonewalling pairs accessed: 395107
119: stonewalling pairs accessed: 397544
500: stonewalling pairs accessed: 400421
319: stonewalling pairs accessed: 201707
146: stonewalling pairs accessed: 394857
271: stonewalling pairs accessed: 201571
17: stonewalling pairs accessed: 394182
50: stonewalling pairs accessed: 398822
136: stonewalling pairs accessed: 206494
499: stonewalling pairs accessed: 393535
221: stonewalling pairs accessed: 199223
154: stonewalling pairs accessed: 201828
480: stonewalling pairs accessed: 399065
282: stonewalling pairs accessed: 200230
51: stonewalling pairs accessed: 395949
512: stonewalling pairs accessed: 396753
30: stonewalling pairs accessed: 200696
371: stonewalling pairs accessed: 391155
65: stonewalling pairs accessed: 391894
31: stonewalling pairs accessed: 198887
46: stonewalling pairs accessed: 205209
340: stonewalling pairs accessed: 390855
222: stonewalling pairs accessed: 203510
139: stonewalling pairs accessed: 199851
483: stonewalling pairs accessed: 390177
200: stonewalling pairs accessed: 206430
89: stonewalling pairs accessed: 199537
518: stonewalling pairs accessed: 396610
254: stonewalling pairs accessed: 200258
187: stonewalling pairs accessed: 201530
173: stonewalling pairs accessed: 199065
341: stonewalling pairs accessed: 391902
462: stonewalling pairs accessed: 201149
526: stonewalling pairs accessed: 199000
539: stonewalling pairs accessed: 198722
153: stonewalling pairs accessed: 201429
226: stonewalling pairs accessed: 396103
455: stonewalling pairs accessed: 392508
461: stonewalling pairs accessed: 202397
363: stonewalling pairs accessed: 209476
434: stonewalling pairs accessed: 392641
180: stonewalling pairs accessed: 392932
317: stonewalling pairs accessed: 198191
234: stonewalling pairs accessed: 203718
489: stonewalling pairs accessed: 203544
197: stonewalling pairs accessed: 390039
374: stonewalling pairs accessed: 391597
224: stonewalling pairs accessed: 391622
525: stonewalling pairs accessed: 202081
419: stonewalling pairs accessed: 392574
431: stonewalling pairs accessed: 200583
247: stonewalling pairs accessed: 394382
406: stonewalling pairs accessed: 392333
55: stonewalling pairs accessed: 393770
68: stonewalling pairs accessed: 391900
112: stonewalling pairs accessed: 386101
514: stonewalling pairs accessed: 398022
274: stonewalling pairs accessed: 399585
338: stonewalling pairs accessed: 393572
188: stonewalling pairs accessed: 202565
14: stonewalling pairs accessed: 201154
349: stonewalling pairs accessed: 198529
71: stonewalling pairs accessed: 391749
394: stonewalling pairs accessed: 200028
123: stonewalling pairs accessed: 204245
135: stonewalling pairs accessed: 393969
86: stonewalling pairs accessed: 393917
255: stonewalling pairs accessed: 202034
336: stonewalling pairs accessed: 396489
2: stonewalling pairs accessed: 392333
378: stonewalling pairs accessed: 205112
38: stonewalling pairs accessed: 394069
465: stonewalling pairs accessed: 391081
310: stonewalling pairs accessed: 391003
43: stonewalling pairs accessed: 201185
497: stonewalling pairs accessed: 395694
34: stonewalling pairs accessed: 394272
375: stonewalling pairs accessed: 397158
236: stonewalling pairs accessed: 202403
134: stonewalling pairs accessed: 391557
443: stonewalling pairs accessed: 201184
211: stonewalling pairs accessed: 386517
280: stonewalling pairs accessed: 203470
359: stonewalling pairs accessed: 401596
287: stonewalling pairs accessed: 204164
262: stonewalling pairs accessed: 394574
412: stonewalling pairs accessed: 202944
301: stonewalling pairs accessed: 198612
415: stonewalling pairs accessed: 197879
294: stonewalling pairs accessed: 396332
21: stonewalling pairs accessed: 396683
84: stonewalling pairs accessed: 394181
257: stonewalling pairs accessed: 401399
305: stonewalling pairs accessed: 393973
40: stonewalling pairs accessed: 200725
151: stonewalling pairs accessed: 391873
420: stonewalling pairs accessed: 397627
529: stonewalling pairs accessed: 389081
361: stonewalling pairs accessed: 201956
485: stonewalling pairs accessed: 394845
402: stonewalling pairs accessed: 395670
207: stonewalling pairs accessed: 202526
339: stonewalling pairs accessed: 398422
144: stonewalling pairs accessed: 396137
103: stonewalling pairs accessed: 392895
385: stonewalling pairs accessed: 393845
293: stonewalling pairs accessed: 395655
484: stonewalling pairs accessed: 393317
446: stonewalling pairs accessed: 202781
503: stonewalling pairs accessed: 393967
166: stonewalling pairs accessed: 390822
528: stonewalling pairs accessed: 397998
41: stonewalling pairs accessed: 199371
297: stonewalling pairs accessed: 197701
252: stonewalling pairs accessed: 203709
486: stonewalling pairs accessed: 391963
380: stonewalling pairs accessed: 202341
413: stonewalling pairs accessed: 203896
309: stonewalling pairs accessed: 399265
142: stonewalling pairs accessed: 199873
149: stonewalling pairs accessed: 400191
421: stonewalling pairs accessed: 396401
322: stonewalling pairs accessed: 391617
59: stonewalling pairs accessed: 200400
85: stonewalling pairs accessed: 396532
251: stonewalling pairs accessed: 204083
267: stonewalling pairs accessed: 199081
209: stonewalling pairs accessed: 396395
194: stonewalling pairs accessed: 395420
315: stonewalling pairs accessed: 196540
165: stonewalling pairs accessed: 399927
79: stonewalling pairs accessed: 201808
11: stonewalling pairs accessed: 206228
19: stonewalling pairs accessed: 392059
184: stonewalling pairs accessed: 202210
439: stonewalling pairs accessed: 393073
114: stonewalling pairs accessed: 396479
459: stonewalling pairs accessed: 201473
98: stonewalling pairs accessed: 389161
502: stonewalling pairs accessed: 396373
126: stonewalling pairs accessed: 200933
387: stonewalling pairs accessed: 390123
81: stonewalling pairs accessed: 392159
506: stonewalling pairs accessed: 200153
216: stonewalling pairs accessed: 198681
476: stonewalling pairs accessed: 201466
129: stonewalling pairs accessed: 397973
508: stonewalling pairs accessed: 199254
331: stonewalling pairs accessed: 204083
542: stonewalling pairs accessed: 198587
347: stonewalling pairs accessed: 197783
152: stonewalling pairs accessed: 200237
398: stonewalling pairs accessed: 200395
318: stonewalling pairs accessed: 201895
517: stonewalling pairs accessed: 400287
291: stonewalling pairs accessed: 395459
212: stonewalling pairs accessed: 394554
62: stonewalling pairs accessed: 202246
20: stonewalling pairs accessed: 387606
472: stonewalling pairs accessed: 199069
422: stonewalling pairs accessed: 391639
364: stonewalling pairs accessed: 200105
177: stonewalling pairs accessed: 396774
376: stonewalling pairs accessed: 202965
414: stonewalling pairs accessed: 200445
118: stonewalling pairs accessed: 392272
346: stonewalling pairs accessed: 201144
496: stonewalling pairs accessed: 393634
124: stonewalling pairs accessed: 204029
1: stonewalling pairs accessed: 397974
457: stonewalling pairs accessed: 203174
332: stonewalling pairs accessed: 204596
444: stonewalling pairs accessed: 201261
395: stonewalling pairs accessed: 204839
396: stonewalling pairs accessed: 202253
437: stonewalling pairs accessed: 386866
175: stonewalling pairs accessed: 200492
63: stonewalling pairs accessed: 206324
181: stonewalling pairs accessed: 395935
393: stonewalling pairs accessed: 204264
183: stonewalling pairs accessed: 387173
452: stonewalling pairs accessed: 396358
164: stonewalling pairs accessed: 390790
42: stonewalling pairs accessed: 201881
99: stonewalling pairs accessed: 394836
321: stonewalling pairs accessed: 393810
161: stonewalling pairs accessed: 401806
381: stonewalling pairs accessed: 201443
509: stonewalling pairs accessed: 202624
426: stonewalling pairs accessed: 200106
75: stonewalling pairs accessed: 198750
522: stonewalling pairs accessed: 204079
92: stonewalling pairs accessed: 199393
77: stonewalling pairs accessed: 198934
492: stonewalling pairs accessed: 200241
13: stonewalling pairs accessed: 202066
246: stonewalling pairs accessed: 395353
540: stonewalling pairs accessed: 199457
110: stonewalling pairs accessed: 204166
125: stonewalling pairs accessed: 201663
45: stonewalling pairs accessed: 201637
536: stonewalling pairs accessed: 199700
106: stonewalling pairs accessed: 202227
218: stonewalling pairs accessed: 206020
366: stonewalling pairs accessed: 204137
524: stonewalling pairs accessed: 203860
408: stonewalling pairs accessed: 198483
111: stonewalling pairs accessed: 203472
520: stonewalling pairs accessed: 201417
399: stonewalling pairs accessed: 201257
281: stonewalling pairs accessed: 203590
447: stonewalling pairs accessed: 204190
24: stonewalling pairs accessed: 203631
91: stonewalling pairs accessed: 203258
427: stonewalling pairs accessed: 201456
237: stonewalling pairs accessed: 201986
477: stonewalling pairs accessed: 202354
159: stonewalling pairs accessed: 198842
138: stonewalling pairs accessed: 202430
239: stonewalling pairs accessed: 203068
312: stonewalling pairs accessed: 205652
458: stonewalling pairs accessed: 199507
105: stonewalling pairs accessed: 201004
12: stonewalling pairs accessed: 195876
137: stonewalling pairs accessed: 201732
60: stonewalling pairs accessed: 204329
478: stonewalling pairs accessed: 201145
stonewalling pairs accessed min: 195356 max: 403135 -- min data: 47.7 GiB mean data: 72.8 GiB time: 300.0s
WARNING: Expected aggregate file size       = 149533581377536.
WARNING: Stat() of aggregate file size      = 149364382892032.
WARNING: Using actual aggregate bytes moved = 57489605263360.
WARNING: maybe caused by deadlineForStonewalling
write     117333     475599     0.000296    268435456  256.00     6.16       461.11     0.000897   467.27     0   
Max Write: 117333.28 MiB/sec (123032.86 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      117333.28  117333.28  117333.28       0.00  469333.11  469333.11  469333.11       0.00  467.27034     300.02     135162.82     0    544  34    1   0     1        1         0    0      1 274877906944   262144 54826360.0 DFS      0
Finished            : Thu Jul  9 20:03:59 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_hard_read
[0] Pool uuid = 9020212b-0f07-4d33-8f84-6ac45cb4d7eb, SVCL = 1

[0] DFS Container namespace uuid = ce610953-295d-464c-af58-bd04279ded70

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 20:37:11 2020
Command line        : /home/kalfizah/io500-new/bin/ior -r -R -s 1600000 -a DFS --dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb --dfs.cont=ce610953-295d-464c-af58-bd04279ded70 --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse/ -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_hard/stonewall
Machine             : Linux daos08.ftm.alcf.anl.gov
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Thu Jul  9 20:37:11 2020
Participating tasks: 544
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1600000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 16
tasks               : 544
clients per node    : 34
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 37.21 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Thu Jul  9 20:37:16 2020

WARNING: Expected aggregate file size       = 40915763200000.
WARNING: Stat() of aggregate file size      = 37284131203072.
WARNING: Using actual aggregate bytes moved = 37284131203072.
read      83737      1886068    420.53      45.91      45.91      4.10       420.53     0.000983   424.63     0   
Max Read:  83736.81 MiB/sec (87804.41 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        83736.81   83736.81   83736.81       0.00 1867861.07 1867861.07 1867861.07       0.00  424.62708         NA            NA     0    544  34    1   0     1        1         0    0 1600000    47008    47008 35556920.0 DFS      0
Finished            : Thu Jul  9 20:44:16 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_hard_write
[0] Pool uuid = 9020212b-0f07-4d33-8f84-6ac45cb4d7eb, SVCL = 1

[0] DFS Container namespace uuid = ce610953-295d-464c-af58-bd04279ded70

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 20:09:23 2020
Command line        : /home/kalfizah/io500-new/bin/ior -w -s 1600000 -a DFS --dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb --dfs.cont=ce610953-295d-464c-af58-bd04279ded70 --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse/ -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux daos08.ftm.alcf.anl.gov
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Thu Jul  9 20:09:23 2020
Participating tasks: 544
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1600000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 16
tasks               : 544
clients per node    : 34
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 37.21 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Thu Jul  9 20:09:29 2020
98: stonewalling pairs accessed: 1455929
114: stonewalling pairs accessed: 1457966
295: stonewalling pairs accessed: 1454476
258: stonewalling pairs accessed: 1453624
22: stonewalling pairs accessed: 1455748
145: stonewalling pairs accessed: 1453030
311: stonewalling pairs accessed: 1453591
224: stonewalling pairs accessed: 1455971
196: stonewalling pairs accessed: 1455077
434: stonewalling pairs accessed: 1453610
37: stonewalling pairs accessed: 1456868
278: stonewalling pairs accessed: 1453432
209: stonewalling pairs accessed: 1455640
375: stonewalling pairs accessed: 1452919
352: stonewalling pairs accessed: 1451370
212: stonewalling pairs accessed: 1455683
130: stonewalling pairs accessed: 1452350
149: stonewalling pairs accessed: 1452857
374: stonewalling pairs accessed: 1452923
529: stonewalling pairs accessed: 1451179
455: stonewalling pairs accessed: 1455691
416: stonewalling pairs accessed: 1455234
146: stonewalling pairs accessed: 1452860
83: stonewalling pairs accessed: 1456489
181: stonewalling pairs accessed: 1456690
310: stonewalling pairs accessed: 1453359
273: stonewalling pairs accessed: 1453322
359: stonewalling pairs accessed: 1452300
80: stonewalling pairs accessed: 1456356
402: stonewalling pairs accessed: 1452924
147: stonewalling pairs accessed: 1452811
325: stonewalling pairs accessed: 1452688
342: stonewalling pairs accessed: 1452815
305: stonewalling pairs accessed: 1453123
423: stonewalling pairs accessed: 1455284
256: stonewalling pairs accessed: 1453571
100: stonewalling pairs accessed: 1455602
162: stonewalling pairs accessed: 1455928
275: stonewalling pairs accessed: 1453364
293: stonewalling pairs accessed: 1454475
454: stonewalling pairs accessed: 1455692
33: stonewalling pairs accessed: 1456925
503: stonewalling pairs accessed: 1451066
160: stonewalling pairs accessed: 1456284
228: stonewalling pairs accessed: 1456223
178: stonewalling pairs accessed: 1456224
51: stonewalling pairs accessed: 1457247
469: stonewalling pairs accessed: 1454369
118: stonewalling pairs accessed: 1457872
417: stonewalling pairs accessed: 1455237
391: stonewalling pairs accessed: 1452033
16: stonewalling pairs accessed: 1455427
356: stonewalling pairs accessed: 1452293
210: stonewalling pairs accessed: 1455518
243: stonewalling pairs accessed: 1453387
117: stonewalling pairs accessed: 1457972
294: stonewalling pairs accessed: 1454475
481: stonewalling pairs accessed: 1451554
183: stonewalling pairs accessed: 1457306
192: stonewalling pairs accessed: 1454661
420: stonewalling pairs accessed: 1455289
226: stonewalling pairs accessed: 1456066
403: stonewalling pairs accessed: 1452919
133: stonewalling pairs accessed: 1452356
390: stonewalling pairs accessed: 1452033
49: stonewalling pairs accessed: 1456625
23: stonewalling pairs accessed: 1455746
304: stonewalling pairs accessed: 1453123
468: stonewalling pairs accessed: 1454479
242: stonewalling pairs accessed: 1453394
371: stonewalling pairs accessed: 1452929
229: stonewalling pairs accessed: 1456330
518: stonewalling pairs accessed: 1455385
113: stonewalling pairs accessed: 1457900
71: stonewalling pairs accessed: 1456706
336: stonewalling pairs accessed: 1452723
20: stonewalling pairs accessed: 1455751
290: stonewalling pairs accessed: 1454522
499: stonewalling pairs accessed: 1450923
261: stonewalling pairs accessed: 1453727
54: stonewalling pairs accessed: 1457005
177: stonewalling pairs accessed: 1456091
103: stonewalling pairs accessed: 1455627
464: stonewalling pairs accessed: 1455200
52: stonewalling pairs accessed: 1457007
370: stonewalling pairs accessed: 1452907
195: stonewalling pairs accessed: 1454705
373: stonewalling pairs accessed: 1453066
70: stonewalling pairs accessed: 1456707
81: stonewalling pairs accessed: 1456367
263: stonewalling pairs accessed: 1453705
480: stonewalling pairs accessed: 1451430
132: stonewalling pairs accessed: 1452353
482: stonewalling pairs accessed: 1451521
323: stonewalling pairs accessed: 1452569
405: stonewalling pairs accessed: 1452931
86: stonewalling pairs accessed: 1456459
129: stonewalling pairs accessed: 1452478
343: stonewalling pairs accessed: 1452816
512: stonewalling pairs accessed: 1455605
180: stonewalling pairs accessed: 1456686
514: stonewalling pairs accessed: 1455602
19: stonewalling pairs accessed: 1455504
421: stonewalling pairs accessed: 1455288
150: stonewalling pairs accessed: 1453044
257: stonewalling pairs accessed: 1453565
519: stonewalling pairs accessed: 1455152
528: stonewalling pairs accessed: 1451005
244: stonewalling pairs accessed: 1453627
50: stonewalling pairs accessed: 1457251
355: stonewalling pairs accessed: 1452107
501: stonewalling pairs accessed: 1451203
182: stonewalling pairs accessed: 1457308
465: stonewalling pairs accessed: 1455213
55: stonewalling pairs accessed: 1457005
32: stonewalling pairs accessed: 1456930
292: stonewalling pairs accessed: 1454477
82: stonewalling pairs accessed: 1456490
531: stonewalling pairs accessed: 1451034
517: stonewalling pairs accessed: 1455601
214: stonewalling pairs accessed: 1455745
513: stonewalling pairs accessed: 1455608
87: stonewalling pairs accessed: 1456459
48: stonewalling pairs accessed: 1457179
308: stonewalling pairs accessed: 1453391
274: stonewalling pairs accessed: 1453378
67: stonewalling pairs accessed: 1456711
5: stonewalling pairs accessed: 1449833
230: stonewalling pairs accessed: 1456330
17: stonewalling pairs accessed: 1455430
151: stonewalling pairs accessed: 1453044
64: stonewalling pairs accessed: 1456561
372: stonewalling pairs accessed: 1453065
530: stonewalling pairs accessed: 1451036
163: stonewalling pairs accessed: 1455934
21: stonewalling pairs accessed: 1455508
262: stonewalling pairs accessed: 1453708
193: stonewalling pairs accessed: 1454612
327: stonewalling pairs accessed: 1452862
112: stonewalling pairs accessed: 1457900
436: stonewalling pairs accessed: 1453763
34: stonewalling pairs accessed: 1456877
259: stonewalling pairs accessed: 1453729
53: stonewalling pairs accessed: 1456655
358: stonewalling pairs accessed: 1452295
225: stonewalling pairs accessed: 1456135
439: stonewalling pairs accessed: 1453779
128: stonewalling pairs accessed: 1452476
500: stonewalling pairs accessed: 1451201
194: stonewalling pairs accessed: 1454701
483: stonewalling pairs accessed: 1451522
85: stonewalling pairs accessed: 1455993
422: stonewalling pairs accessed: 1455284
337: stonewalling pairs accessed: 1452721
471: stonewalling pairs accessed: 1454367
144: stonewalling pairs accessed: 1453032
532: stonewalling pairs accessed: 1450667
498: stonewalling pairs accessed: 1450922
35: stonewalling pairs accessed: 1456931
101: stonewalling pairs accessed: 1455606
438: stonewalling pairs accessed: 1453807
353: stonewalling pairs accessed: 1451368
119: stonewalling pairs accessed: 1457966
176: stonewalling pairs accessed: 1456093
4: stonewalling pairs accessed: 1449835
322: stonewalling pairs accessed: 1452559
227: stonewalling pairs accessed: 1456134
197: stonewalling pairs accessed: 1455082
502: stonewalling pairs accessed: 1451067
369: stonewalling pairs accessed: 1453243
231: stonewalling pairs accessed: 1456330
240: stonewalling pairs accessed: 1453379
116: stonewalling pairs accessed: 1457969
418: stonewalling pairs accessed: 1455235
419: stonewalling pairs accessed: 1455247
213: stonewalling pairs accessed: 1455685
534: stonewalling pairs accessed: 1450433
385: stonewalling pairs accessed: 1451822
7: stonewalling pairs accessed: 1449856
320: stonewalling pairs accessed: 1452699
340: stonewalling pairs accessed: 1452744
66: stonewalling pairs accessed: 1456707
99: stonewalling pairs accessed: 1455955
309: stonewalling pairs accessed: 1453444
102: stonewalling pairs accessed: 1455617
449: stonewalling pairs accessed: 1455783
215: stonewalling pairs accessed: 1455728
208: stonewalling pairs accessed: 1455636
68: stonewalling pairs accessed: 1456707
18: stonewalling pairs accessed: 1455500
211: stonewalling pairs accessed: 1455442
341: stonewalling pairs accessed: 1452749
326: stonewalling pairs accessed: 1452862
161: stonewalling pairs accessed: 1456284
39: stonewalling pairs accessed: 1456919
260: stonewalling pairs accessed: 1453634
338: stonewalling pairs accessed: 1452721
115: stonewalling pairs accessed: 1457986
389: stonewalling pairs accessed: 1452046
134: stonewalling pairs accessed: 1452479
535: stonewalling pairs accessed: 1450673
404: stonewalling pairs accessed: 1452921
306: stonewalling pairs accessed: 1453454
515: stonewalling pairs accessed: 1455608
437: stonewalling pairs accessed: 1453774
135: stonewalling pairs accessed: 1452479
388: stonewalling pairs accessed: 1452046
354: stonewalling pairs accessed: 1452102
291: stonewalling pairs accessed: 1454526
533: stonewalling pairs accessed: 1450666
339: stonewalling pairs accessed: 1452721
69: stonewalling pairs accessed: 1456707
6: stonewalling pairs accessed: 1449800
164: stonewalling pairs accessed: 1456292
131: stonewalling pairs accessed: 1452350
245: stonewalling pairs accessed: 1453505
307: stonewalling pairs accessed: 1453454
277: stonewalling pairs accessed: 1453417
435: stonewalling pairs accessed: 1453585
357: stonewalling pairs accessed: 1452298
165: stonewalling pairs accessed: 1456292
384: stonewalling pairs accessed: 1451804
148: stonewalling pairs accessed: 1453044
470: stonewalling pairs accessed: 1454192
433: stonewalling pairs accessed: 1453843
407: stonewalling pairs accessed: 1452764
458: stonewalling pairs accessed: 1454473
432: stonewalling pairs accessed: 1453843
516: stonewalling pairs accessed: 1455487
318: stonewalling pairs accessed: 1452480
478: stonewalling pairs accessed: 1450211
408: stonewalling pairs accessed: 1452834
281: stonewalling pairs accessed: 1444394
536: stonewalling pairs accessed: 1449748
485: stonewalling pairs accessed: 1451392
239: stonewalling pairs accessed: 1453248
12: stonewalling pairs accessed: 1448760
189: stonewalling pairs accessed: 1454493
463: stonewalling pairs accessed: 1454467
46: stonewalling pairs accessed: 1456524
218: stonewalling pairs accessed: 1452868
8: stonewalling pairs accessed: 1449801
175: stonewalling pairs accessed: 1453832
78: stonewalling pairs accessed: 1454215
248: stonewalling pairs accessed: 1452922
351: stonewalling pairs accessed: 1451455
110: stonewalling pairs accessed: 1455337
92: stonewalling pairs accessed: 1454869
472: stonewalling pairs accessed: 1452907
238: stonewalling pairs accessed: 1452116
188: stonewalling pairs accessed: 1453478
56: stonewalling pairs accessed: 1456409
187: stonewalling pairs accessed: 1454142
167: stonewalling pairs accessed: 1456316
451: stonewalling pairs accessed: 1455751
414: stonewalling pairs accessed: 1453834
395: stonewalling pairs accessed: 1451353
126: stonewalling pairs accessed: 1449844
106: stonewalling pairs accessed: 1455419
523: stonewalling pairs accessed: 1448151
446: stonewalling pairs accessed: 1454229
250: stonewalling pairs accessed: 1447719
450: stonewalling pairs accessed: 1455527
30: stonewalling pairs accessed: 1456738
394: stonewalling pairs accessed: 1451480
13: stonewalling pairs accessed: 1449408
286: stonewalling pairs accessed: 1452251
74: stonewalling pairs accessed: 1454852
24: stonewalling pairs accessed: 1451983
453: stonewalling pairs accessed: 1455751
29: stonewalling pairs accessed: 1438126
15: stonewalling pairs accessed: 1452475
350: stonewalling pairs accessed: 1451360
153: stonewalling pairs accessed: 1452882
140: stonewalling pairs accessed: 1451734
90: stonewalling pairs accessed: 1454716
484: stonewalling pairs accessed: 1451395
216: stonewalling pairs accessed: 1454149
166: stonewalling pairs accessed: 1456316
61: stonewalling pairs accessed: 1456153
487: stonewalling pairs accessed: 1451395
31: stonewalling pairs accessed: 1456745
14: stonewalling pairs accessed: 1448765
169: stonewalling pairs accessed: 1447088
76: stonewalling pairs accessed: 1454299
138: stonewalling pairs accessed: 1451700
328: stonewalling pairs accessed: 1452148
486: stonewalling pairs accessed: 1451395
109: stonewalling pairs accessed: 1454084
47: stonewalling pairs accessed: 1456982
222: stonewalling pairs accessed: 1452695
377: stonewalling pairs accessed: 1451245
284: stonewalling pairs accessed: 1450430
154: stonewalling pairs accessed: 1441570
456: stonewalling pairs accessed: 1453949
173: stonewalling pairs accessed: 1456109
95: stonewalling pairs accessed: 1455155
382: stonewalling pairs accessed: 1447167
521: stonewalling pairs accessed: 1451201
172: stonewalling pairs accessed: 1456109
202: stonewalling pairs accessed: 1453636
504: stonewalling pairs accessed: 1450832
205: stonewalling pairs accessed: 1455177
143: stonewalling pairs accessed: 1451860
510: stonewalling pairs accessed: 1449831
537: stonewalling pairs accessed: 1449755
268: stonewalling pairs accessed: 1452733
266: stonewalling pairs accessed: 1453429
520: stonewalling pairs accessed: 1451236
253: stonewalling pairs accessed: 1452376
303: stonewalling pairs accessed: 1453133
430: stonewalling pairs accessed: 1453602
57: stonewalling pairs accessed: 1456418
204: stonewalling pairs accessed: 1455178
362: stonewalling pairs accessed: 1452317
72: stonewalling pairs accessed: 1454881
269: stonewalling pairs accessed: 1452754
319: stonewalling pairs accessed: 1452668
62: stonewalling pairs accessed: 1456432
185: stonewalling pairs accessed: 1452768
316: stonewalling pairs accessed: 1452522
426: stonewalling pairs accessed: 1453601
120: stonewalling pairs accessed: 1452495
301: stonewalling pairs accessed: 1453101
367: stonewalling pairs accessed: 1452788
94: stonewalling pairs accessed: 1455986
249: stonewalling pairs accessed: 1442364
460: stonewalling pairs accessed: 1454876
442: stonewalling pairs accessed: 1448313
152: stonewalling pairs accessed: 1433093
317: stonewalling pairs accessed: 1452674
511: stonewalling pairs accessed: 1449870
313: stonewalling pairs accessed: 1450359
492: stonewalling pairs accessed: 1449140
10: stonewalling pairs accessed: 1449519
168: stonewalling pairs accessed: 1455785
365: stonewalling pairs accessed: 1452193
527: stonewalling pairs accessed: 1451003
441: stonewalling pairs accessed: 1453527
60: stonewalling pairs accessed: 1456417
234: stonewalling pairs accessed: 1452855
184: stonewalling pairs accessed: 1454082
381: stonewalling pairs accessed: 1450978
63: stonewalling pairs accessed: 1455441
473: stonewalling pairs accessed: 1450338
364: stonewalling pairs accessed: 1452181
298: stonewalling pairs accessed: 1453101
264: stonewalling pairs accessed: 1453692
429: stonewalling pairs accessed: 1452434
79: stonewalling pairs accessed: 1456023
25: stonewalling pairs accessed: 1455546
380: stonewalling pairs accessed: 1449180
314: stonewalling pairs accessed: 1451693
280: stonewalling pairs accessed: 1445758
445: stonewalling pairs accessed: 1453320
111: stonewalling pairs accessed: 1455560
142: stonewalling pairs accessed: 1451823
121: stonewalling pairs accessed: 1452497
396: stonewalling pairs accessed: 1451481
490: stonewalling pairs accessed: 1449506
296: stonewalling pairs accessed: 1452766
461: stonewalling pairs accessed: 1455107
127: stonewalling pairs accessed: 1451107
174: stonewalling pairs accessed: 1452574
233: stonewalling pairs accessed: 1455200
108: stonewalling pairs accessed: 1454070
522: stonewalling pairs accessed: 1447691
312: stonewalling pairs accessed: 1451931
525: stonewalling pairs accessed: 1449775
191: stonewalling pairs accessed: 1453860
206: stonewalling pairs accessed: 1453633
361: stonewalling pairs accessed: 1452134
156: stonewalling pairs accessed: 1449932
122: stonewalling pairs accessed: 1450993
360: stonewalling pairs accessed: 1452031
93: stonewalling pairs accessed: 1455130
223: stonewalling pairs accessed: 1452750
254: stonewalling pairs accessed: 1433815
393: stonewalling pairs accessed: 1451554
348: stonewalling pairs accessed: 1451469
282: stonewalling pairs accessed: 1450731
376: stonewalling pairs accessed: 1451249
125: stonewalling pairs accessed: 1449857
271: stonewalling pairs accessed: 1452327
270: stonewalling pairs accessed: 1452312
425: stonewalling pairs accessed: 1454584
476: stonewalling pairs accessed: 1450211
346: stonewalling pairs accessed: 1450605
392: stonewalling pairs accessed: 1451560
141: stonewalling pairs accessed: 1452436
287: stonewalling pairs accessed: 1452996
302: stonewalling pairs accessed: 1452048
505: stonewalling pairs accessed: 1451049
508: stonewalling pairs accessed: 1450614
474: stonewalling pairs accessed: 1443637
424: stonewalling pairs accessed: 1455252
157: stonewalling pairs accessed: 1452935
399: stonewalling pairs accessed: 1451911
366: stonewalling pairs accessed: 1452790
41: stonewalling pairs accessed: 1456625
124: stonewalling pairs accessed: 1452436
410: stonewalling pairs accessed: 1453069
440: stonewalling pairs accessed: 1453629
349: stonewalling pairs accessed: 1451794
415: stonewalling pairs accessed: 1453836
398: stonewalling pairs accessed: 1451995
329: stonewalling pairs accessed: 1452147
252: stonewalling pairs accessed: 1451705
58: stonewalling pairs accessed: 1456482
488: stonewalling pairs accessed: 1450006
397: stonewalling pairs accessed: 1451919
431: stonewalling pairs accessed: 1453859
462: stonewalling pairs accessed: 1454843
409: stonewalling pairs accessed: 1452777
524: stonewalling pairs accessed: 1449778
42: stonewalling pairs accessed: 1456934
104: stonewalling pairs accessed: 1455381
493: stonewalling pairs accessed: 1449968
447: stonewalling pairs accessed: 1455361
494: stonewalling pairs accessed: 1449239
9: stonewalling pairs accessed: 1449807
236: stonewalling pairs accessed: 1452971
136: stonewalling pairs accessed: 1449038
509: stonewalling pairs accessed: 1450614
495: stonewalling pairs accessed: 1449225
190: stonewalling pairs accessed: 1453848
137: stonewalling pairs accessed: 1451412
220: stonewalling pairs accessed: 1455509
344: stonewalling pairs accessed: 1452366
285: stonewalling pairs accessed: 1451073
207: stonewalling pairs accessed: 1454595
158: stonewalling pairs accessed: 1454818
265: stonewalling pairs accessed: 1453692
237: stonewalling pairs accessed: 1453089
255: stonewalling pairs accessed: 1453374
105: stonewalling pairs accessed: 1455381
477: stonewalling pairs accessed: 1450211
479: stonewalling pairs accessed: 1442748
297: stonewalling pairs accessed: 1453106
221: stonewalling pairs accessed: 1455509
159: stonewalling pairs accessed: 1454818
11: stonewalling pairs accessed: 1449419
27: stonewalling pairs accessed: 1455471
43: stonewalling pairs accessed: 1456934
59: stonewalling pairs accessed: 1436844
75: stonewalling pairs accessed: 1455740
91: stonewalling pairs accessed: 1455488
107: stonewalling pairs accessed: 1455414
123: stonewalling pairs accessed: 1452331
155: stonewalling pairs accessed: 1452885
203: stonewalling pairs accessed: 1453366
219: stonewalling pairs accessed: 1452684
251: stonewalling pairs accessed: 1447724
267: stonewalling pairs accessed: 1453015
283: stonewalling pairs accessed: 1453379
299: stonewalling pairs accessed: 1452074
315: stonewalling pairs accessed: 1450356
331: stonewalling pairs accessed: 1452244
363: stonewalling pairs accessed: 1452298
379: stonewalling pairs accessed: 1449216
411: stonewalling pairs accessed: 1453040
427: stonewalling pairs accessed: 1451118
443: stonewalling pairs accessed: 1453855
459: stonewalling pairs accessed: 1451735
475: stonewalling pairs accessed: 1450750
491: stonewalling pairs accessed: 1449163
507: stonewalling pairs accessed: 1448957
73: stonewalling pairs accessed: 1455091
457: stonewalling pairs accessed: 1455480
489: stonewalling pairs accessed: 1449285
444: stonewalling pairs accessed: 1450523
543: stonewalling pairs accessed: 1449664
467: stonewalling pairs accessed: 1454740
466: stonewalling pairs accessed: 1454740
44: stonewalling pairs accessed: 1456597
387: stonewalling pairs accessed: 1452041
538: stonewalling pairs accessed: 1449664
241: stonewalling pairs accessed: 1453376
289: stonewalling pairs accessed: 1454515
45: stonewalling pairs accessed: 1456597
378: stonewalling pairs accessed: 1448866
26: stonewalling pairs accessed: 1455526
186: stonewalling pairs accessed: 1454284
321: stonewalling pairs accessed: 1452681
324: stonewalling pairs accessed: 1452681
279: stonewalling pairs accessed: 1453415
330: stonewalling pairs accessed: 1450944
506: stonewalling pairs accessed: 1451057
345: stonewalling pairs accessed: 1452349
541: stonewalling pairs accessed: 1449664
36: stonewalling pairs accessed: 1456916
539: stonewalling pairs accessed: 1449664
276: stonewalling pairs accessed: 1453415
246: stonewalling pairs accessed: 1453597
300: stonewalling pairs accessed: 1452866
38: stonewalling pairs accessed: 1456916
413: stonewalling pairs accessed: 1453405
288: stonewalling pairs accessed: 1454515
217: stonewalling pairs accessed: 1454294
2: stonewalling pairs accessed: 1449670
65: stonewalling pairs accessed: 1456705
97: stonewalling pairs accessed: 1456156
272: stonewalling pairs accessed: 1453619
386: stonewalling pairs accessed: 1452041
526: stonewalling pairs accessed: 1449979
170: stonewalling pairs accessed: 1452642
89: stonewalling pairs accessed: 1456226
232: stonewalling pairs accessed: 1454805
247: stonewalling pairs accessed: 1453597
412: stonewalling pairs accessed: 1453405
40: stonewalling pairs accessed: 1456648
28: stonewalling pairs accessed: 1440204
428: stonewalling pairs accessed: 1451101
88: stonewalling pairs accessed: 1456226
139: stonewalling pairs accessed: 1449001
96: stonewalling pairs accessed: 1456156
3: stonewalling pairs accessed: 1449670
77: stonewalling pairs accessed: 1455971
448: stonewalling pairs accessed: 1456217
335: stonewalling pairs accessed: 1450944
401: stonewalling pairs accessed: 1452732
0: stonewalling pairs accessed: 1449670
333: stonewalling pairs accessed: 1450734
406: stonewalling pairs accessed: 1452957
1: stonewalling pairs accessed: 1449670
400: stonewalling pairs accessed: 1452732
347: stonewalling pairs accessed: 1450851
496: stonewalling pairs accessed: 1450810
198: stonewalling pairs accessed: 1455061
497: stonewalling pairs accessed: 1450810
334: stonewalling pairs accessed: 1450944
235: stonewalling pairs accessed: 1453074
171: stonewalling pairs accessed: 1455035
179: stonewalling pairs accessed: 1456601
84: stonewalling pairs accessed: 1456490
332: stonewalling pairs accessed: 1450734
452: stonewalling pairs accessed: 1455764
199: stonewalling pairs accessed: 1455061
540: stonewalling pairs accessed: 1449669
368: stonewalling pairs accessed: 1453245
200: stonewalling pairs accessed: 1454468
383: stonewalling pairs accessed: 1448622
542: stonewalling pairs accessed: 1449669
201: stonewalling pairs accessed: 1454468
stonewalling pairs accessed min: 1433093 max: 1457986 -- min data: 62.7 GiB mean data: 63.6 GiB time: 300.0s
WARNING: Expected aggregate file size       = 40915763200000.
WARNING: Stat() of aggregate file size      = 37284131203072.
WARNING: Using actual aggregate bytes moved = 37284131203072.
WARNING: maybe caused by deadlineForStonewalling
write     114967     2618767    301.20      45.91      45.91      6.41       302.87     0.000755   309.28     0   
Max Write: 114967.47 MiB/sec (120552.13 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      114967.47  114967.47  114967.47       0.00 2564502.48 2564502.48 2564502.48       0.00  309.27807     300.01     118126.10     0    544  34    1   0     1        1         0    0 1600000    47008    47008 35556920.0 DFS      0
Finished            : Thu Jul  9 20:14:32 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

mdtest_easy_delete
-- started at 07/09/2020 20:45:48 --

mdtest-3.3.0+dev was launched with 544 total task(s) on 16 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-r' '-F' '-P' '-d' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_easy' '-n' '4000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb' '--dfs.cont=ce610953-295d-464c-af58-bd04279ded70' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-x' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_easy-stonewall' '-N' '1'
Nodemap: 1000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
544 tasks, 2176000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    2559271.954    2559270.113    2559271.016          0.521
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.071          0.071          0.071          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        524.387        524.387        524.387          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         14.058         14.058         14.058          0.000
-- finished at 07/09/2020 20:54:46 --

mdtest_easy_stat
-- started at 07/09/2020 20:32:07 --

mdtest-3.3.0+dev was launched with 544 total task(s) on 16 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-T' '-F' '-P' '-d' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_easy' '-n' '4000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb' '--dfs.cont=ce610953-295d-464c-af58-bd04279ded70' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-x' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_easy-stonewall' '-N' '1'
Nodemap: 1000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
544 tasks, 2176000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    4521232.418    4521224.552    4521228.185          2.080
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        296.833        296.833        296.833          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 20:37:04 --

mdtest_easy_write
-- started at 07/09/2020 20:04:06 --

mdtest-3.3.0+dev was launched with 544 total task(s) on 16 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-Y' '-C' '-F' '-P' '-d' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_easy' '-n' '4000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb' '--dfs.cont=ce610953-295d-464c-af58-bd04279ded70' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-x' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Nodemap: 1000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
544 tasks, 2176000000 files
Continue stonewall hit min: 2361919 max: 2467002 avg: 2445107.5 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    4386195.871    4386190.146    4386192.871          1.156
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    4477196.823             NA
   Tree creation             :          0.346          0.346          0.346          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        305.971        305.971        305.971          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        297.092             NA
   Tree creation             :          2.893          2.893          2.893          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 20:09:15 --

mdtest_hard_delete
-- started at 07/09/2020 20:57:19 --

mdtest-3.3.0+dev was launched with 544 total task(s) on 16 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_hard' '-n' '1500000' '-x' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb' '--dfs.cont=ce610953-295d-464c-af58-bd04279ded70' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-N' '1'
Nodemap: 1000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
544 tasks, 816000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    2357156.114    2357151.676    2357153.985          1.354
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.001          0.001          0.001          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        122.083        122.082        122.083          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :        722.611        722.611        722.611          0.000
-- finished at 07/09/2020 21:11:24 --

mdtest_hard_read
-- started at 07/09/2020 20:54:54 --

mdtest-3.3.0+dev was launched with 544 total task(s) on 16 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_hard' '-n' '1500000' '-x' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb' '--dfs.cont=ce610953-295d-464c-af58-bd04279ded70' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-N' '1'
Nodemap: 1000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
544 tasks, 816000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :    2100107.866    2100105.947    2100106.875          0.545
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        137.025        137.025        137.025          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 20:57:11 --

mdtest_hard_stat
-- started at 07/09/2020 20:44:24 --

mdtest-3.3.0+dev was launched with 544 total task(s) on 16 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_hard' '-n' '1500000' '-x' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb' '--dfs.cont=ce610953-295d-464c-af58-bd04279ded70' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-N' '1'
Nodemap: 1000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
544 tasks, 816000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    3730500.642    3730489.988    3730494.597          2.795
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         77.139         77.139         77.139          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 20:45:41 --

mdtest_hard_write
-- started at 07/09/2020 20:14:40 --

mdtest-3.3.0+dev was launched with 544 total task(s) on 16 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-Y' '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_hard' '-n' '1500000' '-x' '/tmp/dfuse/datafiles/2020.07.09-19.56.01-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=9020212b-0f07-4d33-8f84-6ac45cb4d7eb' '--dfs.cont=ce610953-295d-464c-af58-bd04279ded70' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-N' '1' '-W' '300'
Nodemap: 1000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
544 tasks, 816000000 files
Continue stonewall hit min: 497510 max: 528984 avg: 508772.8 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     924088.116     924087.087     924087.571          0.293
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     922605.407             NA
   Tree creation             :        208.164        208.164        208.164          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        311.407        311.407        311.407          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.990             NA
   Tree creation             :          0.005          0.005          0.005          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 20:19:51 --

result_summary
[RESULT] BW   phase 1            ior_easy_write              114.583 GiB/s : time 461.11 seconds
[RESULT] IOPS phase 1         mdtest_easy_write             4386.200 kiops : time 305.97 seconds
[RESULT] BW   phase 2            ior_hard_write              112.272 GiB/s : time 302.87 seconds
[RESULT] IOPS phase 2         mdtest_hard_write              924.088 kiops : time 311.41 seconds
[RESULT] IOPS phase 3                      find             2969.300 kiops : time 310.54 seconds
[RESULT] BW   phase 3             ior_easy_read              130.230 GiB/s : time 407.09 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             4521.230 kiops : time 296.83 seconds
[RESULT] BW   phase 4             ior_hard_read               81.774 GiB/s : time 420.53 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat             3730.500 kiops : time  77.14 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete             2559.270 kiops : time 524.39 seconds
[RESULT] IOPS phase 7          mdtest_hard_read             2100.110 kiops : time 137.02 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete             2357.160 kiops : time 852.81 seconds
[SCORE] Bandwidth 108.188 GiB/s : IOPS 2668.57 kiops : TOTAL 537.315