Frontera

Institution TACC
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.8
Client Kernel Version 3.10.0-1127.13.1.el7.x86_64

DATA SERVER

Storage Type Optane Persistent Memory
Volatile Memory 256GB
Storage Interface DDR-T
Network Infiniband
Software Version 1.1
OS Version 7.8

INFORMATION

Client Nodes 10
Client Total Procs 420

METADATA

Easy Write 5,151.86 kIOP/s
Easy Stat 4,672.87 kIOP/s
Easy Delete 2,788.83 kIOP/s
Hard Write 2,343.71 kIOP/s
Hard Read 3,215.92 kIOP/s
Hard Stat 4,081.08 kIOP/s
Hard Delete 2,716.13 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=/home1/06758/mschaara/io-500/install/ior/bin/ior
  io500_mdtest_cmd=/home1/06758/mschaara/io-500/install/ior/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 420 --hostfile /home1/06758/mschaara/io-500/config/cli_hosts_10"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

#  if [[ -s "system-information.txt" ]]; then
#    echo "Warning: please create a system-information.txt description by"
#    echo "copying the information from https://vi4io.org/io500-info-creator/"
#  else
#    cp "system-information.txt" $io500_result_dir
#  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val" && io500_find_mpi="True"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-args) -o $io500_result_dir/find_output.txt"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  echo "Creating tarball"
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
[0] Pool uuid = 372e1597-1708-4f1e-aa78-1cceb4c7c6ec, SVCL = 1

[0] DFS Container namespace uuid = af091f54-7e7d-4c6b-bc70-a5d7d16e4955

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Jul 11 15:09:19 2020
Command line        : /home1/06758/mschaara/io-500/install/ior/bin/ior -r -R -a DFS --dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec --dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955 --dfs.svcl=1 --dfs.prefix=/tmp/dfuse -t 256k -v -b 1000000m -i 1 -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_easy/stonewall
Machine             : Linux c110-001.frontera.tacc.utexas.edu
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Sat Jul 11 15:09:19 2020
Participating tasks: 420
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 420
tasks               : 420
clients per node    : 1
repetitions         : 1
xfersize            : 262144 bytes
blocksize           : 976.56 GiB
aggregate filesize  : 400.54 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Sat Jul 11 15:09:22 2020

WARNING: Expected aggregate file size       = 440401920000000.
WARNING: Stat() of aggregate file size      = 439444110311424.
WARNING: Using actual aggregate bytes moved = 38121850798080.
read      74350      299180     0.000122    1024000000 256.00     2.89       486.07     0.014577   488.98     0   
Max Read:  74350.42 MiB/sec (77962.07 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        74350.42   74350.42   74350.42       0.00  297401.69  297401.69  297401.69       0.00  488.97947         NA            NA     0    420   1    1   0     1        1         0    0      1 1048576000000   262144 36355832.0 DFS      0
Finished            : Sat Jul 11 15:17:29 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_easy_write
[0] Pool uuid = 372e1597-1708-4f1e-aa78-1cceb4c7c6ec, SVCL = 1

[0] DFS Container namespace uuid = af091f54-7e7d-4c6b-bc70-a5d7d16e4955

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Jul 11 14:40:19 2020
Command line        : /home1/06758/mschaara/io-500/install/ior/bin/ior -w -a DFS --dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec --dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955 --dfs.svcl=1 --dfs.prefix=/tmp/dfuse -t 256k -v -b 1000000m -i 1 -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux c110-001.frontera.tacc.utexas.edu
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Sat Jul 11 14:40:19 2020
Participating tasks: 420
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 420
tasks               : 420
clients per node    : 1
repetitions         : 1
xfersize            : 262144 bytes
blocksize           : 976.56 GiB
aggregate filesize  : 400.54 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Sat Jul 11 14:40:25 2020
296: stonewalling pairs accessed: 284030
76: stonewalling pairs accessed: 284125
316: stonewalling pairs accessed: 283787
65: stonewalling pairs accessed: 322386
215: stonewalling pairs accessed: 323898
231: stonewalling pairs accessed: 281569
123: stonewalling pairs accessed: 305093
265: stonewalling pairs accessed: 323323
83: stonewalling pairs accessed: 304798
395: stonewalling pairs accessed: 323873
153: stonewalling pairs accessed: 302892
300: stonewalling pairs accessed: 282707
75: stonewalling pairs accessed: 322839
133: stonewalling pairs accessed: 302797
233: stonewalling pairs accessed: 305021
103: stonewalling pairs accessed: 302869
195: stonewalling pairs accessed: 322419
93: stonewalling pairs accessed: 305473
310: stonewalling pairs accessed: 280525
355: stonewalling pairs accessed: 322209
273: stonewalling pairs accessed: 305676
113: stonewalling pairs accessed: 304938
283: stonewalling pairs accessed: 305500
279: stonewalling pairs accessed: 327737
89: stonewalling pairs accessed: 327913
49: stonewalling pairs accessed: 328725
289: stonewalling pairs accessed: 327893
259: stonewalling pairs accessed: 327782
39: stonewalling pairs accessed: 327851
119: stonewalling pairs accessed: 327848
179: stonewalling pairs accessed: 327762
139: stonewalling pairs accessed: 327814
189: stonewalling pairs accessed: 327242
219: stonewalling pairs accessed: 328263
169: stonewalling pairs accessed: 328148
129: stonewalling pairs accessed: 328157
109: stonewalling pairs accessed: 328032
269: stonewalling pairs accessed: 327601
239: stonewalling pairs accessed: 325664
299: stonewalling pairs accessed: 325976
354: stonewalling pairs accessed: 320778
229: stonewalling pairs accessed: 328349
134: stonewalling pairs accessed: 322824
199: stonewalling pairs accessed: 325056
384: stonewalling pairs accessed: 322931
174: stonewalling pairs accessed: 322932
264: stonewalling pairs accessed: 320260
194: stonewalling pairs accessed: 320429
94: stonewalling pairs accessed: 323787
304: stonewalling pairs accessed: 322535
146: stonewalling pairs accessed: 284849
82: stonewalling pairs accessed: 305641
32: stonewalling pairs accessed: 303006
206: stonewalling pairs accessed: 284364
104: stonewalling pairs accessed: 319757
154: stonewalling pairs accessed: 322604
14: stonewalling pairs accessed: 323290
281: stonewalling pairs accessed: 283549
274: stonewalling pairs accessed: 319474
34: stonewalling pairs accessed: 323200
285: stonewalling pairs accessed: 322881
373: stonewalling pairs accessed: 303172
24: stonewalling pairs accessed: 323120
74: stonewalling pairs accessed: 321057
144: stonewalling pairs accessed: 323615
64: stonewalling pairs accessed: 323214
314: stonewalling pairs accessed: 322795
386: stonewalling pairs accessed: 283710
52: stonewalling pairs accessed: 302671
240: stonewalling pairs accessed: 282952
272: stonewalling pairs accessed: 302965
124: stonewalling pairs accessed: 322989
294: stonewalling pairs accessed: 323103
412: stonewalling pairs accessed: 306366
366: stonewalling pairs accessed: 282042
12: stonewalling pairs accessed: 306904
252: stonewalling pairs accessed: 305049
202: stonewalling pairs accessed: 305275
42: stonewalling pairs accessed: 302390
313: stonewalling pairs accessed: 305285
182: stonewalling pairs accessed: 303029
322: stonewalling pairs accessed: 305120
342: stonewalling pairs accessed: 303387
382: stonewalling pairs accessed: 305496
162: stonewalling pairs accessed: 306705
303: stonewalling pairs accessed: 302957
112: stonewalling pairs accessed: 305498
351: stonewalling pairs accessed: 285197
4: stonewalling pairs accessed: 320171
396: stonewalling pairs accessed: 281794
132: stonewalling pairs accessed: 305806
270: stonewalling pairs accessed: 282577
192: stonewalling pairs accessed: 306526
212: stonewalling pairs accessed: 303077
62: stonewalling pairs accessed: 306916
142: stonewalling pairs accessed: 302407
232: stonewalling pairs accessed: 303867
262: stonewalling pairs accessed: 303214
2: stonewalling pairs accessed: 306541
22: stonewalling pairs accessed: 305417
302: stonewalling pairs accessed: 302454
312: stonewalling pairs accessed: 305792
372: stonewalling pairs accessed: 305100
402: stonewalling pairs accessed: 306031
362: stonewalling pairs accessed: 306005
282: stonewalling pairs accessed: 302418
332: stonewalling pairs accessed: 306496
92: stonewalling pairs accessed: 305539
222: stonewalling pairs accessed: 306644
292: stonewalling pairs accessed: 302710
164: stonewalling pairs accessed: 320378
352: stonewalling pairs accessed: 303128
102: stonewalling pairs accessed: 305645
242: stonewalling pairs accessed: 306832
122: stonewalling pairs accessed: 306189
72: stonewalling pairs accessed: 303193
152: stonewalling pairs accessed: 307142
392: stonewalling pairs accessed: 307259
172: stonewalling pairs accessed: 303576
385: stonewalling pairs accessed: 321901
66: stonewalling pairs accessed: 283843
126: stonewalling pairs accessed: 284212
414: stonewalling pairs accessed: 324574
284: stonewalling pairs accessed: 324724
344: stonewalling pairs accessed: 324301
184: stonewalling pairs accessed: 321787
84: stonewalling pairs accessed: 324452
214: stonewalling pairs accessed: 324614
204: stonewalling pairs accessed: 324411
234: stonewalling pairs accessed: 324460
301: stonewalling pairs accessed: 283857
54: stonewalling pairs accessed: 320544
311: stonewalling pairs accessed: 283956
325: stonewalling pairs accessed: 322171
293: stonewalling pairs accessed: 306249
254: stonewalling pairs accessed: 324534
349: stonewalling pairs accessed: 329482
291: stonewalling pairs accessed: 282125
224: stonewalling pairs accessed: 320501
226: stonewalling pairs accessed: 283971
359: stonewalling pairs accessed: 329598
404: stonewalling pairs accessed: 320891
374: stonewalling pairs accessed: 324624
145: stonewalling pairs accessed: 321828
63: stonewalling pairs accessed: 305696
394: stonewalling pairs accessed: 323999
276: stonewalling pairs accessed: 284014
369: stonewalling pairs accessed: 325719
280: stonewalling pairs accessed: 280574
114: stonewalling pairs accessed: 320927
334: stonewalling pairs accessed: 321136
19: stonewalling pairs accessed: 328857
244: stonewalling pairs accessed: 320954
185: stonewalling pairs accessed: 321589
9: stonewalling pairs accessed: 325684
230: stonewalling pairs accessed: 280781
44: stonewalling pairs accessed: 324465
309: stonewalling pairs accessed: 329926
250: stonewalling pairs accessed: 283673
16: stonewalling pairs accessed: 283099
29: stonewalling pairs accessed: 329437
236: stonewalling pairs accessed: 281935
305: stonewalling pairs accessed: 319507
324: stonewalling pairs accessed: 324687
159: stonewalling pairs accessed: 329023
379: stonewalling pairs accessed: 329611
350: stonewalling pairs accessed: 282512
364: stonewalling pairs accessed: 324721
156: stonewalling pairs accessed: 284089
99: stonewalling pairs accessed: 328968
69: stonewalling pairs accessed: 324980
329: stonewalling pairs accessed: 329115
339: stonewalling pairs accessed: 325085
278: stonewalling pairs accessed: 304835
399: stonewalling pairs accessed: 330109
319: stonewalling pairs accessed: 329624
79: stonewalling pairs accessed: 329245
419: stonewalling pairs accessed: 329373
120: stonewalling pairs accessed: 282386
249: stonewalling pairs accessed: 329767
405: stonewalling pairs accessed: 322060
59: stonewalling pairs accessed: 329240
409: stonewalling pairs accessed: 329361
209: stonewalling pairs accessed: 328812
110: stonewalling pairs accessed: 280372
149: stonewalling pairs accessed: 325382
390: stonewalling pairs accessed: 283437
389: stonewalling pairs accessed: 329255
406: stonewalling pairs accessed: 283609
191: stonewalling pairs accessed: 284641
181: stonewalling pairs accessed: 281895
117: stonewalling pairs accessed: 281555
306: stonewalling pairs accessed: 281852
330: stonewalling pairs accessed: 281883
125: stonewalling pairs accessed: 322826
13: stonewalling pairs accessed: 307006
343: stonewalling pairs accessed: 305345
375: stonewalling pairs accessed: 323539
121: stonewalling pairs accessed: 284128
163: stonewalling pairs accessed: 306648
243: stonewalling pairs accessed: 306742
55: stonewalling pairs accessed: 320278
33: stonewalling pairs accessed: 306169
43: stonewalling pairs accessed: 303744
253: stonewalling pairs accessed: 306873
383: stonewalling pairs accessed: 306197
393: stonewalling pairs accessed: 302819
346: stonewalling pairs accessed: 284848
333: stonewalling pairs accessed: 305324
196: stonewalling pairs accessed: 281649
363: stonewalling pairs accessed: 306471
26: stonewalling pairs accessed: 281543
45: stonewalling pairs accessed: 324450
193: stonewalling pairs accessed: 303632
7: stonewalling pairs accessed: 285107
56: stonewalling pairs accessed: 281927
165: stonewalling pairs accessed: 319982
23: stonewalling pairs accessed: 305062
166: stonewalling pairs accessed: 285394
175: stonewalling pairs accessed: 323819
353: stonewalling pairs accessed: 306675
15: stonewalling pairs accessed: 322870
403: stonewalling pairs accessed: 306550
53: stonewalling pairs accessed: 306633
266: stonewalling pairs accessed: 285262
323: stonewalling pairs accessed: 306702
6: stonewalling pairs accessed: 285083
105: stonewalling pairs accessed: 319944
221: stonewalling pairs accessed: 284149
203: stonewalling pairs accessed: 302678
255: stonewalling pairs accessed: 323664
413: stonewalling pairs accessed: 302467
46: stonewalling pairs accessed: 282353
223: stonewalling pairs accessed: 306057
326: stonewalling pairs accessed: 281881
263: stonewalling pairs accessed: 306440
286: stonewalling pairs accessed: 284869
86: stonewalling pairs accessed: 285019
85: stonewalling pairs accessed: 323455
381: stonewalling pairs accessed: 285526
183: stonewalling pairs accessed: 304565
246: stonewalling pairs accessed: 285040
408: stonewalling pairs accessed: 303064
365: stonewalling pairs accessed: 324020
161: stonewalling pairs accessed: 281249
40: stonewalling pairs accessed: 283869
213: stonewalling pairs accessed: 303090
256: stonewalling pairs accessed: 281821
368: stonewalling pairs accessed: 306627
35: stonewalling pairs accessed: 323939
41: stonewalling pairs accessed: 284708
106: stonewalling pairs accessed: 285523
225: stonewalling pairs accessed: 324253
143: stonewalling pairs accessed: 306537
176: stonewalling pairs accessed: 285773
25: stonewalling pairs accessed: 323328
411: stonewalling pairs accessed: 285607
116: stonewalling pairs accessed: 285410
135: stonewalling pairs accessed: 323837
21: stonewalling pairs accessed: 285035
90: stonewalling pairs accessed: 283640
3: stonewalling pairs accessed: 306029
186: stonewalling pairs accessed: 285199
328: stonewalling pairs accessed: 303142
95: stonewalling pairs accessed: 319696
201: stonewalling pairs accessed: 285233
357: stonewalling pairs accessed: 285301
356: stonewalling pairs accessed: 281282
248: stonewalling pairs accessed: 303032
335: stonewalling pairs accessed: 324064
361: stonewalling pairs accessed: 284768
96: stonewalling pairs accessed: 284357
345: stonewalling pairs accessed: 323893
167: stonewalling pairs accessed: 285463
336: stonewalling pairs accessed: 284591
5: stonewalling pairs accessed: 324049
51: stonewalling pairs accessed: 285034
70: stonewalling pairs accessed: 284104
137: stonewalling pairs accessed: 282227
416: stonewalling pairs accessed: 283495
155: stonewalling pairs accessed: 323345
360: stonewalling pairs accessed: 284297
87: stonewalling pairs accessed: 283283
18: stonewalling pairs accessed: 306306
80: stonewalling pairs accessed: 283368
367: stonewalling pairs accessed: 285220
418: stonewalling pairs accessed: 306215
11: stonewalling pairs accessed: 284963
190: stonewalling pairs accessed: 282568
218: stonewalling pairs accessed: 306343
115: stonewalling pairs accessed: 319660
331: stonewalling pairs accessed: 282310
60: stonewalling pairs accessed: 282682
67: stonewalling pairs accessed: 285059
28: stonewalling pairs accessed: 303234
235: stonewalling pairs accessed: 324005
410: stonewalling pairs accessed: 283158
173: stonewalling pairs accessed: 306104
37: stonewalling pairs accessed: 283667
208: stonewalling pairs accessed: 305438
275: stonewalling pairs accessed: 319544
81: stonewalling pairs accessed: 284752
340: stonewalling pairs accessed: 280475
397: stonewalling pairs accessed: 284618
168: stonewalling pairs accessed: 306632
415: stonewalling pairs accessed: 319518
31: stonewalling pairs accessed: 282167
10: stonewalling pairs accessed: 283841
127: stonewalling pairs accessed: 284255
68: stonewalling pairs accessed: 305936
211: stonewalling pairs accessed: 282157
100: stonewalling pairs accessed: 282791
47: stonewalling pairs accessed: 285618
338: stonewalling pairs accessed: 303127
321: stonewalling pairs accessed: 285555
170: stonewalling pairs accessed: 280685
97: stonewalling pairs accessed: 282477
88: stonewalling pairs accessed: 305375
401: stonewalling pairs accessed: 284889
380: stonewalling pairs accessed: 280747
77: stonewalling pairs accessed: 284116
128: stonewalling pairs accessed: 302743
171: stonewalling pairs accessed: 285001
160: stonewalling pairs accessed: 283818
347: stonewalling pairs accessed: 284751
8: stonewalling pairs accessed: 306380
205: stonewalling pairs accessed: 323803
91: stonewalling pairs accessed: 281717
180: stonewalling pairs accessed: 280406
337: stonewalling pairs accessed: 285098
348: stonewalling pairs accessed: 302910
371: stonewalling pairs accessed: 285206
0: stonewalling pairs accessed: 346246
327: stonewalling pairs accessed: 282151
258: stonewalling pairs accessed: 303616
245: stonewalling pairs accessed: 319801
341: stonewalling pairs accessed: 281861
30: stonewalling pairs accessed: 283885
407: stonewalling pairs accessed: 284973
290: stonewalling pairs accessed: 282610
387: stonewalling pairs accessed: 281313
138: stonewalling pairs accessed: 306215
20: stonewalling pairs accessed: 283771
36: stonewalling pairs accessed: 285353
48: stonewalling pairs accessed: 306277
1: stonewalling pairs accessed: 281857
320: stonewalling pairs accessed: 283456
17: stonewalling pairs accessed: 285350
130: stonewalling pairs accessed: 280563
108: stonewalling pairs accessed: 306410
50: stonewalling pairs accessed: 282798
140: stonewalling pairs accessed: 283415
400: stonewalling pairs accessed: 283598
58: stonewalling pairs accessed: 306534
388: stonewalling pairs accessed: 302525
376: stonewalling pairs accessed: 284888
136: stonewalling pairs accessed: 284128
216: stonewalling pairs accessed: 285154
73: stonewalling pairs accessed: 306268
298: stonewalling pairs accessed: 305400
295: stonewalling pairs accessed: 320581
391: stonewalling pairs accessed: 282119
288: stonewalling pairs accessed: 304910
315: stonewalling pairs accessed: 323824
260: stonewalling pairs accessed: 283449
220: stonewalling pairs accessed: 280943
271: stonewalling pairs accessed: 285325
370: stonewalling pairs accessed: 280774
200: stonewalling pairs accessed: 283846
178: stonewalling pairs accessed: 305868
150: stonewalling pairs accessed: 284450
307: stonewalling pairs accessed: 283629
101: stonewalling pairs accessed: 285065
71: stonewalling pairs accessed: 281367
61: stonewalling pairs accessed: 281774
141: stonewalling pairs accessed: 284888
111: stonewalling pairs accessed: 284366
257: stonewalling pairs accessed: 281464
131: stonewalling pairs accessed: 281566
151: stonewalling pairs accessed: 280969
228: stonewalling pairs accessed: 304725
210: stonewalling pairs accessed: 283639
241: stonewalling pairs accessed: 285154
158: stonewalling pairs accessed: 306210
398: stonewalling pairs accessed: 305586
261: stonewalling pairs accessed: 285025
188: stonewalling pairs accessed: 305866
378: stonewalling pairs accessed: 306578
148: stonewalling pairs accessed: 306148
197: stonewalling pairs accessed: 283662
251: stonewalling pairs accessed: 281376
297: stonewalling pairs accessed: 283796
268: stonewalling pairs accessed: 306205
38: stonewalling pairs accessed: 306537
98: stonewalling pairs accessed: 303029
118: stonewalling pairs accessed: 303485
308: stonewalling pairs accessed: 305441
267: stonewalling pairs accessed: 285023
377: stonewalling pairs accessed: 282646
27: stonewalling pairs accessed: 285134
417: stonewalling pairs accessed: 282317
57: stonewalling pairs accessed: 281855
147: stonewalling pairs accessed: 282420
317: stonewalling pairs accessed: 285335
187: stonewalling pairs accessed: 281498
358: stonewalling pairs accessed: 302315
207: stonewalling pairs accessed: 281713
198: stonewalling pairs accessed: 306891
177: stonewalling pairs accessed: 281215
157: stonewalling pairs accessed: 284625
107: stonewalling pairs accessed: 281742
217: stonewalling pairs accessed: 284668
277: stonewalling pairs accessed: 282420
318: stonewalling pairs accessed: 302953
287: stonewalling pairs accessed: 281721
227: stonewalling pairs accessed: 284985
238: stonewalling pairs accessed: 302920
247: stonewalling pairs accessed: 285383
237: stonewalling pairs accessed: 282322
78: stonewalling pairs accessed: 306163
stonewalling pairs accessed min: 280372 max: 346246 -- min data: 68.5 GiB mean data: 73.8 GiB time: 300.0s
WARNING: Expected aggregate file size       = 440401920000000.
WARNING: Stat() of aggregate file size      = 439444110311424.
WARNING: Using actual aggregate bytes moved = 38121850798080.
WARNING: maybe caused by deadlineForStonewalling
write     99811      404995     0.000075    1024000000 256.00     5.17       359.07     0.000923   364.25     0   
Max Write: 99811.32 MiB/sec (104659.76 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       99811.32   99811.32   99811.32       0.00  399245.28  399245.28  399245.28       0.00  364.24556     300.01     105816.15     0    420   1    1   0     1        1         0    0      1 1048576000000   262144 36355832.0 DFS      0
Finished            : Sat Jul 11 14:46:25 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_hard_read
[0] Pool uuid = 372e1597-1708-4f1e-aa78-1cceb4c7c6ec, SVCL = 1

[0] DFS Container namespace uuid = af091f54-7e7d-4c6b-bc70-a5d7d16e4955

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Jul 11 15:23:28 2020
Command line        : /home1/06758/mschaara/io-500/install/ior/bin/ior -r -R -s 10000000 -a DFS --dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec --dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955 --dfs.svcl=1 --dfs.chunk_size=1880320 --dfs.prefix=/tmp/dfuse -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_hard/stonewall
Machine             : Linux c110-001.frontera.tacc.utexas.edu
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Sat Jul 11 15:23:28 2020
Participating tasks: 420
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 420
tasks               : 420
clients per node    : 1
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 179.56 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Sat Jul 11 15:23:32 2020

WARNING: Expected aggregate file size       = 197433600000000.
WARNING: Stat() of aggregate file size      = 34698481359360.
WARNING: Using actual aggregate bytes moved = 34698481359360.
read      68102      1528316    482.97      45.91      45.91      2.91       482.98     0.013107   485.90     0   
Max Read:  68102.21 MiB/sec (71410.34 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        68102.21   68102.21   68102.21       0.00 1519110.46 1519110.46 1519110.46       0.00  485.90273         NA            NA     0    420   1    1   0     1        1         0    0 10000000    47008    47008 33091050.0 DFS      0
Finished            : Sat Jul 11 15:31:35 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_hard_write
[0] Pool uuid = 372e1597-1708-4f1e-aa78-1cceb4c7c6ec, SVCL = 1

[0] DFS Container namespace uuid = af091f54-7e7d-4c6b-bc70-a5d7d16e4955

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Jul 11 14:51:56 2020
Command line        : /home1/06758/mschaara/io-500/install/ior/bin/ior -w -s 10000000 -a DFS --dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec --dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955 --dfs.svcl=1 --dfs.chunk_size=1880320 --dfs.prefix=/tmp/dfuse -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux c110-001.frontera.tacc.utexas.edu
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Sat Jul 11 14:51:56 2020
Participating tasks: 420
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 420
tasks               : 420
clients per node    : 1
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 179.56 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Sat Jul 11 14:52:01 2020
282: stonewalling pairs accessed: 1355708
292: stonewalling pairs accessed: 1361434
32: stonewalling pairs accessed: 1365589
72: stonewalling pairs accessed: 1367833
197: stonewalling pairs accessed: 1359971
208: stonewalling pairs accessed: 1368974
302: stonewalling pairs accessed: 1358839
312: stonewalling pairs accessed: 1358819
238: stonewalling pairs accessed: 1371723
218: stonewalling pairs accessed: 1368959
311: stonewalling pairs accessed: 1358861
154: stonewalling pairs accessed: 1362592
119: stonewalling pairs accessed: 1346352
228: stonewalling pairs accessed: 1371691
156: stonewalling pairs accessed: 1362588
386: stonewalling pairs accessed: 1380658
297: stonewalling pairs accessed: 1361392
16: stonewalling pairs accessed: 1380185
416: stonewalling pairs accessed: 1380718
166: stonewalling pairs accessed: 1360096
96: stonewalling pairs accessed: 1367700
306: stonewalling pairs accessed: 1358895
26: stonewalling pairs accessed: 1365634
145: stonewalling pairs accessed: 1362609
66: stonewalling pairs accessed: 1367620
36: stonewalling pairs accessed: 1365652
225: stonewalling pairs accessed: 1371693
126: stonewalling pairs accessed: 1367397
405: stonewalling pairs accessed: 1380748
196: stonewalling pairs accessed: 1360083
295: stonewalling pairs accessed: 1361455
316: stonewalling pairs accessed: 1358951
86: stonewalling pairs accessed: 1367855
235: stonewalling pairs accessed: 1371726
65: stonewalling pairs accessed: 1367597
46: stonewalling pairs accessed: 1368070
25: stonewalling pairs accessed: 1365651
76: stonewalling pairs accessed: 1367893
396: stonewalling pairs accessed: 1380596
45: stonewalling pairs accessed: 1368075
276: stonewalling pairs accessed: 1376170
15: stonewalling pairs accessed: 1380209
95: stonewalling pairs accessed: 1367863
216: stonewalling pairs accessed: 1369004
215: stonewalling pairs accessed: 1368945
335: stonewalling pairs accessed: 1358923
266: stonewalling pairs accessed: 1376172
206: stonewalling pairs accessed: 1368977
175: stonewalling pairs accessed: 1360109
135: stonewalling pairs accessed: 1367378
165: stonewalling pairs accessed: 1360093
346: stonewalling pairs accessed: 1347271
375: stonewalling pairs accessed: 1375682
285: stonewalling pairs accessed: 1361426
116: stonewalling pairs accessed: 1346371
205: stonewalling pairs accessed: 1368981
355: stonewalling pairs accessed: 1347265
75: stonewalling pairs accessed: 1367701
245: stonewalling pairs accessed: 1355626
315: stonewalling pairs accessed: 1358881
356: stonewalling pairs accessed: 1347251
55: stonewalling pairs accessed: 1365623
275: stonewalling pairs accessed: 1376163
115: stonewalling pairs accessed: 1346350
336: stonewalling pairs accessed: 1358962
186: stonewalling pairs accessed: 1360101
265: stonewalling pairs accessed: 1376154
236: stonewalling pairs accessed: 1371740
325: stonewalling pairs accessed: 1358916
345: stonewalling pairs accessed: 1358984
226: stonewalling pairs accessed: 1371709
35: stonewalling pairs accessed: 1365630
376: stonewalling pairs accessed: 1375697
85: stonewalling pairs accessed: 1367866
255: stonewalling pairs accessed: 1347685
296: stonewalling pairs accessed: 1361420
5: stonewalling pairs accessed: 1372576
125: stonewalling pairs accessed: 1367374
305: stonewalling pairs accessed: 1358856
246: stonewalling pairs accessed: 1355610
415: stonewalling pairs accessed: 1380743
187: stonewalling pairs accessed: 1359961
146: stonewalling pairs accessed: 1362619
365: stonewalling pairs accessed: 1375659
395: stonewalling pairs accessed: 1380623
155: stonewalling pairs accessed: 1362565
286: stonewalling pairs accessed: 1361479
195: stonewalling pairs accessed: 1359981
366: stonewalling pairs accessed: 1375675
406: stonewalling pairs accessed: 1380749
105: stonewalling pairs accessed: 1346345
385: stonewalling pairs accessed: 1380620
6: stonewalling pairs accessed: 1380132
106: stonewalling pairs accessed: 1346334
176: stonewalling pairs accessed: 1359981
256: stonewalling pairs accessed: 1355621
136: stonewalling pairs accessed: 1367348
326: stonewalling pairs accessed: 1358968
185: stonewalling pairs accessed: 1359987
56: stonewalling pairs accessed: 1368097
357: stonewalling pairs accessed: 1347240
387: stonewalling pairs accessed: 1380612
150: stonewalling pairs accessed: 1354067
319: stonewalling pairs accessed: 1358894
172: stonewalling pairs accessed: 1360118
402: stonewalling pairs accessed: 1380756
62: stonewalling pairs accessed: 1367621
192: stonewalling pairs accessed: 1360013
342: stonewalling pairs accessed: 1347291
4: stonewalling pairs accessed: 1380245
182: stonewalling pairs accessed: 1360096
242: stonewalling pairs accessed: 1355647
143: stonewalling pairs accessed: 1362674
122: stonewalling pairs accessed: 1367445
404: stonewalling pairs accessed: 1380822
352: stonewalling pairs accessed: 1347287
327: stonewalling pairs accessed: 1358881
92: stonewalling pairs accessed: 1367740
162: stonewalling pairs accessed: 1360128
322: stonewalling pairs accessed: 1358954
57: stonewalling pairs accessed: 1368110
362: stonewalling pairs accessed: 1375715
413: stonewalling pairs accessed: 1380825
22: stonewalling pairs accessed: 1365654
372: stonewalling pairs accessed: 1375728
2: stonewalling pairs accessed: 1380214
343: stonewalling pairs accessed: 1330077
102: stonewalling pairs accessed: 1346352
403: stonewalling pairs accessed: 1380790
253: stonewalling pairs accessed: 1355686
27: stonewalling pairs accessed: 1365642
40: stonewalling pairs accessed: 1368096
113: stonewalling pairs accessed: 1346377
204: stonewalling pairs accessed: 1369024
153: stonewalling pairs accessed: 1362677
123: stonewalling pairs accessed: 1367460
93: stonewalling pairs accessed: 1367919
177: stonewalling pairs accessed: 1360018
374: stonewalling pairs accessed: 1373493
43: stonewalling pairs accessed: 1368114
377: stonewalling pairs accessed: 1375687
103: stonewalling pairs accessed: 1367780
17: stonewalling pairs accessed: 1391054
13: stonewalling pairs accessed: 1380255
87: stonewalling pairs accessed: 1367652
144: stonewalling pairs accessed: 1362612
131: stonewalling pairs accessed: 1367436
83: stonewalling pairs accessed: 1367928
414: stonewalling pairs accessed: 1380815
363: stonewalling pairs accessed: 1375734
214: stonewalling pairs accessed: 1369066
243: stonewalling pairs accessed: 1371825
167: stonewalling pairs accessed: 1360054
324: stonewalling pairs accessed: 1359007
120: stonewalling pairs accessed: 1367389
133: stonewalling pairs accessed: 1367465
337: stonewalling pairs accessed: 1358905
184: stonewalling pairs accessed: 1360099
359: stonewalling pairs accessed: 1347294
323: stonewalling pairs accessed: 1359017
97: stonewalling pairs accessed: 1367660
164: stonewalling pairs accessed: 1360134
12: stonewalling pairs accessed: 1378777
1: stonewalling pairs accessed: 1380249
417: stonewalling pairs accessed: 1380767
174: stonewalling pairs accessed: 1360169
82: stonewalling pairs accessed: 1367898
379: stonewalling pairs accessed: 1375733
367: stonewalling pairs accessed: 1371452
254: stonewalling pairs accessed: 1355687
252: stonewalling pairs accessed: 1355660
281: stonewalling pairs accessed: 1361504
19: stonewalling pairs accessed: 1380241
257: stonewalling pairs accessed: 1355644
84: stonewalling pairs accessed: 1367913
332: stonewalling pairs accessed: 1358985
50: stonewalling pairs accessed: 1368095
129: stonewalling pairs accessed: 1367452
313: stonewalling pairs accessed: 1358994
7: stonewalling pairs accessed: 1380211
364: stonewalling pairs accessed: 1375727
330: stonewalling pairs accessed: 1358939
81: stonewalling pairs accessed: 1367934
47: stonewalling pairs accessed: 1368095
44: stonewalling pairs accessed: 1368139
80: stonewalling pairs accessed: 1367748
351: stonewalling pairs accessed: 1359023
333: stonewalling pairs accessed: 1359016
247: stonewalling pairs accessed: 1355632
130: stonewalling pairs accessed: 1360461
361: stonewalling pairs accessed: 1375701
287: stonewalling pairs accessed: 1361437
410: stonewalling pairs accessed: 1380648
51: stonewalling pairs accessed: 1368152
37: stonewalling pairs accessed: 1368284
94: stonewalling pairs accessed: 1367927
320: stonewalling pairs accessed: 1358908
341: stonewalling pairs accessed: 1347315
207: stonewalling pairs accessed: 1369006
391: stonewalling pairs accessed: 1380756
0: stonewalling pairs accessed: 1757476
11: stonewalling pairs accessed: 1386523
111: stonewalling pairs accessed: 1346391
10: stonewalling pairs accessed: 1380155
100: stonewalling pairs accessed: 1367408
390: stonewalling pairs accessed: 1380651
303: stonewalling pairs accessed: 1358932
110: stonewalling pairs accessed: 1346373
41: stonewalling pairs accessed: 1368159
39: stonewalling pairs accessed: 1365690
170: stonewalling pairs accessed: 1360096
401: stonewalling pairs accessed: 1380783
49: stonewalling pairs accessed: 1368129
400: stonewalling pairs accessed: 1380775
331: stonewalling pairs accessed: 1359028
20: stonewalling pairs accessed: 1368289
411: stonewalling pairs accessed: 1380804
109: stonewalling pairs accessed: 1346430
121: stonewalling pairs accessed: 1367419
329: stonewalling pairs accessed: 1358989
409: stonewalling pairs accessed: 1380815
360: stonewalling pairs accessed: 1360591
59: stonewalling pairs accessed: 1368152
340: stonewalling pairs accessed: 1359023
349: stonewalling pairs accessed: 1347302
380: stonewalling pairs accessed: 1380638
369: stonewalling pairs accessed: 1375716
381: stonewalling pairs accessed: 1380664
350: stonewalling pairs accessed: 1347285
321: stonewalling pairs accessed: 1359012
240: stonewalling pairs accessed: 1355639
29: stonewalling pairs accessed: 1365708
419: stonewalling pairs accessed: 1380821
101: stonewalling pairs accessed: 1346433
139: stonewalling pairs accessed: 1367417
328: stonewalling pairs accessed: 1359021
339: stonewalling pairs accessed: 1358965
389: stonewalling pairs accessed: 1380672
388: stonewalling pairs accessed: 1380664
244: stonewalling pairs accessed: 1371837
78: stonewalling pairs accessed: 1367977
398: stonewalling pairs accessed: 1380720
28: stonewalling pairs accessed: 1368129
358: stonewalling pairs accessed: 1347316
310: stonewalling pairs accessed: 1358879
418: stonewalling pairs accessed: 1380813
68: stonewalling pairs accessed: 1367966
88: stonewalling pairs accessed: 1367763
48: stonewalling pairs accessed: 1368167
38: stonewalling pairs accessed: 1359486
338: stonewalling pairs accessed: 1359012
408: stonewalling pairs accessed: 1380807
348: stonewalling pairs accessed: 1347330
368: stonewalling pairs accessed: 1375712
18: stonewalling pairs accessed: 1380242
168: stonewalling pairs accessed: 1360142
118: stonewalling pairs accessed: 1346411
178: stonewalling pairs accessed: 1360132
378: stonewalling pairs accessed: 1375702
108: stonewalling pairs accessed: 1346416
8: stonewalling pairs accessed: 1380228
91: stonewalling pairs accessed: 1367950
248: stonewalling pairs accessed: 1355719
397: stonewalling pairs accessed: 1380719
347: stonewalling pairs accessed: 1347355
138: stonewalling pairs accessed: 1367504
301: stonewalling pairs accessed: 1359030
283: stonewalling pairs accessed: 1361648
128: stonewalling pairs accessed: 1363212
291: stonewalling pairs accessed: 1361602
52: stonewalling pairs accessed: 1356684
3: stonewalling pairs accessed: 1380324
169: stonewalling pairs accessed: 1360281
237: stonewalling pairs accessed: 1371852
217: stonewalling pairs accessed: 1369133
152: stonewalling pairs accessed: 1362787
272: stonewalling pairs accessed: 1376334
227: stonewalling pairs accessed: 1371860
383: stonewalling pairs accessed: 1380829
241: stonewalling pairs accessed: 1355818
277: stonewalling pairs accessed: 1376321
163: stonewalling pairs accessed: 1360266
263: stonewalling pairs accessed: 1376348
267: stonewalling pairs accessed: 1376320
149: stonewalling pairs accessed: 1362814
183: stonewalling pairs accessed: 1360180
353: stonewalling pairs accessed: 1347468
21: stonewalling pairs accessed: 1368431
317: stonewalling pairs accessed: 1359013
344: stonewalling pairs accessed: 1347471
161: stonewalling pairs accessed: 1357926
284: stonewalling pairs accessed: 1361648
304: stonewalling pairs accessed: 1359143
24: stonewalling pairs accessed: 1368434
124: stonewalling pairs accessed: 1367603
399: stonewalling pairs accessed: 1380834
384: stonewalling pairs accessed: 1380830
104: stonewalling pairs accessed: 1367914
58: stonewalling pairs accessed: 1368296
14: stonewalling pairs accessed: 1391205
148: stonewalling pairs accessed: 1362813
98: stonewalling pairs accessed: 1368079
188: stonewalling pairs accessed: 1360172
198: stonewalling pairs accessed: 1360264
261: stonewalling pairs accessed: 1376366
67: stonewalling pairs accessed: 1367812
249: stonewalling pairs accessed: 1355832
258: stonewalling pairs accessed: 1355836
278: stonewalling pairs accessed: 1376350
268: stonewalling pairs accessed: 1376353
293: stonewalling pairs accessed: 1361679
30: stonewalling pairs accessed: 1342235
191: stonewalling pairs accessed: 1360223
60: stonewalling pairs accessed: 1367802
370: stonewalling pairs accessed: 1364278
158: stonewalling pairs accessed: 1362820
209: stonewalling pairs accessed: 1369213
159: stonewalling pairs accessed: 1358306
224: stonewalling pairs accessed: 1371953
373: stonewalling pairs accessed: 1375901
9: stonewalling pairs accessed: 1380430
199: stonewalling pairs accessed: 1360203
132: stonewalling pairs accessed: 1367621
270: stonewalling pairs accessed: 1376388
212: stonewalling pairs accessed: 1369237
160: stonewalling pairs accessed: 1360196
229: stonewalling pairs accessed: 1371959
54: stonewalling pairs accessed: 1368321
392: stonewalling pairs accessed: 1380870
112: stonewalling pairs accessed: 1346543
71: stonewalling pairs accessed: 1368106
73: stonewalling pairs accessed: 1368160
61: stonewalling pairs accessed: 1367869
189: stonewalling pairs accessed: 1360340
181: stonewalling pairs accessed: 1360309
262: stonewalling pairs accessed: 1376404
34: stonewalling pairs accessed: 1368496
318: stonewalling pairs accessed: 1348677
42: stonewalling pairs accessed: 1365849
308: stonewalling pairs accessed: 1359227
234: stonewalling pairs accessed: 1371996
142: stonewalling pairs accessed: 1362868
298: stonewalling pairs accessed: 1361684
23: stonewalling pairs accessed: 1365915
307: stonewalling pairs accessed: 1359121
221: stonewalling pairs accessed: 1371986
289: stonewalling pairs accessed: 1355997
70: stonewalling pairs accessed: 1367857
77: stonewalling pairs accessed: 1367822
279: stonewalling pairs accessed: 1376424
213: stonewalling pairs accessed: 1369263
107: stonewalling pairs accessed: 1346570
382: stonewalling pairs accessed: 1380880
202: stonewalling pairs accessed: 1369232
90: stonewalling pairs accessed: 1367965
173: stonewalling pairs accessed: 1360355
250: stonewalling pairs accessed: 1355842
193: stonewalling pairs accessed: 1360332
53: stonewalling pairs accessed: 1368383
393: stonewalling pairs accessed: 1380968
271: stonewalling pairs accessed: 1376426
180: stonewalling pairs accessed: 1360205
259: stonewalling pairs accessed: 1355876
171: stonewalling pairs accessed: 1360345
114: stonewalling pairs accessed: 1346639
219: stonewalling pairs accessed: 1369243
151: stonewalling pairs accessed: 1362888
274: stonewalling pairs accessed: 1376402
251: stonewalling pairs accessed: 1372039
231: stonewalling pairs accessed: 1372003
31: stonewalling pairs accessed: 1365907
141: stonewalling pairs accessed: 1362893
294: stonewalling pairs accessed: 1361744
194: stonewalling pairs accessed: 1360323
334: stonewalling pairs accessed: 1359223
223: stonewalling pairs accessed: 1372014
314: stonewalling pairs accessed: 1359219
201: stonewalling pairs accessed: 1369248
134: stonewalling pairs accessed: 1367676
273: stonewalling pairs accessed: 1376438
79: stonewalling pairs accessed: 1367903
288: stonewalling pairs accessed: 1361753
203: stonewalling pairs accessed: 1369286
211: stonewalling pairs accessed: 1369271
89: stonewalling pairs accessed: 1368137
179: stonewalling pairs accessed: 1360287
64: stonewalling pairs accessed: 1368147
69: stonewalling pairs accessed: 1368202
137: stonewalling pairs accessed: 1367646
127: stonewalling pairs accessed: 1367630
309: stonewalling pairs accessed: 1359199
157: stonewalling pairs accessed: 1362868
280: stonewalling pairs accessed: 1361699
147: stonewalling pairs accessed: 1362864
233: stonewalling pairs accessed: 1372037
264: stonewalling pairs accessed: 1376464
140: stonewalling pairs accessed: 1362854
63: stonewalling pairs accessed: 1368232
354: stonewalling pairs accessed: 1347576
232: stonewalling pairs accessed: 1372044
222: stonewalling pairs accessed: 1372043
117: stonewalling pairs accessed: 1346631
190: stonewalling pairs accessed: 1360276
33: stonewalling pairs accessed: 1365969
74: stonewalling pairs accessed: 1368179
371: stonewalling pairs accessed: 1375971
210: stonewalling pairs accessed: 1367474
407: stonewalling pairs accessed: 1381030
299: stonewalling pairs accessed: 1361781
412: stonewalling pairs accessed: 1372927
99: stonewalling pairs accessed: 1368198
300: stonewalling pairs accessed: 1359155
239: stonewalling pairs accessed: 1372070
394: stonewalling pairs accessed: 1381033
220: stonewalling pairs accessed: 1372017
260: stonewalling pairs accessed: 1376478
269: stonewalling pairs accessed: 1376501
290: stonewalling pairs accessed: 1361752
200: stonewalling pairs accessed: 1369297
230: stonewalling pairs accessed: 1372061
stonewalling pairs accessed min: 1330077 max: 1757476 -- min data: 58.2 GiB mean data: 59.8 GiB time: 300.0s
WARNING: Expected aggregate file size       = 197433600000000.
WARNING: Stat() of aggregate file size      = 34698481359360.
WARNING: Using actual aggregate bytes moved = 34698481359360.
WARNING: maybe caused by deadlineForStonewalling
write     85418      1931575    300.13      45.91      45.91      5.25       382.14     0.001705   387.40     0   
Max Write: 85418.47 MiB/sec (89567.76 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       85418.47   85418.47   85418.47       0.00 1905372.70 1905372.70 1905372.70       0.00  387.39923     300.05      85773.58     0    420   1    1   0     1        1         0    0 10000000    47008    47008 33091050.0 DFS      0
Finished            : Sat Jul 11 14:58:24 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

mdtest_easy_delete
-- started at 07/11/2020 15:34:54 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 420 node(s)
Command line used: /home1/06758/mschaara/io-500/install/ior/bin/mdtest '-r' '-F' '-P' '-Y' '-d' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_easy' '-n' '10000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec' '--dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse' '-x' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_easy-stonewall' '-N' '1'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    2788826.002    2788807.558    2788812.189          5.473
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.101          0.101          0.101          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        574.764        574.761        574.763          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          9.949          9.949          9.949          0.000
-- finished at 07/11/2020 15:44:39 --

mdtest_easy_stat
-- started at 07/11/2020 15:17:37 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 420 node(s)
Command line used: /home1/06758/mschaara/io-500/install/ior/bin/mdtest '-T' '-F' '-P' '-Y' '-d' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_easy' '-n' '10000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec' '--dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse' '-x' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_easy-stonewall' '-N' '1'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    4672871.674    4672827.247    4672838.250         12.519
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        343.027        343.024        343.026          0.001
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/11/2020 15:23:20 --

mdtest_easy_write
-- started at 07/11/2020 14:46:33 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 420 node(s)
Command line used: /home1/06758/mschaara/io-500/install/ior/bin/mdtest '-C' '-F' '-P' '-Y' '-d' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_easy' '-n' '10000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec' '--dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse' '-x' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files
Continue stonewall hit min: 3621510 max: 3816446 avg: 3762980.7 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    5151862.700    5151697.279    5151711.464         30.746
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    5320804.592             NA
   Tree creation             :          0.339          0.339          0.339          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        311.142        311.132        311.141          0.002
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        297.033             NA
   Tree creation             :          2.951          2.951          2.951          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/11/2020 14:51:47 --

mdtest_hard_delete
-- started at 07/11/2020 15:48:47 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 420 node(s)
Command line used: /home1/06758/mschaara/io-500/install/ior/bin/mdtest '-r' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_hard' '-n' '10000000' '-x' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec' '--dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse' '-N' '1'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    2716127.911    2716077.538    2716084.848         10.939
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.001          0.001          0.001          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        272.704        272.698        272.703          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :       1543.572       1543.572       1543.572          0.000
-- finished at 07/11/2020 16:19:04 --

mdtest_hard_read
-- started at 07/11/2020 15:44:47 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 420 node(s)
Command line used: /home1/06758/mschaara/io-500/install/ior/bin/mdtest '-X' '-E' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_hard' '-n' '10000000' '-x' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec' '--dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse' '-N' '1'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :    3215916.973    3215849.691    3215869.902         14.363
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        230.323        230.318        230.321          0.001
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/11/2020 15:48:38 --

mdtest_hard_stat
-- started at 07/11/2020 15:31:44 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 420 node(s)
Command line used: /home1/06758/mschaara/io-500/install/ior/bin/mdtest '-T' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_hard' '-n' '10000000' '-x' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec' '--dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse' '-N' '1'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    4081084.443    4080983.629    4081021.702         20.567
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        181.496        181.492        181.495          0.001
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/11/2020 15:34:46 --

mdtest_hard_write
-- started at 07/11/2020 14:58:32 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 420 node(s)
Command line used: /home1/06758/mschaara/io-500/install/ior/bin/mdtest '-C' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_hard' '-n' '10000000' '-x' '/tmp/dfuse/datafiles/2020.07.11-14.40.08-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=372e1597-1708-4f1e-aa78-1cceb4c7c6ec' '--dfs.cont=af091f54-7e7d-4c6b-bc70-a5d7d16e4955' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse' '-N' '1' '-W' '300'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files
Continue stonewall hit min: 1659276 max: 1763533 avg: 1731495.0 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    2343709.594    2343683.378    2343688.549          6.560
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    2424211.210             NA
   Tree creation             :        103.369        103.369        103.369          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        316.034        316.031        316.033          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.985             NA
   Tree creation             :          0.010          0.010          0.010          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/11/2020 15:03:48 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               97.472 GiB/s : time 359.07 seconds
[RESULT] IOPS phase 1         mdtest_easy_write             5151.860 kiops : time 311.14 seconds
[RESULT] BW   phase 2            ior_hard_write               83.416 GiB/s : time 382.14 seconds
[RESULT] IOPS phase 2         mdtest_hard_write             2343.710 kiops : time 316.03 seconds
[RESULT] IOPS phase 3                      find             2339.180 kiops : time 322.80 seconds
[RESULT] BW   phase 3             ior_easy_read               72.607 GiB/s : time 486.07 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             4672.870 kiops : time 343.03 seconds
[RESULT] BW   phase 4             ior_hard_read               66.506 GiB/s : time 482.98 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat             4081.080 kiops : time 181.50 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete             2788.830 kiops : time 574.76 seconds
[RESULT] IOPS phase 7          mdtest_hard_read             3215.920 kiops : time 230.32 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete             2716.130 kiops : time 1826.08 seconds
[SCORE] Bandwidth 79.1575 GiB/s : IOPS 3271.49 kiops : TOTAL 508.884