Wolf

Institution Intel
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.7
Client Kernel Version 3.10.0-1062.12.1.el7.x86_64

DATA SERVER

Storage Type Optane Persistent Memory
Volatile Memory 512GB
Storage Interface DDR-T
Network Omnipath
Software Version 1.1
OS Version 7.7

INFORMATION

Client Nodes 10
Client Total Procs 420

METADATA

Easy Write 4,828.03 kIOP/s
Easy Stat 4,600.36 kIOP/s
Easy Delete 2,795.34 kIOP/s
Hard Write 2,317.04 kIOP/s
Hard Read 3,231.39 kIOP/s
Hard Stat 4,109.81 kIOP/s
Hard Delete 2,768.94 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=/home/daos/io-500/install/ior/bin/ior
  io500_mdtest_cmd=/home/daos/io-500/install/ior/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 420 -iface ib0 --bind-to socket --hostfile /home/daos/io-500/config/cli_hosts"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

#  if [[ -s "system-information.txt" ]]; then
#    echo "Warning: please create a system-information.txt description by"
#    echo "copying the information from https://vi4io.org/io500-info-creator/"
#  else
#    cp "system-information.txt" $io500_result_dir
#  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val" && io500_find_mpi="True"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
[0] Pool uuid = 9ae1db5f-54fa-4822-81cd-2faaaeacc442, SVCL = 1

[0] DFS Container namespace uuid = 9f8cf146-1aec-46d8-8789-5a0746355be1

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Jun 29 05:14:59 2020
Command line        : /home/daos/io-500/install/ior/bin/ior -r -R -a DFS --dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442 --dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1 --dfs.svcl=1 --dfs.prefix=/mnt/dfuse/ -t 128k -v -b 200000m -i 1 -C -Q 1 -g -G 27 -k -e -o /mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_easy/stonewall
Machine             : Linux wolf-79.wolf.hpdd.intel.com
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Mon Jun 29 05:14:59 2020
Participating tasks: 420
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 420
clients per node    : 42
repetitions         : 1
xfersize            : 131072 bytes
blocksize           : 195.31 GiB
aggregate filesize  : 80.11 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Mon Jun 29 05:14:59 2020

WARNING: Expected aggregate file size       = 88080384000000.
WARNING: Stat() of aggregate file size      = 88014406680576.
WARNING: Using actual aggregate bytes moved = 60369909841920.
read      217942     1743695    0.000165    204800000  128.00     0.022957   264.14     0.000844   264.17     0   
Max Read:  217942.08 MiB/sec (228528.84 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       217942.08  217942.08  217942.08       0.00 1743536.65 1743536.65 1743536.65       0.00  264.16758         NA            NA     0    420  42    1   0     1        1         0    0      1 209715200000   131072 57573232.0 DFS      0
Finished            : Mon Jun 29 05:19:23 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_easy_write
[0] Pool uuid = 9ae1db5f-54fa-4822-81cd-2faaaeacc442, SVCL = 1

[0] DFS Container namespace uuid = 9f8cf146-1aec-46d8-8789-5a0746355be1

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Jun 29 04:47:03 2020
Command line        : /home/daos/io-500/install/ior/bin/ior -w -a DFS --dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442 --dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1 --dfs.svcl=1 --dfs.prefix=/mnt/dfuse/ -t 128k -v -b 200000m -i 1 -C -Q 1 -g -G 27 -k -e -o /mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux wolf-79.wolf.hpdd.intel.com
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Mon Jun 29 04:47:03 2020
Participating tasks: 420
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 420
clients per node    : 42
repetitions         : 1
xfersize            : 131072 bytes
blocksize           : 195.31 GiB
aggregate filesize  : 80.11 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Mon Jun 29 04:47:03 2020
40: stonewalling pairs accessed: 1057873
70: stonewalling pairs accessed: 1051868
200: stonewalling pairs accessed: 1053127
334: stonewalling pairs accessed: 1062774
74: stonewalling pairs accessed: 1061561
102: stonewalling pairs accessed: 1057312
206: stonewalling pairs accessed: 1059692
94: stonewalling pairs accessed: 1060886
152: stonewalling pairs accessed: 1066171
316: stonewalling pairs accessed: 1061049
294: stonewalling pairs accessed: 1053489
10: stonewalling pairs accessed: 1062405
191: stonewalling pairs accessed: 1061799
242: stonewalling pairs accessed: 1060602
66: stonewalling pairs accessed: 1057012
184: stonewalling pairs accessed: 1063917
90: stonewalling pairs accessed: 1055731
321: stonewalling pairs accessed: 1055795
382: stonewalling pairs accessed: 1048940
416: stonewalling pairs accessed: 1058252
4: stonewalling pairs accessed: 1057179
250: stonewalling pairs accessed: 1037092
412: stonewalling pairs accessed: 1060112
156: stonewalling pairs accessed: 1057761
24: stonewalling pairs accessed: 1058956
0: stonewalling pairs accessed: 1055322
122: stonewalling pairs accessed: 1059106
256: stonewalling pairs accessed: 1058626
44: stonewalling pairs accessed: 1054968
30: stonewalling pairs accessed: 1049902
32: stonewalling pairs accessed: 1056971
26: stonewalling pairs accessed: 1055038
68: stonewalling pairs accessed: 1076183
43: stonewalling pairs accessed: 1055962
64: stonewalling pairs accessed: 1057180
50: stonewalling pairs accessed: 1052784
189: stonewalling pairs accessed: 1079020
82: stonewalling pairs accessed: 1056578
46: stonewalling pairs accessed: 1048797
208: stonewalling pairs accessed: 1084225
103: stonewalling pairs accessed: 1055563
114: stonewalling pairs accessed: 1051562
80: stonewalling pairs accessed: 1051829
9: stonewalling pairs accessed: 1068747
12: stonewalling pairs accessed: 1060096
196: stonewalling pairs accessed: 1060861
268: stonewalling pairs accessed: 1082945
323: stonewalling pairs accessed: 1049544
124: stonewalling pairs accessed: 1059770
120: stonewalling pairs accessed: 1048918
109: stonewalling pairs accessed: 1084208
142: stonewalling pairs accessed: 1060580
6: stonewalling pairs accessed: 1054717
408: stonewalling pairs accessed: 1084169
393: stonewalling pairs accessed: 1060668
134: stonewalling pairs accessed: 1039602
130: stonewalling pairs accessed: 1059528
159: stonewalling pairs accessed: 1078764
2: stonewalling pairs accessed: 1048751
16: stonewalling pairs accessed: 1055992
418: stonewalling pairs accessed: 1087489
63: stonewalling pairs accessed: 1059340
174: stonewalling pairs accessed: 1057448
140: stonewalling pairs accessed: 1053498
179: stonewalling pairs accessed: 1082978
172: stonewalling pairs accessed: 1060095
36: stonewalling pairs accessed: 1055749
255: stonewalling pairs accessed: 1053755
128: stonewalling pairs accessed: 1087637
93: stonewalling pairs accessed: 1059290
224: stonewalling pairs accessed: 1055971
190: stonewalling pairs accessed: 1051718
269: stonewalling pairs accessed: 1080547
72: stonewalling pairs accessed: 1056653
56: stonewalling pairs accessed: 1059916
305: stonewalling pairs accessed: 1051710
188: stonewalling pairs accessed: 1086374
343: stonewalling pairs accessed: 1060083
254: stonewalling pairs accessed: 1059774
270: stonewalling pairs accessed: 1054826
289: stonewalling pairs accessed: 1085449
192: stonewalling pairs accessed: 1059381
96: stonewalling pairs accessed: 1067372
145: stonewalling pairs accessed: 1056351
48: stonewalling pairs accessed: 1083310
113: stonewalling pairs accessed: 1061265
274: stonewalling pairs accessed: 1055245
280: stonewalling pairs accessed: 1048910
39: stonewalling pairs accessed: 1081940
212: stonewalling pairs accessed: 1062360
186: stonewalling pairs accessed: 1048483
215: stonewalling pairs accessed: 1054884
148: stonewalling pairs accessed: 1084436
133: stonewalling pairs accessed: 1058984
284: stonewalling pairs accessed: 1058002
290: stonewalling pairs accessed: 1057078
59: stonewalling pairs accessed: 1084086
222: stonewalling pairs accessed: 1052557
276: stonewalling pairs accessed: 1056755
345: stonewalling pairs accessed: 1057276
288: stonewalling pairs accessed: 1082034
253: stonewalling pairs accessed: 1058827
324: stonewalling pairs accessed: 1057848
310: stonewalling pairs accessed: 1049121
69: stonewalling pairs accessed: 1086109
252: stonewalling pairs accessed: 1038297
336: stonewalling pairs accessed: 1055169
95: stonewalling pairs accessed: 1063934
138: stonewalling pairs accessed: 1082010
3: stonewalling pairs accessed: 1051382
344: stonewalling pairs accessed: 1053128
320: stonewalling pairs accessed: 1051567
169: stonewalling pairs accessed: 1087508
272: stonewalling pairs accessed: 1053542
406: stonewalling pairs accessed: 1055750
205: stonewalling pairs accessed: 1054109
198: stonewalling pairs accessed: 1081431
163: stonewalling pairs accessed: 1056952
354: stonewalling pairs accessed: 1063041
350: stonewalling pairs accessed: 1057579
219: stonewalling pairs accessed: 1079670
312: stonewalling pairs accessed: 1060992
176: stonewalling pairs accessed: 1060030
235: stonewalling pairs accessed: 1055549
218: stonewalling pairs accessed: 1088813
283: stonewalling pairs accessed: 1050534
374: stonewalling pairs accessed: 1056424
370: stonewalling pairs accessed: 1059785
229: stonewalling pairs accessed: 1082755
42: stonewalling pairs accessed: 1049968
296: stonewalling pairs accessed: 1064619
335: stonewalling pairs accessed: 1039311
398: stonewalling pairs accessed: 1078577
353: stonewalling pairs accessed: 1054488
414: stonewalling pairs accessed: 1061526
410: stonewalling pairs accessed: 1059985
319: stonewalling pairs accessed: 1074521
346: stonewalling pairs accessed: 1051351
35: stonewalling pairs accessed: 1065346
78: stonewalling pairs accessed: 1089302
363: stonewalling pairs accessed: 1056972
34: stonewalling pairs accessed: 1054243
20: stonewalling pairs accessed: 1046952
339: stonewalling pairs accessed: 1081076
62: stonewalling pairs accessed: 1057737
366: stonewalling pairs accessed: 1056437
85: stonewalling pairs accessed: 1051466
248: stonewalling pairs accessed: 1078979
373: stonewalling pairs accessed: 1060993
84: stonewalling pairs accessed: 1062339
60: stonewalling pairs accessed: 1043415
379: stonewalling pairs accessed: 1058467
332: stonewalling pairs accessed: 1060469
376: stonewalling pairs accessed: 1053196
275: stonewalling pairs accessed: 1057893
258: stonewalling pairs accessed: 1082061
413: stonewalling pairs accessed: 1063713
154: stonewalling pairs accessed: 1053719
100: stonewalling pairs accessed: 1054624
389: stonewalling pairs accessed: 1086454
97: stonewalling pairs accessed: 1057787
22: stonewalling pairs accessed: 1042205
76: stonewalling pairs accessed: 1058403
285: stonewalling pairs accessed: 1057253
348: stonewalling pairs accessed: 1075188
13: stonewalling pairs accessed: 1048575
194: stonewalling pairs accessed: 1065449
150: stonewalling pairs accessed: 1054874
399: stonewalling pairs accessed: 1077029
187: stonewalling pairs accessed: 1055922
52: stonewalling pairs accessed: 1056789
126: stonewalling pairs accessed: 1060701
325: stonewalling pairs accessed: 1054869
18: stonewalling pairs accessed: 1080322
73: stonewalling pairs accessed: 1040048
214: stonewalling pairs accessed: 1060277
170: stonewalling pairs accessed: 1063259
409: stonewalling pairs accessed: 1082847
137: stonewalling pairs accessed: 1054800
202: stonewalling pairs accessed: 1052564
216: stonewalling pairs accessed: 1033664
355: stonewalling pairs accessed: 1050921
28: stonewalling pairs accessed: 1089638
193: stonewalling pairs accessed: 1061276
234: stonewalling pairs accessed: 1059496
180: stonewalling pairs accessed: 1054310
29: stonewalling pairs accessed: 1085505
147: stonewalling pairs accessed: 1055845
302: stonewalling pairs accessed: 1064159
226: stonewalling pairs accessed: 1058739
375: stonewalling pairs accessed: 1049896
178: stonewalling pairs accessed: 1084499
203: stonewalling pairs accessed: 1062367
244: stonewalling pairs accessed: 1057808
210: stonewalling pairs accessed: 1054388
89: stonewalling pairs accessed: 1084465
197: stonewalling pairs accessed: 1057821
92: stonewalling pairs accessed: 1059451
236: stonewalling pairs accessed: 1058860
415: stonewalling pairs accessed: 1060512
238: stonewalling pairs accessed: 1088437
223: stonewalling pairs accessed: 1054333
384: stonewalling pairs accessed: 1056579
220: stonewalling pairs accessed: 1054855
119: stonewalling pairs accessed: 1075182
257: stonewalling pairs accessed: 1057865
132: stonewalling pairs accessed: 1058300
356: stonewalling pairs accessed: 1056067
75: stonewalling pairs accessed: 1055764
328: stonewalling pairs accessed: 1087559
33: stonewalling pairs accessed: 1058263
394: stonewalling pairs accessed: 1065587
230: stonewalling pairs accessed: 1047939
129: stonewalling pairs accessed: 1083146
7: stonewalling pairs accessed: 1051293
182: stonewalling pairs accessed: 1058892
386: stonewalling pairs accessed: 1051549
115: stonewalling pairs accessed: 1060209
118: stonewalling pairs accessed: 1084948
53: stonewalling pairs accessed: 1058809
14: stonewalling pairs accessed: 1053863
360: stonewalling pairs accessed: 1051318
139: stonewalling pairs accessed: 1081081
37: stonewalling pairs accessed: 1059549
232: stonewalling pairs accessed: 1057590
106: stonewalling pairs accessed: 1060363
125: stonewalling pairs accessed: 1059945
358: stonewalling pairs accessed: 1078587
83: stonewalling pairs accessed: 1055871
54: stonewalling pairs accessed: 1062881
390: stonewalling pairs accessed: 1055985
209: stonewalling pairs accessed: 1080331
67: stonewalling pairs accessed: 1051494
342: stonewalling pairs accessed: 1059849
116: stonewalling pairs accessed: 1063488
135: stonewalling pairs accessed: 1050009
58: stonewalling pairs accessed: 1078119
123: stonewalling pairs accessed: 1055619
164: stonewalling pairs accessed: 1064398
110: stonewalling pairs accessed: 1058409
249: stonewalling pairs accessed: 1083193
177: stonewalling pairs accessed: 1058135
1: stonewalling pairs accessed: 1058289
262: stonewalling pairs accessed: 1055144
146: stonewalling pairs accessed: 1063729
155: stonewalling pairs accessed: 1046989
108: stonewalling pairs accessed: 1084858
143: stonewalling pairs accessed: 1062139
204: stonewalling pairs accessed: 1053988
160: stonewalling pairs accessed: 1052504
279: stonewalling pairs accessed: 1068674
27: stonewalling pairs accessed: 1056040
11: stonewalling pairs accessed: 1053722
282: stonewalling pairs accessed: 1061561
86: stonewalling pairs accessed: 1059262
45: stonewalling pairs accessed: 1046363
228: stonewalling pairs accessed: 1084530
183: stonewalling pairs accessed: 1049908
304: stonewalling pairs accessed: 1056235
260: stonewalling pairs accessed: 1059414
329: stonewalling pairs accessed: 1079777
47: stonewalling pairs accessed: 1061480
21: stonewalling pairs accessed: 1057835
352: stonewalling pairs accessed: 1060850
306: stonewalling pairs accessed: 1052761
65: stonewalling pairs accessed: 1051343
8: stonewalling pairs accessed: 1088937
263: stonewalling pairs accessed: 1057909
364: stonewalling pairs accessed: 1064067
400: stonewalling pairs accessed: 1053120
349: stonewalling pairs accessed: 1079675
117: stonewalling pairs accessed: 1061239
31: stonewalling pairs accessed: 1066282
392: stonewalling pairs accessed: 1062252
136: stonewalling pairs accessed: 1059191
165: stonewalling pairs accessed: 1055559
338: stonewalling pairs accessed: 1074515
313: stonewalling pairs accessed: 1056722
104: stonewalling pairs accessed: 1059821
340: stonewalling pairs accessed: 1053015
359: stonewalling pairs accessed: 1076637
227: stonewalling pairs accessed: 1054982
41: stonewalling pairs accessed: 1058184
112: stonewalling pairs accessed: 1061967
286: stonewalling pairs accessed: 1054771
195: stonewalling pairs accessed: 1064111
98: stonewalling pairs accessed: 1074361
403: stonewalling pairs accessed: 1048855
144: stonewalling pairs accessed: 1053646
240: stonewalling pairs accessed: 1057365
19: stonewalling pairs accessed: 1079964
277: stonewalling pairs accessed: 1035216
51: stonewalling pairs accessed: 1058797
292: stonewalling pairs accessed: 1061625
166: stonewalling pairs accessed: 1052351
245: stonewalling pairs accessed: 1051295
158: stonewalling pairs accessed: 1088357
243: stonewalling pairs accessed: 1055105
264: stonewalling pairs accessed: 1055803
330: stonewalling pairs accessed: 1053929
239: stonewalling pairs accessed: 1083953
317: stonewalling pairs accessed: 1066152
61: stonewalling pairs accessed: 1063854
322: stonewalling pairs accessed: 1053458
246: stonewalling pairs accessed: 1050033
295: stonewalling pairs accessed: 1062358
168: stonewalling pairs accessed: 1076826
273: stonewalling pairs accessed: 1058790
314: stonewalling pairs accessed: 1053209
380: stonewalling pairs accessed: 1054144
79: stonewalling pairs accessed: 1080891
327: stonewalling pairs accessed: 1045657
71: stonewalling pairs accessed: 1060332
362: stonewalling pairs accessed: 1053723
326: stonewalling pairs accessed: 1051602
385: stonewalling pairs accessed: 1063865
88: stonewalling pairs accessed: 1081693
303: stonewalling pairs accessed: 1059735
404: stonewalling pairs accessed: 1058783
300: stonewalling pairs accessed: 1040268
199: stonewalling pairs accessed: 1077314
337: stonewalling pairs accessed: 1058243
81: stonewalling pairs accessed: 1058581
372: stonewalling pairs accessed: 1055374
396: stonewalling pairs accessed: 1054124
15: stonewalling pairs accessed: 1061417
308: stonewalling pairs accessed: 1076786
153: stonewalling pairs accessed: 1065698
299: stonewalling pairs accessed: 1081005
347: stonewalling pairs accessed: 1054212
91: stonewalling pairs accessed: 1040050
402: stonewalling pairs accessed: 1057188
266: stonewalling pairs accessed: 1051958
25: stonewalling pairs accessed: 1053873
378: stonewalling pairs accessed: 1065528
213: stonewalling pairs accessed: 1056520
149: stonewalling pairs accessed: 1081021
367: stonewalling pairs accessed: 1056713
101: stonewalling pairs accessed: 1052656
162: stonewalling pairs accessed: 1056647
105: stonewalling pairs accessed: 1057033
298: stonewalling pairs accessed: 1084432
233: stonewalling pairs accessed: 1059129
369: stonewalling pairs accessed: 1085880
397: stonewalling pairs accessed: 1060797
111: stonewalling pairs accessed: 1056013
265: stonewalling pairs accessed: 1064038
368: stonewalling pairs accessed: 1093265
333: stonewalling pairs accessed: 1062592
99: stonewalling pairs accessed: 1078686
77: stonewalling pairs accessed: 1052306
121: stonewalling pairs accessed: 1054126
395: stonewalling pairs accessed: 1055140
38: stonewalling pairs accessed: 1086806
23: stonewalling pairs accessed: 1055118
309: stonewalling pairs accessed: 1080976
87: stonewalling pairs accessed: 1056622
131: stonewalling pairs accessed: 1059578
55: stonewalling pairs accessed: 1063112
318: stonewalling pairs accessed: 1080459
173: stonewalling pairs accessed: 1058955
49: stonewalling pairs accessed: 1096633
127: stonewalling pairs accessed: 1043283
141: stonewalling pairs accessed: 1063722
185: stonewalling pairs accessed: 1057382
388: stonewalling pairs accessed: 1084846
383: stonewalling pairs accessed: 1059833
259: stonewalling pairs accessed: 1077087
167: stonewalling pairs accessed: 1058618
151: stonewalling pairs accessed: 1056696
225: stonewalling pairs accessed: 1058975
278: stonewalling pairs accessed: 1088072
293: stonewalling pairs accessed: 1060888
419: stonewalling pairs accessed: 1067968
237: stonewalling pairs accessed: 1054832
161: stonewalling pairs accessed: 1056330
5: stonewalling pairs accessed: 1053615
267: stonewalling pairs accessed: 1055269
171: stonewalling pairs accessed: 1058996
405: stonewalling pairs accessed: 1053885
287: stonewalling pairs accessed: 1054113
181: stonewalling pairs accessed: 1053245
365: stonewalling pairs accessed: 1055759
387: stonewalling pairs accessed: 1040980
201: stonewalling pairs accessed: 1056248
175: stonewalling pairs accessed: 1057495
207: stonewalling pairs accessed: 1059002
211: stonewalling pairs accessed: 1057335
315: stonewalling pairs accessed: 1058749
157: stonewalling pairs accessed: 1058303
221: stonewalling pairs accessed: 1059507
357: stonewalling pairs accessed: 1062067
231: stonewalling pairs accessed: 1060369
17: stonewalling pairs accessed: 1061856
241: stonewalling pairs accessed: 1053644
107: stonewalling pairs accessed: 1052662
251: stonewalling pairs accessed: 1059819
247: stonewalling pairs accessed: 1052214
261: stonewalling pairs accessed: 1058132
307: stonewalling pairs accessed: 1052674
271: stonewalling pairs accessed: 1062416
377: stonewalling pairs accessed: 1056587
281: stonewalling pairs accessed: 1048960
217: stonewalling pairs accessed: 1062408
291: stonewalling pairs accessed: 1058147
297: stonewalling pairs accessed: 1060823
301: stonewalling pairs accessed: 1058109
407: stonewalling pairs accessed: 1055037
311: stonewalling pairs accessed: 1054650
417: stonewalling pairs accessed: 1067980
331: stonewalling pairs accessed: 1058955
57: stonewalling pairs accessed: 1060884
341: stonewalling pairs accessed: 1059001
361: stonewalling pairs accessed: 1056260
371: stonewalling pairs accessed: 1066536
381: stonewalling pairs accessed: 1060747
391: stonewalling pairs accessed: 1061363
401: stonewalling pairs accessed: 1054169
411: stonewalling pairs accessed: 1061248
351: stonewalling pairs accessed: 1054614
stonewalling pairs accessed min: 1033664 max: 1096633 -- min data: 126.2 GiB mean data: 129.6 GiB time: 300.6s
WARNING: Expected aggregate file size       = 88080384000000.
WARNING: Stat() of aggregate file size      = 88014406680576.
WARNING: Using actual aggregate bytes moved = 60369909841920.
WARNING: maybe caused by deadlineForStonewalling
write     182566     1460675    0.000195    204800000  128.00     0.032274   315.32     0.000498   315.36     0   
Max Write: 182565.55 MiB/sec (191433.86 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      182565.55  182565.55  182565.55       0.00 1460524.43 1460524.43 1460524.43       0.00  315.35649     300.56     185411.45     0    420  42    1   0     1        1         0    0      1 209715200000   131072 57573232.0 DFS      0
Finished            : Mon Jun 29 04:52:18 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_hard_read
[0] Pool uuid = 9ae1db5f-54fa-4822-81cd-2faaaeacc442, SVCL = 1

[0] DFS Container namespace uuid = 9f8cf146-1aec-46d8-8789-5a0746355be1

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Jun 29 05:25:44 2020
Command line        : /home/daos/io-500/install/ior/bin/ior -r -R -s 10000000 -a DFS --dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442 --dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1 --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/mnt/dfuse/ -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_hard/stonewall
Machine             : Linux wolf-79.wolf.hpdd.intel.com
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Mon Jun 29 05:25:44 2020
Participating tasks: 420
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 420
clients per node    : 42
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 179.56 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Mon Jun 29 05:25:44 2020

WARNING: Expected aggregate file size       = 197433600000000.
WARNING: Stat() of aggregate file size      = 39703817986560.
WARNING: Using actual aggregate bytes moved = 39703817986560.
read      164910     3678937    229.58      45.91      45.91      0.023330   229.58     0.001186   229.61     0   
Max Read:  164910.27 MiB/sec (172920.95 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       164910.27  164910.27  164910.27       0.00 3678543.03 3678543.03 3678543.03       0.00  229.60675         NA            NA     0    420  42    1   0     1        1         0    0 10000000    47008    47008 37864512.0 DFS      0
Finished            : Mon Jun 29 05:29:34 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_hard_write
[0] Pool uuid = 9ae1db5f-54fa-4822-81cd-2faaaeacc442, SVCL = 1

[0] DFS Container namespace uuid = 9f8cf146-1aec-46d8-8789-5a0746355be1

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Jun 29 04:58:24 2020
Command line        : /home/daos/io-500/install/ior/bin/ior -w -s 10000000 -a DFS --dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442 --dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1 --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/mnt/dfuse/ -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux wolf-79.wolf.hpdd.intel.com
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Mon Jun 29 04:58:24 2020
Participating tasks: 420
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 420
clients per node    : 42
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 179.56 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Mon Jun 29 04:58:24 2020
20: stonewalling pairs accessed: 1985664
150: stonewalling pairs accessed: 1983708
240: stonewalling pairs accessed: 1977594
280: stonewalling pairs accessed: 1992183
210: stonewalling pairs accessed: 1977001
350: stonewalling pairs accessed: 2005065
70: stonewalling pairs accessed: 1995471
120: stonewalling pairs accessed: 1990668
230: stonewalling pairs accessed: 2005963
250: stonewalling pairs accessed: 1993409
260: stonewalling pairs accessed: 1988708
320: stonewalling pairs accessed: 1982301
370: stonewalling pairs accessed: 2002237
410: stonewalling pairs accessed: 2003167
50: stonewalling pairs accessed: 1975919
100: stonewalling pairs accessed: 1996542
270: stonewalling pairs accessed: 1996684
400: stonewalling pairs accessed: 1993779
80: stonewalling pairs accessed: 1983128
190: stonewalling pairs accessed: 2009759
310: stonewalling pairs accessed: 1995904
380: stonewalling pairs accessed: 1996981
30: stonewalling pairs accessed: 1992635
60: stonewalling pairs accessed: 1993864
290: stonewalling pairs accessed: 1987883
340: stonewalling pairs accessed: 1977625
419: stonewalling pairs accessed: 2003178
40: stonewalling pairs accessed: 1977377
244: stonewalling pairs accessed: 1977596
167: stonewalling pairs accessed: 1993517
342: stonewalling pairs accessed: 1977624
246: stonewalling pairs accessed: 1977592
173: stonewalling pairs accessed: 1971276
129: stonewalling pairs accessed: 1990670
200: stonewalling pairs accessed: 1991406
275: stonewalling pairs accessed: 1996688
28: stonewalling pairs accessed: 1985676
231: stonewalling pairs accessed: 2005980
24: stonewalling pairs accessed: 1985668
407: stonewalling pairs accessed: 1993793
382: stonewalling pairs accessed: 1997018
296: stonewalling pairs accessed: 1987862
253: stonewalling pairs accessed: 1993425
249: stonewalling pairs accessed: 1977600
90: stonewalling pairs accessed: 2007283
195: stonewalling pairs accessed: 2009768
218: stonewalling pairs accessed: 1977011
391: stonewalling pairs accessed: 1997325
144: stonewalling pairs accessed: 2010985
177: stonewalling pairs accessed: 1971266
312: stonewalling pairs accessed: 1995900
26: stonewalling pairs accessed: 1985674
143: stonewalling pairs accessed: 2010992
339: stonewalling pairs accessed: 2004644
220: stonewalling pairs accessed: 1994701
295: stonewalling pairs accessed: 1987905
368: stonewalling pairs accessed: 1993731
21: stonewalling pairs accessed: 1985672
334: stonewalling pairs accessed: 2004633
197: stonewalling pairs accessed: 2009782
12: stonewalling pairs accessed: 1996486
226: stonewalling pairs accessed: 1994708
163: stonewalling pairs accessed: 1993507
359: stonewalling pairs accessed: 2005073
130: stonewalling pairs accessed: 1989538
365: stonewalling pairs accessed: 1993698
58: stonewalling pairs accessed: 1975914
261: stonewalling pairs accessed: 1988710
74: stonewalling pairs accessed: 1995452
87: stonewalling pairs accessed: 1983130
22: stonewalling pairs accessed: 1985677
266: stonewalling pairs accessed: 1988708
363: stonewalling pairs accessed: 1993725
79: stonewalling pairs accessed: 1995500
390: stonewalling pairs accessed: 1997322
385: stonewalling pairs accessed: 1996989
138: stonewalling pairs accessed: 1989544
151: stonewalling pairs accessed: 1983718
134: stonewalling pairs accessed: 1989540
247: stonewalling pairs accessed: 1977590
42: stonewalling pairs accessed: 1977378
166: stonewalling pairs accessed: 1993507
343: stonewalling pairs accessed: 1977615
389: stonewalling pairs accessed: 1997017
10: stonewalling pairs accessed: 1996516
25: stonewalling pairs accessed: 1985674
18: stonewalling pairs accessed: 1996507
221: stonewalling pairs accessed: 1994698
164: stonewalling pairs accessed: 1993500
67: stonewalling pairs accessed: 1993883
162: stonewalling pairs accessed: 1993536
256: stonewalling pairs accessed: 1993412
243: stonewalling pairs accessed: 1977591
159: stonewalling pairs accessed: 1983715
330: stonewalling pairs accessed: 2004626
335: stonewalling pairs accessed: 2004636
128: stonewalling pairs accessed: 1990680
361: stonewalling pairs accessed: 1993706
194: stonewalling pairs accessed: 2009766
7: stonewalling pairs accessed: 1970205
262: stonewalling pairs accessed: 1988710
216: stonewalling pairs accessed: 1977002
103: stonewalling pairs accessed: 1996546
379: stonewalling pairs accessed: 2002245
360: stonewalling pairs accessed: 1993725
375: stonewalling pairs accessed: 2002253
398: stonewalling pairs accessed: 1997337
331: stonewalling pairs accessed: 2004639
344: stonewalling pairs accessed: 1977629
77: stonewalling pairs accessed: 1995489
82: stonewalling pairs accessed: 1983145
36: stonewalling pairs accessed: 1992635
293: stonewalling pairs accessed: 1987906
199: stonewalling pairs accessed: 2009780
140: stonewalling pairs accessed: 2010987
325: stonewalling pairs accessed: 1982331
178: stonewalling pairs accessed: 1971278
411: stonewalling pairs accessed: 2003175
44: stonewalling pairs accessed: 1977376
37: stonewalling pairs accessed: 1992656
92: stonewalling pairs accessed: 2007328
76: stonewalling pairs accessed: 1995498
393: stonewalling pairs accessed: 1997339
279: stonewalling pairs accessed: 1996675
180: stonewalling pairs accessed: 1966149
85: stonewalling pairs accessed: 1983128
48: stonewalling pairs accessed: 1977386
61: stonewalling pairs accessed: 1993879
54: stonewalling pairs accessed: 1975903
287: stonewalling pairs accessed: 1992187
322: stonewalling pairs accessed: 1982326
96: stonewalling pairs accessed: 2007302
3: stonewalling pairs accessed: 1970194
99: stonewalling pairs accessed: 2007313
170: stonewalling pairs accessed: 1971249
115: stonewalling pairs accessed: 1974676
8: stonewalling pairs accessed: 1970232
121: stonewalling pairs accessed: 1990687
274: stonewalling pairs accessed: 1996672
357: stonewalling pairs accessed: 2005052
402: stonewalling pairs accessed: 1993793
116: stonewalling pairs accessed: 1974672
73: stonewalling pairs accessed: 1995492
59: stonewalling pairs accessed: 1975904
110: stonewalling pairs accessed: 1974654
205: stonewalling pairs accessed: 1991395
338: stonewalling pairs accessed: 2004656
351: stonewalling pairs accessed: 2005065
374: stonewalling pairs accessed: 2002224
147: stonewalling pairs accessed: 2010975
122: stonewalling pairs accessed: 1990686
156: stonewalling pairs accessed: 1983734
133: stonewalling pairs accessed: 1989526
89: stonewalling pairs accessed: 1983129
0: stonewalling pairs accessed: 1970199
235: stonewalling pairs accessed: 2005965
418: stonewalling pairs accessed: 2003184
51: stonewalling pairs accessed: 1975914
14: stonewalling pairs accessed: 1996484
337: stonewalling pairs accessed: 2004650
152: stonewalling pairs accessed: 1983721
306: stonewalling pairs accessed: 1998524
193: stonewalling pairs accessed: 2009760
229: stonewalling pairs accessed: 1994710
300: stonewalling pairs accessed: 1998519
5: stonewalling pairs accessed: 1970226
198: stonewalling pairs accessed: 2009788
91: stonewalling pairs accessed: 2007307
84: stonewalling pairs accessed: 1983126
397: stonewalling pairs accessed: 1997347
352: stonewalling pairs accessed: 2005063
356: stonewalling pairs accessed: 2005057
203: stonewalling pairs accessed: 1991395
259: stonewalling pairs accessed: 1993441
160: stonewalling pairs accessed: 1993525
155: stonewalling pairs accessed: 1983694
408: stonewalling pairs accessed: 1993798
321: stonewalling pairs accessed: 1982328
384: stonewalling pairs accessed: 1997001
257: stonewalling pairs accessed: 1993427
62: stonewalling pairs accessed: 1993872
406: stonewalling pairs accessed: 1993782
263: stonewalling pairs accessed: 1988711
69: stonewalling pairs accessed: 1993893
415: stonewalling pairs accessed: 2003163
98: stonewalling pairs accessed: 2007330
131: stonewalling pairs accessed: 1989542
304: stonewalling pairs accessed: 1998542
277: stonewalling pairs accessed: 1996660
252: stonewalling pairs accessed: 1993432
16: stonewalling pairs accessed: 1996507
123: stonewalling pairs accessed: 1990675
369: stonewalling pairs accessed: 1993709
45: stonewalling pairs accessed: 1977392
378: stonewalling pairs accessed: 2002258
161: stonewalling pairs accessed: 1993530
94: stonewalling pairs accessed: 2007325
297: stonewalling pairs accessed: 1987914
362: stonewalling pairs accessed: 1993705
46: stonewalling pairs accessed: 1977384
213: stonewalling pairs accessed: 1976998
409: stonewalling pairs accessed: 1993800
305: stonewalling pairs accessed: 1998541
188: stonewalling pairs accessed: 1966166
171: stonewalling pairs accessed: 1971277
174: stonewalling pairs accessed: 1971276
377: stonewalling pairs accessed: 2002231
392: stonewalling pairs accessed: 1997319
56: stonewalling pairs accessed: 1975914
353: stonewalling pairs accessed: 2005077
19: stonewalling pairs accessed: 1996500
395: stonewalling pairs accessed: 1997333
268: stonewalling pairs accessed: 1988713
401: stonewalling pairs accessed: 1993795
284: stonewalling pairs accessed: 1992192
27: stonewalling pairs accessed: 1985673
112: stonewalling pairs accessed: 1974649
66: stonewalling pairs accessed: 1993861
383: stonewalling pairs accessed: 1997007
119: stonewalling pairs accessed: 1974706
145: stonewalling pairs accessed: 2010985
208: stonewalling pairs accessed: 1991429
141: stonewalling pairs accessed: 2010977
404: stonewalling pairs accessed: 1993798
137: stonewalling pairs accessed: 1989536
32: stonewalling pairs accessed: 1992638
136: stonewalling pairs accessed: 1989542
403: stonewalling pairs accessed: 1993792
209: stonewalling pairs accessed: 1991406
15: stonewalling pairs accessed: 1996495
258: stonewalling pairs accessed: 1993448
31: stonewalling pairs accessed: 1992651
154: stonewalling pairs accessed: 1983731
187: stonewalling pairs accessed: 1966139
72: stonewalling pairs accessed: 1995425
186: stonewalling pairs accessed: 1966151
23: stonewalling pairs accessed: 1985665
329: stonewalling pairs accessed: 1982341
135: stonewalling pairs accessed: 1989545
278: stonewalling pairs accessed: 1996689
291: stonewalling pairs accessed: 1987884
204: stonewalling pairs accessed: 1991412
387: stonewalling pairs accessed: 1997013
182: stonewalling pairs accessed: 1966162
196: stonewalling pairs accessed: 2009762
113: stonewalling pairs accessed: 1974660
349: stonewalling pairs accessed: 1977633
125: stonewalling pairs accessed: 1990668
328: stonewalling pairs accessed: 1982335
71: stonewalling pairs accessed: 1995498
254: stonewalling pairs accessed: 1993437
417: stonewalling pairs accessed: 2003183
212: stonewalling pairs accessed: 1977002
376: stonewalling pairs accessed: 2002243
153: stonewalling pairs accessed: 1983704
399: stonewalling pairs accessed: 1997344
245: stonewalling pairs accessed: 1977605
148: stonewalling pairs accessed: 2010996
381: stonewalling pairs accessed: 1996993
364: stonewalling pairs accessed: 1993717
217: stonewalling pairs accessed: 1977007
272: stonewalling pairs accessed: 1996653
396: stonewalling pairs accessed: 1997345
303: stonewalling pairs accessed: 1998528
39: stonewalling pairs accessed: 1992650
255: stonewalling pairs accessed: 1993445
68: stonewalling pairs accessed: 1993891
1: stonewalling pairs accessed: 1970198
114: stonewalling pairs accessed: 1974702
127: stonewalling pairs accessed: 1990693
372: stonewalling pairs accessed: 2002241
106: stonewalling pairs accessed: 1996533
313: stonewalling pairs accessed: 1995882
299: stonewalling pairs accessed: 1987901
35: stonewalling pairs accessed: 1992626
298: stonewalling pairs accessed: 1987912
191: stonewalling pairs accessed: 2009756
414: stonewalling pairs accessed: 2003171
317: stonewalling pairs accessed: 1995896
52: stonewalling pairs accessed: 1975914
176: stonewalling pairs accessed: 1971277
13: stonewalling pairs accessed: 1996481
309: stonewalling pairs accessed: 1998548
215: stonewalling pairs accessed: 1976998
238: stonewalling pairs accessed: 2005986
11: stonewalling pairs accessed: 1996517
224: stonewalling pairs accessed: 1994702
17: stonewalling pairs accessed: 1996490
172: stonewalling pairs accessed: 1971240
286: stonewalling pairs accessed: 1992188
33: stonewalling pairs accessed: 1992643
9: stonewalling pairs accessed: 1970236
405: stonewalling pairs accessed: 1993791
348: stonewalling pairs accessed: 1977642
81: stonewalling pairs accessed: 1983160
294: stonewalling pairs accessed: 1987921
57: stonewalling pairs accessed: 1975908
192: stonewalling pairs accessed: 2009761
6: stonewalling pairs accessed: 1970235
273: stonewalling pairs accessed: 1996665
29: stonewalling pairs accessed: 1985668
105: stonewalling pairs accessed: 1996560
358: stonewalling pairs accessed: 2005081
371: stonewalling pairs accessed: 2002213
214: stonewalling pairs accessed: 1977013
97: stonewalling pairs accessed: 2007322
302: stonewalling pairs accessed: 1998525
416: stonewalling pairs accessed: 2003175
223: stonewalling pairs accessed: 1994708
49: stonewalling pairs accessed: 1977393
165: stonewalling pairs accessed: 1993519
118: stonewalling pairs accessed: 1974646
241: stonewalling pairs accessed: 1977590
4: stonewalling pairs accessed: 1970198
107: stonewalling pairs accessed: 1996533
332: stonewalling pairs accessed: 2004644
236: stonewalling pairs accessed: 2005949
413: stonewalling pairs accessed: 2003179
109: stonewalling pairs accessed: 1996565
175: stonewalling pairs accessed: 1971278
38: stonewalling pairs accessed: 1992650
181: stonewalling pairs accessed: 1966149
314: stonewalling pairs accessed: 1995900
117: stonewalling pairs accessed: 1974664
102: stonewalling pairs accessed: 1996543
146: stonewalling pairs accessed: 2010990
373: stonewalling pairs accessed: 2002216
139: stonewalling pairs accessed: 1989537
355: stonewalling pairs accessed: 2005072
88: stonewalling pairs accessed: 1983146
341: stonewalling pairs accessed: 1977635
234: stonewalling pairs accessed: 2005946
157: stonewalling pairs accessed: 1983707
222: stonewalling pairs accessed: 1994711
316: stonewalling pairs accessed: 1995887
183: stonewalling pairs accessed: 1966150
149: stonewalling pairs accessed: 2010995
265: stonewalling pairs accessed: 1988718
288: stonewalling pairs accessed: 1992206
211: stonewalling pairs accessed: 1976997
124: stonewalling pairs accessed: 1990675
267: stonewalling pairs accessed: 1988713
292: stonewalling pairs accessed: 1987881
86: stonewalling pairs accessed: 1983160
233: stonewalling pairs accessed: 2005981
169: stonewalling pairs accessed: 1993557
285: stonewalling pairs accessed: 1992197
78: stonewalling pairs accessed: 1995505
201: stonewalling pairs accessed: 1991410
264: stonewalling pairs accessed: 1988714
307: stonewalling pairs accessed: 1998540
132: stonewalling pairs accessed: 1989537
336: stonewalling pairs accessed: 2004627
93: stonewalling pairs accessed: 2007308
179: stonewalling pairs accessed: 1971265
55: stonewalling pairs accessed: 1975921
158: stonewalling pairs accessed: 1983736
251: stonewalling pairs accessed: 1993409
324: stonewalling pairs accessed: 1982326
207: stonewalling pairs accessed: 1991394
282: stonewalling pairs accessed: 1992193
366: stonewalling pairs accessed: 1993718
63: stonewalling pairs accessed: 1993861
189: stonewalling pairs accessed: 1966177
65: stonewalling pairs accessed: 1993872
308: stonewalling pairs accessed: 1998549
101: stonewalling pairs accessed: 1996543
394: stonewalling pairs accessed: 1997354
347: stonewalling pairs accessed: 1977629
232: stonewalling pairs accessed: 2005985
386: stonewalling pairs accessed: 1997007
53: stonewalling pairs accessed: 1975914
219: stonewalling pairs accessed: 1976991
248: stonewalling pairs accessed: 1977604
311: stonewalling pairs accessed: 1995899
104: stonewalling pairs accessed: 1996539
327: stonewalling pairs accessed: 1982328
142: stonewalling pairs accessed: 2010993
326: stonewalling pairs accessed: 1982331
323: stonewalling pairs accessed: 1982318
239: stonewalling pairs accessed: 2005966
318: stonewalling pairs accessed: 1995912
111: stonewalling pairs accessed: 1974701
64: stonewalling pairs accessed: 1993876
47: stonewalling pairs accessed: 1977400
202: stonewalling pairs accessed: 1991431
126: stonewalling pairs accessed: 1990674
43: stonewalling pairs accessed: 1977385
269: stonewalling pairs accessed: 1988718
228: stonewalling pairs accessed: 1994702
301: stonewalling pairs accessed: 1998519
354: stonewalling pairs accessed: 2005068
227: stonewalling pairs accessed: 1994702
412: stonewalling pairs accessed: 2003170
276: stonewalling pairs accessed: 1996655
333: stonewalling pairs accessed: 2004655
289: stonewalling pairs accessed: 1992193
108: stonewalling pairs accessed: 1996568
271: stonewalling pairs accessed: 1996665
34: stonewalling pairs accessed: 1992650
237: stonewalling pairs accessed: 2005978
2: stonewalling pairs accessed: 1970198
346: stonewalling pairs accessed: 1977629
283: stonewalling pairs accessed: 1992193
319: stonewalling pairs accessed: 1995912
75: stonewalling pairs accessed: 1995493
168: stonewalling pairs accessed: 1993547
281: stonewalling pairs accessed: 1992205
184: stonewalling pairs accessed: 1966176
367: stonewalling pairs accessed: 1993718
242: stonewalling pairs accessed: 1977622
206: stonewalling pairs accessed: 1991426
83: stonewalling pairs accessed: 1983175
95: stonewalling pairs accessed: 2007295
388: stonewalling pairs accessed: 1997034
41: stonewalling pairs accessed: 1977401
345: stonewalling pairs accessed: 1977629
225: stonewalling pairs accessed: 1994702
185: stonewalling pairs accessed: 1966185
315: stonewalling pairs accessed: 1995992
stonewalling pairs accessed min: 1966139 max: 2010996 -- min data: 86.1 GiB mean data: 87.2 GiB time: 300.2s
WARNING: Expected aggregate file size       = 197433600000000.
WARNING: Stat() of aggregate file size      = 39703817986560.
WARNING: Using actual aggregate bytes moved = 39703817986560.
WARNING: maybe caused by deadlineForStonewalling
write     123514     2755466    306.00      45.91      45.91      0.037577   306.52     0.002753   306.56     0   
Max Write: 123514.11 MiB/sec (129513.93 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      123514.11  123514.11  123514.11       0.00 2755146.65 2755146.65 2755146.65       0.00  306.56020     300.21     124872.63     0    420  42    1   0     1        1         0    0 10000000    47008    47008 37864512.0 DFS      0
Finished            : Mon Jun 29 05:03:31 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

mdtest_easy_delete
-- started at 06/29/2020 05:33:08 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 10 node(s)
Command line used: /home/daos/io-500/install/ior/bin/mdtest '-r' '-F' '-P' '-d' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_easy' '-n' '10000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442' '--dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/mnt/dfuse/' '-x' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_easy-stonewall' '-N' '1'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    2795341.981    2795336.879    2795339.161          1.442
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.091          0.091          0.091          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        569.019        569.018        569.019          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         10.972         10.972         10.972          0.000
-- finished at 06/29/2020 05:42:48 --

mdtest_easy_stat
-- started at 06/29/2020 05:19:38 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 10 node(s)
Command line used: /home/daos/io-500/install/ior/bin/mdtest '-T' '-F' '-P' '-d' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_easy' '-n' '10000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442' '--dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/mnt/dfuse/' '-x' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_easy-stonewall' '-N' '1'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    4600364.227    4600352.540    4600357.600          3.773
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        345.756        345.755        345.756          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/29/2020 05:25:24 --

mdtest_easy_write
-- started at 06/29/2020 04:52:36 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 10 node(s)
Command line used: /home/daos/io-500/install/ior/bin/mdtest '-Y' '-C' '-F' '-P' '-d' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_easy' '-n' '10000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442' '--dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/mnt/dfuse/' '-x' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files
Continue stonewall hit min: 3424004 max: 3787145 avg: 3694808.7 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    4828033.910    4828031.455    4828032.683          0.731
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    5166266.675             NA
   Tree creation             :         29.297         29.297         29.297          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        329.451        329.451        329.451          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.375             NA
   Tree creation             :          0.034          0.034          0.034          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/29/2020 04:58:05 --

mdtest_hard_delete
-- started at 06/29/2020 05:47:15 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 10 node(s)
Command line used: /home/daos/io-500/install/ior/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_hard' '-n' '10000000' '-x' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442' '--dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/mnt/dfuse/' '-N' '1'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    2768936.394    2768925.365    2768929.528          3.539
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.001          0.001          0.001          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        271.303        271.302        271.302          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :       1760.853       1760.853       1760.853          0.000
-- finished at 06/29/2020 06:21:08 --

mdtest_hard_read
-- started at 06/29/2020 05:43:07 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 10 node(s)
Command line used: /home/daos/io-500/install/ior/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_hard' '-n' '10000000' '-x' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442' '--dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/mnt/dfuse/' '-N' '1'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :    3231387.776    3231378.061    3231381.925          2.980
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        232.476        232.475        232.475          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/29/2020 05:47:00 --

mdtest_hard_stat
-- started at 06/29/2020 05:29:49 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 10 node(s)
Command line used: /home/daos/io-500/install/ior/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_hard' '-n' '10000000' '-x' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442' '--dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/mnt/dfuse/' '-N' '1'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    4109814.668    4109798.111    4109802.910          5.308
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        182.787        182.786        182.787          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/29/2020 05:32:52 --

mdtest_hard_write
-- started at 06/29/2020 05:03:46 --

mdtest-3.3.0+dev was launched with 420 total task(s) on 10 node(s)
Command line used: /home/daos/io-500/install/ior/bin/mdtest '-Y' '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_hard' '-n' '10000000' '-x' '/mnt/dfuse/datafiles/2020.06.29-04.46.44-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=9ae1db5f-54fa-4822-81cd-2faaaeacc442' '--dfs.cont=9f8cf146-1aec-46d8-8789-5a0746355be1' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/mnt/dfuse/' '-N' '1' '-W' '300'
Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
420 tasks, 4200000000 files
Continue stonewall hit min: 1646018 max: 1788611 avg: 1750425.0 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    2317039.975    2317037.440    2317038.782          0.914
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    2448913.059             NA
   Tree creation             :        455.091        455.091        455.091          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        324.214        324.214        324.214          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.206             NA
   Tree creation             :          0.002          0.002          0.002          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/29/2020 05:09:10 --

result_summary
[RESULT] BW   phase 1            ior_easy_write              178.287 GiB/s : time 315.32 seconds
[RESULT] IOPS phase 1         mdtest_easy_write             4828.030 kiops : time 329.45 seconds
[RESULT] BW   phase 2            ior_hard_write              120.619 GiB/s : time 306.52 seconds
[RESULT] IOPS phase 2         mdtest_hard_write             2317.040 kiops : time 324.21 seconds
[RESULT] IOPS phase 3                      find             4194.620 kiops : time 331.37 seconds
[RESULT] BW   phase 3             ior_easy_read              212.834 GiB/s : time 264.14 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             4600.360 kiops : time 345.76 seconds
[RESULT] BW   phase 4             ior_hard_read              161.045 GiB/s : time 229.58 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat             4109.810 kiops : time 182.79 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete             2795.340 kiops : time 569.02 seconds
[RESULT] IOPS phase 7          mdtest_hard_read             3231.390 kiops : time 232.48 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete             2768.940 kiops : time 2047.56 seconds
[SCORE] Bandwidth 164.771 GiB/s : IOPS 3493.56 kiops : TOTAL 758.708