Presque

Institution Argonne National Laboratory
Client Procs Per Node
Client Operating System RHEL
Client Operating System Version 7.7
Client Kernel Version 3.10.0-1062.4.3.el7.x86_64

DATA SERVER

Storage Type NVMe-SSD
Volatile Memory 384
Storage Interface NVMe
Network 100gE
Software Version 1.1
OS Version 7.7

INFORMATION

Client Nodes 10
Client Total Procs 380
Metadata Nodes 8
Metadata Storage Devices 12
Data Nodes 8
Data Storage Devices 12

METADATA

Easy Write 3,315.18 kIOP/s
Easy Stat 3,403.63 kIOP/s
Easy Delete 1,916.96 kIOP/s
Hard Write 725.92 kIOP/s
Hard Read 1,575.54 kIOP/s
Hard Stat 2,900.14 kIOP/s
Hard Delete 1,822.73 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=/home/kalfizah/io500-new/bin/ior
  io500_mdtest_cmd=/home/kalfizah/io500-new/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
#  io500_mpiargs="-envnone --bind-to socket -np 12"
  io500_mpiargs="--bind-to socket -np 380 -f /home/kalfizah/io500-new/hfile"
#  io500_mpiargs="-np 420 --hostfile /home/daos/io-500/config/cli_hosts"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

#  if [[ -s "system-information.txt" ]]; then
#    echo "Warning: please create a system-information.txt description by"
#    echo "copying the information from https://vi4io.org/io500-info-creator/"
#  else
#    cp "system-information.txt" $io500_result_dir
#  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val" && io500_find_mpi="True"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
[0] Pool uuid = a04776a5-bd54-4bf9-ae46-c989b9407c8d, SVCL = 1

[0] DFS Container namespace uuid = b0599c58-2822-45b2-9e09-23a335940082

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Jul 10 10:54:39 2020
Command line        : /home/kalfizah/io500-new/bin/ior -r -R -a DFS --dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d --dfs.cont=b0599c58-2822-45b2-9e09-23a335940082 --dfs.svcl=1 --dfs.prefix=/tmp/dfuse/ -t 256k -v -b 256gm -i 1 -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_easy/stonewall
Machine             : Linux daos08.ftm.alcf.anl.gov
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Fri Jul 10 10:54:39 2020
Participating tasks: 380
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 380
clients per node    : 38
repetitions         : 1
xfersize            : 262144 bytes
blocksize           : 256 GiB
aggregate filesize  : 95 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Fri Jul 10 10:54:42 2020

WARNING: Expected aggregate file size       = 104453604638720.
WARNING: Stat() of aggregate file size      = 104294604079104.
WARNING: Using actual aggregate bytes moved = 44033391984640.
read      135116     544650     0.000294    268435456  256.00     2.39       308.41     0.000906   310.80     0   
Max Read:  135115.97 MiB/sec (141679.37 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       135115.97  135115.97  135115.97       0.00  540463.90  540463.90  540463.90       0.00  310.79608         NA            NA     0    380  38    1   0     1        1         0    0      1 274877906944   262144 41993516.0 DFS      0
Finished            : Fri Jul 10 10:59:51 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_easy_write
[0] Pool uuid = a04776a5-bd54-4bf9-ae46-c989b9407c8d, SVCL = 1

[0] DFS Container namespace uuid = b0599c58-2822-45b2-9e09-23a335940082

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Jul 10 10:28:24 2020
Command line        : /home/kalfizah/io500-new/bin/ior -w -a DFS --dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d --dfs.cont=b0599c58-2822-45b2-9e09-23a335940082 --dfs.svcl=1 --dfs.prefix=/tmp/dfuse/ -t 256k -v -b 256gm -i 1 -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux daos08.ftm.alcf.anl.gov
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Fri Jul 10 10:28:24 2020
Participating tasks: 380
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 380
clients per node    : 38
repetitions         : 1
xfersize            : 262144 bytes
blocksize           : 256 GiB
aggregate filesize  : 95 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Fri Jul 10 10:28:29 2020
141: stonewalling pairs accessed: 440003
211: stonewalling pairs accessed: 437569
341: stonewalling pairs accessed: 438824
91: stonewalling pairs accessed: 438345
131: stonewalling pairs accessed: 438870
231: stonewalling pairs accessed: 439649
351: stonewalling pairs accessed: 436964
61: stonewalling pairs accessed: 437655
101: stonewalling pairs accessed: 439928
81: stonewalling pairs accessed: 440242
311: stonewalling pairs accessed: 437492
371: stonewalling pairs accessed: 437985
221: stonewalling pairs accessed: 436473
98: stonewalling pairs accessed: 439614
271: stonewalling pairs accessed: 438831
218: stonewalling pairs accessed: 440020
9: stonewalling pairs accessed: 438587
67: stonewalling pairs accessed: 440430
41: stonewalling pairs accessed: 438657
228: stonewalling pairs accessed: 439195
69: stonewalling pairs accessed: 436862
77: stonewalling pairs accessed: 439113
171: stonewalling pairs accessed: 436534
298: stonewalling pairs accessed: 438401
129: stonewalling pairs accessed: 440259
287: stonewalling pairs accessed: 435842
301: stonewalling pairs accessed: 439529
18: stonewalling pairs accessed: 439127
149: stonewalling pairs accessed: 436361
26: stonewalling pairs accessed: 437725
31: stonewalling pairs accessed: 437084
128: stonewalling pairs accessed: 438476
249: stonewalling pairs accessed: 440009
37: stonewalling pairs accessed: 438788
214: stonewalling pairs accessed: 440502
356: stonewalling pairs accessed: 438620
60: stonewalling pairs accessed: 439779
191: stonewalling pairs accessed: 438065
148: stonewalling pairs accessed: 437727
19: stonewalling pairs accessed: 438858
57: stonewalling pairs accessed: 437579
284: stonewalling pairs accessed: 436927
286: stonewalling pairs accessed: 439596
80: stonewalling pairs accessed: 439984
71: stonewalling pairs accessed: 437663
168: stonewalling pairs accessed: 436448
79: stonewalling pairs accessed: 438892
87: stonewalling pairs accessed: 436988
15: stonewalling pairs accessed: 438675
344: stonewalling pairs accessed: 440248
316: stonewalling pairs accessed: 438926
170: stonewalling pairs accessed: 437789
181: stonewalling pairs accessed: 438470
78: stonewalling pairs accessed: 439317
99: stonewalling pairs accessed: 437810
153: stonewalling pairs accessed: 439718
177: stonewalling pairs accessed: 440141
55: stonewalling pairs accessed: 436413
124: stonewalling pairs accessed: 439473
56: stonewalling pairs accessed: 439857
180: stonewalling pairs accessed: 438358
361: stonewalling pairs accessed: 440754
28: stonewalling pairs accessed: 438625
109: stonewalling pairs accessed: 439388
13: stonewalling pairs accessed: 439419
277: stonewalling pairs accessed: 438367
12: stonewalling pairs accessed: 439625
95: stonewalling pairs accessed: 438242
264: stonewalling pairs accessed: 438505
200: stonewalling pairs accessed: 440399
1: stonewalling pairs accessed: 440331
208: stonewalling pairs accessed: 437890
119: stonewalling pairs accessed: 437309
33: stonewalling pairs accessed: 438915
307: stonewalling pairs accessed: 439089
102: stonewalling pairs accessed: 436500
105: stonewalling pairs accessed: 441316
334: stonewalling pairs accessed: 439499
96: stonewalling pairs accessed: 439806
240: stonewalling pairs accessed: 440163
11: stonewalling pairs accessed: 438334
278: stonewalling pairs accessed: 439329
159: stonewalling pairs accessed: 438929
173: stonewalling pairs accessed: 439833
257: stonewalling pairs accessed: 436755
132: stonewalling pairs accessed: 437745
205: stonewalling pairs accessed: 438356
364: stonewalling pairs accessed: 439352
106: stonewalling pairs accessed: 436560
270: stonewalling pairs accessed: 437771
21: stonewalling pairs accessed: 436789
188: stonewalling pairs accessed: 437741
199: stonewalling pairs accessed: 438904
263: stonewalling pairs accessed: 439052
377: stonewalling pairs accessed: 438634
362: stonewalling pairs accessed: 440228
215: stonewalling pairs accessed: 437884
74: stonewalling pairs accessed: 437744
166: stonewalling pairs accessed: 436775
350: stonewalling pairs accessed: 437583
111: stonewalling pairs accessed: 439932
318: stonewalling pairs accessed: 438608
229: stonewalling pairs accessed: 439096
293: stonewalling pairs accessed: 441665
7: stonewalling pairs accessed: 440587
2: stonewalling pairs accessed: 439062
285: stonewalling pairs accessed: 440702
204: stonewalling pairs accessed: 439503
206: stonewalling pairs accessed: 437013
0: stonewalling pairs accessed: 438474
151: stonewalling pairs accessed: 434656
378: stonewalling pairs accessed: 437703
259: stonewalling pairs accessed: 438141
363: stonewalling pairs accessed: 437879
197: stonewalling pairs accessed: 438318
52: stonewalling pairs accessed: 439502
355: stonewalling pairs accessed: 439171
254: stonewalling pairs accessed: 437460
266: stonewalling pairs accessed: 434874
20: stonewalling pairs accessed: 439400
161: stonewalling pairs accessed: 439142
8: stonewalling pairs accessed: 438012
269: stonewalling pairs accessed: 437823
83: stonewalling pairs accessed: 437437
347: stonewalling pairs accessed: 439030
62: stonewalling pairs accessed: 439841
85: stonewalling pairs accessed: 438772
34: stonewalling pairs accessed: 438650
376: stonewalling pairs accessed: 438560
40: stonewalling pairs accessed: 436525
291: stonewalling pairs accessed: 438211
138: stonewalling pairs accessed: 440124
319: stonewalling pairs accessed: 439424
193: stonewalling pairs accessed: 438550
137: stonewalling pairs accessed: 437627
92: stonewalling pairs accessed: 436004
115: stonewalling pairs accessed: 437853
54: stonewalling pairs accessed: 438952
36: stonewalling pairs accessed: 438928
110: stonewalling pairs accessed: 436384
121: stonewalling pairs accessed: 441386
158: stonewalling pairs accessed: 437026
349: stonewalling pairs accessed: 438311
233: stonewalling pairs accessed: 438516
147: stonewalling pairs accessed: 436949
122: stonewalling pairs accessed: 441164
265: stonewalling pairs accessed: 438205
224: stonewalling pairs accessed: 434395
86: stonewalling pairs accessed: 439940
120: stonewalling pairs accessed: 437138
201: stonewalling pairs accessed: 439334
258: stonewalling pairs accessed: 436931
289: stonewalling pairs accessed: 441016
93: stonewalling pairs accessed: 436376
97: stonewalling pairs accessed: 437829
192: stonewalling pairs accessed: 439936
295: stonewalling pairs accessed: 435951
304: stonewalling pairs accessed: 437778
116: stonewalling pairs accessed: 436693
160: stonewalling pairs accessed: 437299
251: stonewalling pairs accessed: 436788
268: stonewalling pairs accessed: 439506
279: stonewalling pairs accessed: 438056
343: stonewalling pairs accessed: 438324
117: stonewalling pairs accessed: 439044
212: stonewalling pairs accessed: 440366
325: stonewalling pairs accessed: 438210
24: stonewalling pairs accessed: 437701
136: stonewalling pairs accessed: 437107
220: stonewalling pairs accessed: 437810
241: stonewalling pairs accessed: 438401
288: stonewalling pairs accessed: 438606
299: stonewalling pairs accessed: 438939
63: stonewalling pairs accessed: 438971
297: stonewalling pairs accessed: 438471
302: stonewalling pairs accessed: 439206
275: stonewalling pairs accessed: 439534
174: stonewalling pairs accessed: 438326
156: stonewalling pairs accessed: 441809
310: stonewalling pairs accessed: 439007
261: stonewalling pairs accessed: 437252
368: stonewalling pairs accessed: 439023
369: stonewalling pairs accessed: 436934
353: stonewalling pairs accessed: 438837
27: stonewalling pairs accessed: 439255
322: stonewalling pairs accessed: 439851
135: stonewalling pairs accessed: 439296
194: stonewalling pairs accessed: 438877
306: stonewalling pairs accessed: 439494
360: stonewalling pairs accessed: 440174
281: stonewalling pairs accessed: 439298
58: stonewalling pairs accessed: 440605
219: stonewalling pairs accessed: 439106
373: stonewalling pairs accessed: 438587
167: stonewalling pairs accessed: 438875
82: stonewalling pairs accessed: 438031
165: stonewalling pairs accessed: 439865
294: stonewalling pairs accessed: 436095
366: stonewalling pairs accessed: 437465
90: stonewalling pairs accessed: 439280
321: stonewalling pairs accessed: 438730
238: stonewalling pairs accessed: 437329
139: stonewalling pairs accessed: 440435
23: stonewalling pairs accessed: 438565
317: stonewalling pairs accessed: 439331
182: stonewalling pairs accessed: 438991
35: stonewalling pairs accessed: 437647
154: stonewalling pairs accessed: 438496
146: stonewalling pairs accessed: 436989
300: stonewalling pairs accessed: 438481
51: stonewalling pairs accessed: 436264
248: stonewalling pairs accessed: 438054
359: stonewalling pairs accessed: 439842
43: stonewalling pairs accessed: 439996
17: stonewalling pairs accessed: 438567
202: stonewalling pairs accessed: 438560
75: stonewalling pairs accessed: 437610
184: stonewalling pairs accessed: 436786
216: stonewalling pairs accessed: 438529
50: stonewalling pairs accessed: 438009
331: stonewalling pairs accessed: 439658
328: stonewalling pairs accessed: 437597
59: stonewalling pairs accessed: 437117
53: stonewalling pairs accessed: 440991
267: stonewalling pairs accessed: 436667
252: stonewalling pairs accessed: 439294
175: stonewalling pairs accessed: 437541
244: stonewalling pairs accessed: 438336
256: stonewalling pairs accessed: 438748
330: stonewalling pairs accessed: 438889
108: stonewalling pairs accessed: 438997
169: stonewalling pairs accessed: 439148
283: stonewalling pairs accessed: 436489
337: stonewalling pairs accessed: 436459
312: stonewalling pairs accessed: 439308
195: stonewalling pairs accessed: 438533
324: stonewalling pairs accessed: 438273
16: stonewalling pairs accessed: 439360
150: stonewalling pairs accessed: 437074
358: stonewalling pairs accessed: 437394
179: stonewalling pairs accessed: 437515
123: stonewalling pairs accessed: 439986
357: stonewalling pairs accessed: 439815
162: stonewalling pairs accessed: 439282
225: stonewalling pairs accessed: 437089
4: stonewalling pairs accessed: 438360
46: stonewalling pairs accessed: 438330
340: stonewalling pairs accessed: 438298
178: stonewalling pairs accessed: 437275
329: stonewalling pairs accessed: 437897
163: stonewalling pairs accessed: 437097
207: stonewalling pairs accessed: 435518
42: stonewalling pairs accessed: 436615
25: stonewalling pairs accessed: 440180
44: stonewalling pairs accessed: 436411
326: stonewalling pairs accessed: 435691
280: stonewalling pairs accessed: 440370
48: stonewalling pairs accessed: 437973
49: stonewalling pairs accessed: 436862
213: stonewalling pairs accessed: 440993
157: stonewalling pairs accessed: 438636
72: stonewalling pairs accessed: 438954
65: stonewalling pairs accessed: 439381
94: stonewalling pairs accessed: 438891
346: stonewalling pairs accessed: 438132
10: stonewalling pairs accessed: 442037
338: stonewalling pairs accessed: 439474
189: stonewalling pairs accessed: 437465
243: stonewalling pairs accessed: 439080
247: stonewalling pairs accessed: 438502
32: stonewalling pairs accessed: 436138
315: stonewalling pairs accessed: 437497
114: stonewalling pairs accessed: 436900
176: stonewalling pairs accessed: 440217
30: stonewalling pairs accessed: 436376
88: stonewalling pairs accessed: 439780
29: stonewalling pairs accessed: 435548
273: stonewalling pairs accessed: 440875
217: stonewalling pairs accessed: 440158
112: stonewalling pairs accessed: 440045
375: stonewalling pairs accessed: 437041
314: stonewalling pairs accessed: 438172
226: stonewalling pairs accessed: 436552
70: stonewalling pairs accessed: 437572
308: stonewalling pairs accessed: 437989
209: stonewalling pairs accessed: 439058
3: stonewalling pairs accessed: 438572
237: stonewalling pairs accessed: 438559
292: stonewalling pairs accessed: 441369
145: stonewalling pairs accessed: 441035
274: stonewalling pairs accessed: 440699
76: stonewalling pairs accessed: 440062
100: stonewalling pairs accessed: 439001
348: stonewalling pairs accessed: 441968
379: stonewalling pairs accessed: 437691
73: stonewalling pairs accessed: 438515
367: stonewalling pairs accessed: 438374
342: stonewalling pairs accessed: 437524
235: stonewalling pairs accessed: 438867
144: stonewalling pairs accessed: 436347
336: stonewalling pairs accessed: 439937
130: stonewalling pairs accessed: 437393
118: stonewalling pairs accessed: 438808
39: stonewalling pairs accessed: 438913
313: stonewalling pairs accessed: 438412
227: stonewalling pairs accessed: 437622
352: stonewalling pairs accessed: 437951
125: stonewalling pairs accessed: 441114
374: stonewalling pairs accessed: 436306
66: stonewalling pairs accessed: 438700
140: stonewalling pairs accessed: 439642
89: stonewalling pairs accessed: 439312
333: stonewalling pairs accessed: 439488
47: stonewalling pairs accessed: 439263
142: stonewalling pairs accessed: 439779
335: stonewalling pairs accessed: 437827
104: stonewalling pairs accessed: 437528
126: stonewalling pairs accessed: 436049
190: stonewalling pairs accessed: 439290
38: stonewalling pairs accessed: 438126
223: stonewalling pairs accessed: 438863
107: stonewalling pairs accessed: 436485
282: stonewalling pairs accessed: 438251
155: stonewalling pairs accessed: 437807
234: stonewalling pairs accessed: 435430
186: stonewalling pairs accessed: 436442
210: stonewalling pairs accessed: 440793
103: stonewalling pairs accessed: 437118
127: stonewalling pairs accessed: 439375
172: stonewalling pairs accessed: 436463
255: stonewalling pairs accessed: 438598
14: stonewalling pairs accessed: 436689
276: stonewalling pairs accessed: 437977
230: stonewalling pairs accessed: 440760
253: stonewalling pairs accessed: 438183
222: stonewalling pairs accessed: 440004
365: stonewalling pairs accessed: 441315
134: stonewalling pairs accessed: 440012
246: stonewalling pairs accessed: 438966
250: stonewalling pairs accessed: 437687
303: stonewalling pairs accessed: 436544
242: stonewalling pairs accessed: 441262
245: stonewalling pairs accessed: 439081
64: stonewalling pairs accessed: 438199
296: stonewalling pairs accessed: 438183
290: stonewalling pairs accessed: 439543
198: stonewalling pairs accessed: 435113
133: stonewalling pairs accessed: 439510
22: stonewalling pairs accessed: 436832
345: stonewalling pairs accessed: 440983
164: stonewalling pairs accessed: 440169
196: stonewalling pairs accessed: 437623
320: stonewalling pairs accessed: 436917
143: stonewalling pairs accessed: 435646
232: stonewalling pairs accessed: 437734
185: stonewalling pairs accessed: 440344
236: stonewalling pairs accessed: 437037
370: stonewalling pairs accessed: 438592
203: stonewalling pairs accessed: 438974
262: stonewalling pairs accessed: 439397
6: stonewalling pairs accessed: 437020
260: stonewalling pairs accessed: 438454
113: stonewalling pairs accessed: 439686
332: stonewalling pairs accessed: 437195
339: stonewalling pairs accessed: 437012
323: stonewalling pairs accessed: 438329
152: stonewalling pairs accessed: 440687
84: stonewalling pairs accessed: 439826
272: stonewalling pairs accessed: 437669
239: stonewalling pairs accessed: 438661
187: stonewalling pairs accessed: 437956
45: stonewalling pairs accessed: 440449
309: stonewalling pairs accessed: 437529
5: stonewalling pairs accessed: 440252
305: stonewalling pairs accessed: 440316
372: stonewalling pairs accessed: 438749
68: stonewalling pairs accessed: 439289
327: stonewalling pairs accessed: 436718
183: stonewalling pairs accessed: 437400
354: stonewalling pairs accessed: 435951
stonewalling pairs accessed min: 434395 max: 442037 -- min data: 106.1 GiB mean data: 107.1 GiB time: 300.0s
WARNING: Expected aggregate file size       = 104453604638720.
WARNING: Stat() of aggregate file size      = 104294604079104.
WARNING: Using actual aggregate bytes moved = 44033391984640.
WARNING: maybe caused by deadlineForStonewalling
write     136479     554219     0.000288    268435456  256.00     4.61       303.08     0.000837   307.69     0   
Max Write: 136479.06 MiB/sec (143108.66 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      136479.06  136479.06  136479.06       0.00  545916.23  545916.23  545916.23       0.00  307.69201     300.00     138857.10     0    380  38    1   0     1        1         0    0      1 274877906944   262144 41993516.0 DFS      0
Finished            : Fri Jul 10 10:33:33 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_hard_read
[0] Pool uuid = a04776a5-bd54-4bf9-ae46-c989b9407c8d, SVCL = 1

[0] DFS Container namespace uuid = b0599c58-2822-45b2-9e09-23a335940082

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Jul 10 11:05:04 2020
Command line        : /home/kalfizah/io500-new/bin/ior -r -R -s 1600000 -a DFS --dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d --dfs.cont=b0599c58-2822-45b2-9e09-23a335940082 --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse/ -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_hard/stonewall
Machine             : Linux daos08.ftm.alcf.anl.gov
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Fri Jul 10 11:05:04 2020
Participating tasks: 380
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1600000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 380
clients per node    : 38
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 25.99 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Fri Jul 10 11:05:06 2020

WARNING: Expected aggregate file size       = 28580864000000.
WARNING: Stat() of aggregate file size      = 26758101535360.
WARNING: Using actual aggregate bytes moved = 26758101535360.
read      60607      1360356    418.44      45.91      45.91      2.61       418.44     0.001476   421.05     0   
Max Read:  60606.54 MiB/sec (63550.57 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        60606.54   60606.54   60606.54       0.00 1351909.61 1351909.61 1351909.61       0.00  421.05213         NA            NA     0    380  38    1   0     1        1         0    0 1600000    47008    47008 25518514.0 DFS      0
Finished            : Fri Jul 10 11:12:05 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

ior_hard_write
[0] Pool uuid = a04776a5-bd54-4bf9-ae46-c989b9407c8d, SVCL = 1

[0] DFS Container namespace uuid = b0599c58-2822-45b2-9e09-23a335940082

IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Jul 10 10:38:56 2020
Command line        : /home/kalfizah/io500-new/bin/ior -w -s 1600000 -a DFS --dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d --dfs.cont=b0599c58-2822-45b2-9e09-23a335940082 --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse/ -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux daos08.ftm.alcf.anl.gov
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Fri Jul 10 10:38:56 2020
Participating tasks: 380
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1600000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 380
clients per node    : 38
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 25.99 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Fri Jul 10 10:39:01 2020
329: stonewalling pairs accessed: 1490916
59: stonewalling pairs accessed: 1481065
127: stonewalling pairs accessed: 1489492
151: stonewalling pairs accessed: 1486045
219: stonewalling pairs accessed: 1488361
157: stonewalling pairs accessed: 1486035
131: stonewalling pairs accessed: 1486098
327: stonewalling pairs accessed: 1490929
161: stonewalling pairs accessed: 1490996
100: stonewalling pairs accessed: 1496626
159: stonewalling pairs accessed: 1486027
187: stonewalling pairs accessed: 1489092
171: stonewalling pairs accessed: 1489703
36: stonewalling pairs accessed: 1490414
8: stonewalling pairs accessed: 1497135
90: stonewalling pairs accessed: 1495706
359: stonewalling pairs accessed: 1483303
307: stonewalling pairs accessed: 1484743
271: stonewalling pairs accessed: 1488642
106: stonewalling pairs accessed: 1496626
138: stonewalling pairs accessed: 1486101
310: stonewalling pairs accessed: 1484987
34: stonewalling pairs accessed: 1490412
39: stonewalling pairs accessed: 1490407
57: stonewalling pairs accessed: 1481138
91: stonewalling pairs accessed: 1495646
206: stonewalling pairs accessed: 1487011
258: stonewalling pairs accessed: 1494753
250: stonewalling pairs accessed: 1494601
244: stonewalling pairs accessed: 1482755
139: stonewalling pairs accessed: 1485681
217: stonewalling pairs accessed: 1488357
81: stonewalling pairs accessed: 1487178
85: stonewalling pairs accessed: 1487105
116: stonewalling pairs accessed: 1493258
278: stonewalling pairs accessed: 1488579
304: stonewalling pairs accessed: 1484659
259: stonewalling pairs accessed: 1494874
287: stonewalling pairs accessed: 1496476
341: stonewalling pairs accessed: 1477776
275: stonewalling pairs accessed: 1488583
286: stonewalling pairs accessed: 1496187
43: stonewalling pairs accessed: 1482016
378: stonewalling pairs accessed: 1485362
20: stonewalling pairs accessed: 1475884
374: stonewalling pairs accessed: 1485208
299: stonewalling pairs accessed: 1475429
107: stonewalling pairs accessed: 1496626
11: stonewalling pairs accessed: 1490246
56: stonewalling pairs accessed: 1481066
218: stonewalling pairs accessed: 1488355
160: stonewalling pairs accessed: 1490599
64: stonewalling pairs accessed: 1483897
339: stonewalling pairs accessed: 1476715
117: stonewalling pairs accessed: 1493258
241: stonewalling pairs accessed: 1482746
276: stonewalling pairs accessed: 1488607
178: stonewalling pairs accessed: 1490200
320: stonewalling pairs accessed: 1490919
109: stonewalling pairs accessed: 1496624
137: stonewalling pairs accessed: 1485385
361: stonewalling pairs accessed: 1484714
66: stonewalling pairs accessed: 1483718
53: stonewalling pairs accessed: 1481075
198: stonewalling pairs accessed: 1487804
180: stonewalling pairs accessed: 1489073
124: stonewalling pairs accessed: 1489424
249: stonewalling pairs accessed: 1482759
257: stonewalling pairs accessed: 1494879
31: stonewalling pairs accessed: 1490382
296: stonewalling pairs accessed: 1475432
288: stonewalling pairs accessed: 1496499
280: stonewalling pairs accessed: 1496444
154: stonewalling pairs accessed: 1486051
199: stonewalling pairs accessed: 1487802
237: stonewalling pairs accessed: 1485523
191: stonewalling pairs accessed: 1487803
306: stonewalling pairs accessed: 1484654
328: stonewalling pairs accessed: 1490917
290: stonewalling pairs accessed: 1475398
194: stonewalling pairs accessed: 1487867
279: stonewalling pairs accessed: 1488586
347: stonewalling pairs accessed: 1477786
311: stonewalling pairs accessed: 1484916
326: stonewalling pairs accessed: 1490926
63: stonewalling pairs accessed: 1483479
348: stonewalling pairs accessed: 1477790
340: stonewalling pairs accessed: 1477776
224: stonewalling pairs accessed: 1484914
309: stonewalling pairs accessed: 1484658
377: stonewalling pairs accessed: 1485490
1: stonewalling pairs accessed: 1497257
15: stonewalling pairs accessed: 1490195
346: stonewalling pairs accessed: 1477793
163: stonewalling pairs accessed: 1490997
358: stonewalling pairs accessed: 1483650
50: stonewalling pairs accessed: 1480639
344: stonewalling pairs accessed: 1477792
169: stonewalling pairs accessed: 1490597
67: stonewalling pairs accessed: 1483714
211: stonewalling pairs accessed: 1488450
35: stonewalling pairs accessed: 1490327
76: stonewalling pairs accessed: 1497957
373: stonewalling pairs accessed: 1485363
18: stonewalling pairs accessed: 1490273
70: stonewalling pairs accessed: 1497958
354: stonewalling pairs accessed: 1483975
269: stonewalling pairs accessed: 1481100
77: stonewalling pairs accessed: 1497959
321: stonewalling pairs accessed: 1490923
95: stonewalling pairs accessed: 1495662
156: stonewalling pairs accessed: 1486049
72: stonewalling pairs accessed: 1497701
333: stonewalling pairs accessed: 1476735
28: stonewalling pairs accessed: 1475895
130: stonewalling pairs accessed: 1485468
14: stonewalling pairs accessed: 1490214
369: stonewalling pairs accessed: 1484997
207: stonewalling pairs accessed: 1487012
111: stonewalling pairs accessed: 1493249
115: stonewalling pairs accessed: 1493388
176: stonewalling pairs accessed: 1490213
242: stonewalling pairs accessed: 1482602
363: stonewalling pairs accessed: 1484278
38: stonewalling pairs accessed: 1490414
170: stonewalling pairs accessed: 1490208
24: stonewalling pairs accessed: 1475885
69: stonewalling pairs accessed: 1483478
277: stonewalling pairs accessed: 1488641
121: stonewalling pairs accessed: 1489422
125: stonewalling pairs accessed: 1489496
196: stonewalling pairs accessed: 1487803
292: stonewalling pairs accessed: 1475430
13: stonewalling pairs accessed: 1490195
58: stonewalling pairs accessed: 1480644
210: stonewalling pairs accessed: 1488391
84: stonewalling pairs accessed: 1487098
289: stonewalling pairs accessed: 1496337
297: stonewalling pairs accessed: 1475425
371: stonewalling pairs accessed: 1485202
165: stonewalling pairs accessed: 1490692
246: stonewalling pairs accessed: 1482612
342: stonewalling pairs accessed: 1477660
223: stonewalling pairs accessed: 1484897
88: stonewalling pairs accessed: 1487447
220: stonewalling pairs accessed: 1484914
114: stonewalling pairs accessed: 1493265
349: stonewalling pairs accessed: 1477786
367: stonewalling pairs accessed: 1484719
21: stonewalling pairs accessed: 1475874
215: stonewalling pairs accessed: 1488384
356: stonewalling pairs accessed: 1483973
32: stonewalling pairs accessed: 1490401
283: stonewalling pairs accessed: 1496518
108: stonewalling pairs accessed: 1496640
260: stonewalling pairs accessed: 1481103
134: stonewalling pairs accessed: 1485813
89: stonewalling pairs accessed: 1487099
87: stonewalling pairs accessed: 1487103
51: stonewalling pairs accessed: 1481077
225: stonewalling pairs accessed: 1484917
6: stonewalling pairs accessed: 1497256
302: stonewalling pairs accessed: 1484749
33: stonewalling pairs accessed: 1490410
128: stonewalling pairs accessed: 1489497
330: stonewalling pairs accessed: 1476718
184: stonewalling pairs accessed: 1489105
319: stonewalling pairs accessed: 1484984
177: stonewalling pairs accessed: 1490202
331: stonewalling pairs accessed: 1476715
245: stonewalling pairs accessed: 1482764
26: stonewalling pairs accessed: 1475885
12: stonewalling pairs accessed: 1490257
103: stonewalling pairs accessed: 1496641
158: stonewalling pairs accessed: 1486274
10: stonewalling pairs accessed: 1490267
204: stonewalling pairs accessed: 1487016
189: stonewalling pairs accessed: 1489100
17: stonewalling pairs accessed: 1490249
251: stonewalling pairs accessed: 1494601
255: stonewalling pairs accessed: 1494888
126: stonewalling pairs accessed: 1489175
82: stonewalling pairs accessed: 1487103
113: stonewalling pairs accessed: 1493379
188: stonewalling pairs accessed: 1489071
80: stonewalling pairs accessed: 1487180
214: stonewalling pairs accessed: 1488452
9: stonewalling pairs accessed: 1497135
197: stonewalling pairs accessed: 1487867
291: stonewalling pairs accessed: 1475396
285: stonewalling pairs accessed: 1496480
136: stonewalling pairs accessed: 1485384
112: stonewalling pairs accessed: 1493258
193: stonewalling pairs accessed: 1487807
238: stonewalling pairs accessed: 1485586
110: stonewalling pairs accessed: 1493256
274: stonewalling pairs accessed: 1488582
79: stonewalling pairs accessed: 1497700
27: stonewalling pairs accessed: 1475881
71: stonewalling pairs accessed: 1497700
295: stonewalling pairs accessed: 1475399
316: stonewalling pairs accessed: 1484916
132: stonewalling pairs accessed: 1485801
203: stonewalling pairs accessed: 1487013
298: stonewalling pairs accessed: 1475426
190: stonewalling pairs accessed: 1487805
284: stonewalling pairs accessed: 1496465
19: stonewalling pairs accessed: 1490146
317: stonewalling pairs accessed: 1484911
351: stonewalling pairs accessed: 1483308
305: stonewalling pairs accessed: 1484659
366: stonewalling pairs accessed: 1484998
152: stonewalling pairs accessed: 1486057
323: stonewalling pairs accessed: 1490936
318: stonewalling pairs accessed: 1484993
240: stonewalling pairs accessed: 1482616
324: stonewalling pairs accessed: 1490937
209: stonewalling pairs accessed: 1486557
337: stonewalling pairs accessed: 1476741
61: stonewalling pairs accessed: 1483673
315: stonewalling pairs accessed: 1484921
336: stonewalling pairs accessed: 1476735
202: stonewalling pairs accessed: 1486842
83: stonewalling pairs accessed: 1487103
338: stonewalling pairs accessed: 1476743
300: stonewalling pairs accessed: 1484663
334: stonewalling pairs accessed: 1476740
49: stonewalling pairs accessed: 1481919
37: stonewalling pairs accessed: 1490394
181: stonewalling pairs accessed: 1489104
325: stonewalling pairs accessed: 1490931
376: stonewalling pairs accessed: 1485491
212: stonewalling pairs accessed: 1488370
93: stonewalling pairs accessed: 1495649
308: stonewalling pairs accessed: 1484654
30: stonewalling pairs accessed: 1490392
364: stonewalling pairs accessed: 1484701
179: stonewalling pairs accessed: 1490217
97: stonewalling pairs accessed: 1495667
281: stonewalling pairs accessed: 1496440
335: stonewalling pairs accessed: 1476717
216: stonewalling pairs accessed: 1488363
282: stonewalling pairs accessed: 1496519
313: stonewalling pairs accessed: 1484987
118: stonewalling pairs accessed: 1493379
150: stonewalling pairs accessed: 1486277
54: stonewalling pairs accessed: 1481077
247: stonewalling pairs accessed: 1482542
355: stonewalling pairs accessed: 1483685
256: stonewalling pairs accessed: 1494601
322: stonewalling pairs accessed: 1490929
343: stonewalling pairs accessed: 1477782
168: stonewalling pairs accessed: 1490997
270: stonewalling pairs accessed: 1488644
104: stonewalling pairs accessed: 1496180
357: stonewalling pairs accessed: 1483433
25: stonewalling pairs accessed: 1475901
186: stonewalling pairs accessed: 1489071
332: stonewalling pairs accessed: 1476737
133: stonewalling pairs accessed: 1486108
228: stonewalling pairs accessed: 1484896
350: stonewalling pairs accessed: 1483422
164: stonewalling pairs accessed: 1490574
167: stonewalling pairs accessed: 1490997
65: stonewalling pairs accessed: 1483672
16: stonewalling pairs accessed: 1490201
362: stonewalling pairs accessed: 1484734
153: stonewalling pairs accessed: 1486279
48: stonewalling pairs accessed: 1481807
40: stonewalling pairs accessed: 1481808
254: stonewalling pairs accessed: 1494605
75: stonewalling pairs accessed: 1497958
46: stonewalling pairs accessed: 1481919
22: stonewalling pairs accessed: 1475887
183: stonewalling pairs accessed: 1489035
248: stonewalling pairs accessed: 1482758
60: stonewalling pairs accessed: 1483482
294: stonewalling pairs accessed: 1475397
105: stonewalling pairs accessed: 1496635
52: stonewalling pairs accessed: 1480649
253: stonewalling pairs accessed: 1494880
174: stonewalling pairs accessed: 1489677
135: stonewalling pairs accessed: 1485340
102: stonewalling pairs accessed: 1496185
273: stonewalling pairs accessed: 1488644
155: stonewalling pairs accessed: 1486277
122: stonewalling pairs accessed: 1489036
293: stonewalling pairs accessed: 1475433
185: stonewalling pairs accessed: 1489100
172: stonewalling pairs accessed: 1489639
353: stonewalling pairs accessed: 1483428
195: stonewalling pairs accessed: 1487808
182: stonewalling pairs accessed: 1489078
23: stonewalling pairs accessed: 1475885
345: stonewalling pairs accessed: 1477786
192: stonewalling pairs accessed: 1487809
123: stonewalling pairs accessed: 1489376
365: stonewalling pairs accessed: 1484698
252: stonewalling pairs accessed: 1494889
233: stonewalling pairs accessed: 1485445
45: stonewalling pairs accessed: 1481807
272: stonewalling pairs accessed: 1488646
173: stonewalling pairs accessed: 1490208
175: stonewalling pairs accessed: 1490217
352: stonewalling pairs accessed: 1483579
213: stonewalling pairs accessed: 1488457
372: stonewalling pairs accessed: 1485496
303: stonewalling pairs accessed: 1484750
162: stonewalling pairs accessed: 1490568
142: stonewalling pairs accessed: 1495621
140: stonewalling pairs accessed: 1495536
146: stonewalling pairs accessed: 1495621
144: stonewalling pairs accessed: 1495621
149: stonewalling pairs accessed: 1495536
166: stonewalling pairs accessed: 1490568
143: stonewalling pairs accessed: 1495536
360: stonewalling pairs accessed: 1484228
145: stonewalling pairs accessed: 1495621
62: stonewalling pairs accessed: 1483287
119: stonewalling pairs accessed: 1493809
147: stonewalling pairs accessed: 1495621
5: stonewalling pairs accessed: 1497116
368: stonewalling pairs accessed: 1484228
47: stonewalling pairs accessed: 1482293
148: stonewalling pairs accessed: 1495621
7: stonewalling pairs accessed: 1497116
266: stonewalling pairs accessed: 1481230
370: stonewalling pairs accessed: 1485170
78: stonewalling pairs accessed: 1497700
41: stonewalling pairs accessed: 1482293
86: stonewalling pairs accessed: 1487049
44: stonewalling pairs accessed: 1482293
230: stonewalling pairs accessed: 1485393
141: stonewalling pairs accessed: 1495621
74: stonewalling pairs accessed: 1497700
239: stonewalling pairs accessed: 1485555
201: stonewalling pairs accessed: 1486889
73: stonewalling pairs accessed: 1497700
200: stonewalling pairs accessed: 1486889
379: stonewalling pairs accessed: 1485170
263: stonewalling pairs accessed: 1481230
261: stonewalling pairs accessed: 1481230
205: stonewalling pairs accessed: 1486889
232: stonewalling pairs accessed: 1485393
236: stonewalling pairs accessed: 1485393
231: stonewalling pairs accessed: 1485393
268: stonewalling pairs accessed: 1481230
120: stonewalling pairs accessed: 1489389
301: stonewalling pairs accessed: 1484694
265: stonewalling pairs accessed: 1481230
208: stonewalling pairs accessed: 1486889
264: stonewalling pairs accessed: 1481230
42: stonewalling pairs accessed: 1482293
267: stonewalling pairs accessed: 1481230
262: stonewalling pairs accessed: 1481230
0: stonewalling pairs accessed: 1497116
235: stonewalling pairs accessed: 1485555
243: stonewalling pairs accessed: 1482622
234: stonewalling pairs accessed: 1485555
101: stonewalling pairs accessed: 1496749
227: stonewalling pairs accessed: 1484864
4: stonewalling pairs accessed: 1497116
29: stonewalling pairs accessed: 1475870
221: stonewalling pairs accessed: 1484864
68: stonewalling pairs accessed: 1483770
3: stonewalling pairs accessed: 1497116
222: stonewalling pairs accessed: 1484864
314: stonewalling pairs accessed: 1484953
375: stonewalling pairs accessed: 1485207
229: stonewalling pairs accessed: 1484864
312: stonewalling pairs accessed: 1484953
2: stonewalling pairs accessed: 1497116
92: stonewalling pairs accessed: 1495602
94: stonewalling pairs accessed: 1495602
99: stonewalling pairs accessed: 1495602
226: stonewalling pairs accessed: 1484864
98: stonewalling pairs accessed: 1495602
129: stonewalling pairs accessed: 1488890
96: stonewalling pairs accessed: 1495602
55: stonewalling pairs accessed: 1480675
stonewalling pairs accessed min: 1475396 max: 1497959 -- min data: 64.6 GiB mean data: 65.1 GiB time: 300.0s
WARNING: Expected aggregate file size       = 28580864000000.
WARNING: Stat() of aggregate file size      = 26758101535360.
WARNING: Using actual aggregate bytes moved = 26758101535360.
WARNING: maybe caused by deadlineForStonewalling
write     82858      1876963    300.20      45.91      45.91      4.71       303.27     0.000891   307.98     0   
Max Write: 82857.75 MiB/sec (86882.65 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       82857.75   82857.75   82857.75       0.00 1848252.40 1848252.40 1848252.40       0.00  307.97981     300.00      84457.03     0    380  38    1   0     1        1         0    0 1600000    47008    47008 25518514.0 DFS      0
Finished            : Fri Jul 10 10:44:04 2020
[0] Disconnecting from DAOS POOL

[0] Finalizing DAOS..

mdtest_easy_delete
-- started at 07/10/2020 11:13:37 --

mdtest-3.3.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-r' '-F' '-P' '-d' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_easy' '-n' '4000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d' '--dfs.cont=b0599c58-2822-45b2-9e09-23a335940082' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-x' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_easy-stonewall' '-N' '1'
Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
380 tasks, 1520000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    1916960.725    1916957.488    1916959.529          0.887
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.065          0.065          0.065          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        529.063        529.062        529.063          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         15.409         15.409         15.409          0.000
-- finished at 07/10/2020 11:22:42 --

mdtest_easy_stat
-- started at 07/10/2020 10:59:58 --

mdtest-3.3.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-T' '-F' '-P' '-d' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_easy' '-n' '4000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d' '--dfs.cont=b0599c58-2822-45b2-9e09-23a335940082' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-x' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_easy-stonewall' '-N' '1'
Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
380 tasks, 1520000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    3403627.377    3403622.753    3403625.410          1.451
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        297.974        297.974        297.974          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/10/2020 11:04:56 --

mdtest_easy_write
-- started at 07/10/2020 10:33:40 --

mdtest-3.3.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-Y' '-C' '-F' '-P' '-d' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_easy' '-n' '4000000' '-u' '-L' '-a' 'DFS' '--dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d' '--dfs.cont=b0599c58-2822-45b2-9e09-23a335940082' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-x' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
380 tasks, 1520000000 files
Continue stonewall hit min: 2566688 max: 2668925 avg: 2651717.0 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    3315180.732    3315173.989    3315176.649          1.870
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    3385547.056             NA
   Tree creation             :          0.424          0.424          0.424          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        305.924        305.923        305.924          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        297.634             NA
   Tree creation             :          2.361          2.361          2.361          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/10/2020 10:38:49 --

mdtest_hard_delete
-- started at 07/10/2020 11:25:19 --

mdtest-3.3.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_hard' '-n' '1500000' '-x' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d' '--dfs.cont=b0599c58-2822-45b2-9e09-23a335940082' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-N' '1'
Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
380 tasks, 570000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    1822727.477    1822724.231    1822725.670          0.906
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.002          0.002          0.002          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        123.154        123.154        123.154          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :        563.164        563.164        563.164          0.000
-- finished at 07/10/2020 11:36:46 --

mdtest_hard_read
-- started at 07/10/2020 11:22:49 --

mdtest-3.3.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_hard' '-n' '1500000' '-x' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d' '--dfs.cont=b0599c58-2822-45b2-9e09-23a335940082' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-N' '1'
Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
380 tasks, 570000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :    1575541.025    1575538.174    1575539.997          0.838
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        142.476        142.476        142.476          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/10/2020 11:25:12 --

mdtest_hard_stat
-- started at 07/10/2020 11:12:13 --

mdtest-3.3.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_hard' '-n' '1500000' '-x' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d' '--dfs.cont=b0599c58-2822-45b2-9e09-23a335940082' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-N' '1'
Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
380 tasks, 570000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    2900139.139    2900131.662    2900135.956          2.178
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         77.402         77.402         77.402          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/10/2020 11:13:30 --

mdtest_hard_write
-- started at 07/10/2020 10:44:12 --

mdtest-3.3.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: /home/kalfizah/io500-new/bin/mdtest '-Y' '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_hard' '-n' '1500000' '-x' '/tmp/dfuse/datafiles/2020.07.10-10.28.13-scr/mdt_hard-stonewall' '-a' 'DFS' '--dfs.pool=a04776a5-bd54-4bf9-ae46-c989b9407c8d' '--dfs.cont=b0599c58-2822-45b2-9e09-23a335940082' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse/' '-N' '1' '-W' '300'
Nodemap: 10000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2166 Shifting ranks by 1 for each phase.
380 tasks, 570000000 files
Continue stonewall hit min: 562947 max: 590728 avg: 573134.3 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     725915.658     725914.739     725915.239          0.343
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     726003.091             NA
   Tree creation             :         73.348         73.348         73.348          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        309.233        309.232        309.233          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.986             NA
   Tree creation             :          0.014          0.014          0.014          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/10/2020 10:49:21 --

result_summary
[RESULT] BW   phase 1            ior_easy_write              133.280 GiB/s : time 303.08 seconds
[RESULT] IOPS phase 1         mdtest_easy_write             3315.180 kiops : time 305.92 seconds
[RESULT] BW   phase 2            ior_hard_write               80.916 GiB/s : time 303.27 seconds
[RESULT] IOPS phase 2         mdtest_hard_write              725.916 kiops : time 309.23 seconds
[RESULT] IOPS phase 3                      find             2177.510 kiops : time 311.02 seconds
[RESULT] BW   phase 3             ior_easy_read              131.949 GiB/s : time 308.41 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             3403.630 kiops : time 297.97 seconds
[RESULT] BW   phase 4             ior_hard_read               59.186 GiB/s : time 418.44 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat             2900.140 kiops : time  77.40 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete             1916.960 kiops : time 529.06 seconds
[RESULT] IOPS phase 7          mdtest_hard_read             1575.540 kiops : time 142.48 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete             1822.730 kiops : time 693.87 seconds
[SCORE] Bandwidth 95.7981 GiB/s : IOPS 2026.8 kiops : TOTAL 440.64