bracewell

Institution CSIRO
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.7
Client Kernel Version 3.10

DATA SERVER

Storage Type NVMe
Volatile Memory 192
Storage Interface NVMe
Network IB-EDR
Software Version 2.12.3
OS Version 7.7

INFORMATION

Client Nodes 10
Client Total Procs 400
Metadata Nodes 8
Metadata Storage Devices 4
Data Nodes 8
Data Storage Devices 20

METADATA

Easy Write 206.33 kIOP/s
Easy Stat 485.45 kIOP/s
Easy Delete 186.40 kIOP/s
Hard Write 150.71 kIOP/s
Hard Read 208.81 kIOP/s
Hard Stat 378.14 kIOP/s
Hard Delete 124.15 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

hostfile="host.io500"
host_cnt=`cat $hostfile | wc -l`
thread_cnt=`tail -n 1 $hostfile | sed -e "s/.*=//"`

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  #io500_mpirun="mpiexec"
  io500_mpirun="/scratch2/tools/bin/mpirun"
  #io500_mpiargs="-np 2"
  io500_mpiargs="--hostfile ${hostfile} --allow-run-as-root --mca orte_base_help_aggregate 0 singularity.sh -B /usr/lib64 -B /sys/class/infiniband_verbs -B /bin -B /sbin /scratch/centos8.sif"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}

  mkdir -p $workdir-scr/ior_easy
  mkdir -p $workdir-app/ior-easy
  mkdir -p $workdir-scr/ior_hard
  mkdir -p $workdir-app/ior-hard
  ../pinfiles_grl.sh $((host_cnt * thread_cnt)) $workdir-scr/ior_easy
  ../pinfiles_grl.sh $((host_cnt * thread_cnt)) $workdir-app/ior-easy
  #mkdir $io500_mdt_easy $workdir-scr/mdt_hard
  #mkdir $io500_mdt_easy $workdir-scr/mdt_easy
  #mkdir $io500_mdt_easy $workdir-app/mdtest-hard
  #mkdir $io500_mdt_easy $workdir-app/mdtest-easy

# lustre extras

  ## Use DNE2 striped directories for mdt_easy
# root can do the next 2 but not a user
  lfs setdirstripe -c -1 $workdir-scr/mdt_easy
  lfs setdirstripe -D -c -1 $workdir-scr/mdt_easy
  lfs setstripe -L mdt -E 1M "$workdir-scr/mdt_easy"
  lfs setdirstripe -c -1 $workdir-app/mdtest-easy
  lfs setdirstripe -D -c -1 $workdir-app/mdtest-easy
  lfs setstripe -L mdt -E 1M "$workdir-app/mdtest-easy"

  # 2 stripes, 16M stripe-size
  #lfs setstripe -p lustrefs.bss017 $io500_ior_easy

  # Use Lustre OST overstriping
  #lfs setstripe -c 48 -o 2,14,26,38,50,62,74,86,98,110,122,134,146,158,170,182,4,16,28,40,52,64,76,88,100,112,124,136,148,160,172,184,6,18,30,42,54,66,78,90,102,114,126,138,150,162,174,186 -S 16M $io500_ior_hard
  #lfs setstripe -c 32 -o 2,14,26,38,50,62,74,86,98,110,122,134,146,158,170,182,4,16,28,40,52,64,76,88,100,112,124,136,148,160,172,184 -S 16M $io500_ior_hard
  #lfs setstripe -c 8 -o 2,26,50,74,98,122,146,170 -S 16M $io500_ior_hard
  lfs setstripe -C 576 -S 16M $workdir-scr/ior_hard
  lfs setstripe -C 576 -S 16M $workdir-app/ior-hard

  # Use DNE2 striped directory for mdt_hard
  lfs setdirstripe -c -1 $workdir-scr/mdt_hard
  lfs setdirstripe -D -c -1 $workdir-scr/mdt_hard
  lfs setstripe -L mdt -E 1M "$workdir-scr/mdt_hard"
  lfs setdirstripe -c -1 $workdir-app/mdtest-hard
  lfs setdirstripe -D -c -1 $workdir-app/mdtest-hard
  lfs setstripe -L mdt -E 1M "$workdir-app/mdtest-hard"

}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jul 14 15:55:53 2020
Command line        : /scratch2/leh015/io500-app/bin/ior -r -R -a POSIX -t 16m --posix.odirect -v -b 130000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datafiles/2020.07.14-13.32.08-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datafiles/2020.07.14-13.32.08-scr/ior_easy/stonewall
Machine             : Linux bss022
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jul 14 15:55:53 2020
Path                : /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr/ior_easy
FS                  : 457.2 TiB   Used FS: 11.6%   Inodes: 3984.8 Mi   Used Inodes: 7.1%
Participating tasks: 400
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datafiles/2020.07.14-13.32.08-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 16 MiB
blocksize           : 126.95 GiB
aggregate filesize  : 49.59 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Tue Jul 14 15:55:53 2020

read      136828     8554       0.046748    133120000  16384      0.093511   379.93     0.085107   380.04     0   
Max Read:  136827.89 MiB/sec (143474.45 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       136827.89  136827.89  136827.89       0.00    8551.74    8551.74    8551.74       0.00  380.03947         NA            NA     0    400  40    1   1     1        1         0    0      1 136314880000 16777216 52000000.0 POSIX      0
Finished            : Tue Jul 14 16:02:13 2020
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jul 14 15:09:04 2020
Command line        : /scratch2/leh015/io500-app/bin/ior -w -a POSIX -t 16m --posix.odirect -v -b 130000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datafiles/2020.07.14-13.32.08-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datafiles/2020.07.14-13.32.08-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux bss022
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jul 14 15:09:04 2020
Path                : /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr/ior_easy
FS                  : 457.2 TiB   Used FS: 10.6%   Inodes: 3779.5 Mi   Used Inodes: 2.0%
Participating tasks: 400
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datafiles/2020.07.14-13.32.08-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 16 MiB
blocksize           : 126.95 GiB
aggregate filesize  : 49.59 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Tue Jul 14 15:09:04 2020
211: stonewalling pairs accessed: 8125
201: stonewalling pairs accessed: 8125
210: stonewalling pairs accessed: 8125
222: stonewalling pairs accessed: 8125
215: stonewalling pairs accessed: 8125
207: stonewalling pairs accessed: 8125
229: stonewalling pairs accessed: 8125
228: stonewalling pairs accessed: 8125
129: stonewalling pairs accessed: 8125
155: stonewalling pairs accessed: 8125
219: stonewalling pairs accessed: 8125
362: stonewalling pairs accessed: 8125
301: stonewalling pairs accessed: 8125
203: stonewalling pairs accessed: 8125
214: stonewalling pairs accessed: 8125
379: stonewalling pairs accessed: 8125
221: stonewalling pairs accessed: 8125
84: stonewalling pairs accessed: 8125
344: stonewalling pairs accessed: 8125
226: stonewalling pairs accessed: 8125
239: stonewalling pairs accessed: 8125
98: stonewalling pairs accessed: 8125
213: stonewalling pairs accessed: 8125
234: stonewalling pairs accessed: 8125
195: stonewalling pairs accessed: 8125
218: stonewalling pairs accessed: 8125
220: stonewalling pairs accessed: 8125
162: stonewalling pairs accessed: 8125
212: stonewalling pairs accessed: 8125
367: stonewalling pairs accessed: 8125
103: stonewalling pairs accessed: 8125
161: stonewalling pairs accessed: 8125
372: stonewalling pairs accessed: 8125
124: stonewalling pairs accessed: 8125
308: stonewalling pairs accessed: 8125
292: stonewalling pairs accessed: 8125
94: stonewalling pairs accessed: 8125
281: stonewalling pairs accessed: 8125
223: stonewalling pairs accessed: 8125
216: stonewalling pairs accessed: 8125
111: stonewalling pairs accessed: 8125
141: stonewalling pairs accessed: 8125
205: stonewalling pairs accessed: 8125
236: stonewalling pairs accessed: 8125
342: stonewalling pairs accessed: 8125
164: stonewalling pairs accessed: 8125
291: stonewalling pairs accessed: 8125
179: stonewalling pairs accessed: 8125
121: stonewalling pairs accessed: 8125
166: stonewalling pairs accessed: 8125
356: stonewalling pairs accessed: 8125
122: stonewalling pairs accessed: 8125
392: stonewalling pairs accessed: 8125
360: stonewalling pairs accessed: 8125
199: stonewalling pairs accessed: 8125
290: stonewalling pairs accessed: 8125
137: stonewalling pairs accessed: 8125
285: stonewalling pairs accessed: 8125
147: stonewalling pairs accessed: 8125
358: stonewalling pairs accessed: 8125
387: stonewalling pairs accessed: 8125
192: stonewalling pairs accessed: 8125
117: stonewalling pairs accessed: 8125
232: stonewalling pairs accessed: 8125
159: stonewalling pairs accessed: 8125
138: stonewalling pairs accessed: 8125
177: stonewalling pairs accessed: 8125
293: stonewalling pairs accessed: 8125
283: stonewalling pairs accessed: 8125
182: stonewalling pairs accessed: 8125
163: stonewalling pairs accessed: 8125
17: stonewalling pairs accessed: 8125
386: stonewalling pairs accessed: 8125
224: stonewalling pairs accessed: 8125
225: stonewalling pairs accessed: 8125
180: stonewalling pairs accessed: 8125
244: stonewalling pairs accessed: 8125
183: stonewalling pairs accessed: 8125
299: stonewalling pairs accessed: 8125
289: stonewalling pairs accessed: 8125
366: stonewalling pairs accessed: 8125
193: stonewalling pairs accessed: 8125
307: stonewalling pairs accessed: 8125
186: stonewalling pairs accessed: 8125
151: stonewalling pairs accessed: 8125
135: stonewalling pairs accessed: 8125
300: stonewalling pairs accessed: 8125
140: stonewalling pairs accessed: 8125
303: stonewalling pairs accessed: 8125
305: stonewalling pairs accessed: 8125
8: stonewalling pairs accessed: 8125
100: stonewalling pairs accessed: 8125
373: stonewalling pairs accessed: 8125
50: stonewalling pairs accessed: 8125
316: stonewalling pairs accessed: 8125
310: stonewalling pairs accessed: 8125
313: stonewalling pairs accessed: 8125
127: stonewalling pairs accessed: 8125
376: stonewalling pairs accessed: 8125
158: stonewalling pairs accessed: 8125
19: stonewalling pairs accessed: 8125
286: stonewalling pairs accessed: 8125
206: stonewalling pairs accessed: 8125
370: stonewalling pairs accessed: 8125
287: stonewalling pairs accessed: 8125
3: stonewalling pairs accessed: 8125
97: stonewalling pairs accessed: 8125
208: stonewalling pairs accessed: 8125
115: stonewalling pairs accessed: 8125
104: stonewalling pairs accessed: 8125
378: stonewalling pairs accessed: 8125
384: stonewalling pairs accessed: 8125
123: stonewalling pairs accessed: 8125
154: stonewalling pairs accessed: 8125
82: stonewalling pairs accessed: 8125
198: stonewalling pairs accessed: 8125
160: stonewalling pairs accessed: 8125
238: stonewalling pairs accessed: 8125
297: stonewalling pairs accessed: 8125
359: stonewalling pairs accessed: 8125
348: stonewalling pairs accessed: 8125
150: stonewalling pairs accessed: 8125
125: stonewalling pairs accessed: 8125
145: stonewalling pairs accessed: 8125
322: stonewalling pairs accessed: 8125
383: stonewalling pairs accessed: 8125
374: stonewalling pairs accessed: 8125
92: stonewalling pairs accessed: 8125
237: stonewalling pairs accessed: 8125
230: stonewalling pairs accessed: 8125
185: stonewalling pairs accessed: 8125
350: stonewalling pairs accessed: 8125
119: stonewalling pairs accessed: 8125
327: stonewalling pairs accessed: 8125
397: stonewalling pairs accessed: 8125
156: stonewalling pairs accessed: 8125
328: stonewalling pairs accessed: 8125
391: stonewalling pairs accessed: 8125
189: stonewalling pairs accessed: 8125
96: stonewalling pairs accessed: 8125
187: stonewalling pairs accessed: 8125
339: stonewalling pairs accessed: 8125
394: stonewalling pairs accessed: 8125
349: stonewalling pairs accessed: 8125
361: stonewalling pairs accessed: 8125
12: stonewalling pairs accessed: 8125
320: stonewalling pairs accessed: 8125
396: stonewalling pairs accessed: 8125
18: stonewalling pairs accessed: 8125
142: stonewalling pairs accessed: 8125
107: stonewalling pairs accessed: 8125
101: stonewalling pairs accessed: 8125
83: stonewalling pairs accessed: 8125
246: stonewalling pairs accessed: 8125
81: stonewalling pairs accessed: 8125
165: stonewalling pairs accessed: 8125
227: stonewalling pairs accessed: 8125
91: stonewalling pairs accessed: 8125
87: stonewalling pairs accessed: 8125
265: stonewalling pairs accessed: 8125
133: stonewalling pairs accessed: 8125
46: stonewalling pairs accessed: 8125
245: stonewalling pairs accessed: 8125
312: stonewalling pairs accessed: 8125
99: stonewalling pairs accessed: 8125
267: stonewalling pairs accessed: 8125
262: stonewalling pairs accessed: 8125
202: stonewalling pairs accessed: 8125
217: stonewalling pairs accessed: 8125
13: stonewalling pairs accessed: 8125
190: stonewalling pairs accessed: 8125
85: stonewalling pairs accessed: 8125
171: stonewalling pairs accessed: 8125
64: stonewalling pairs accessed: 8125
45: stonewalling pairs accessed: 8125
35: stonewalling pairs accessed: 8125
294: stonewalling pairs accessed: 8125
242: stonewalling pairs accessed: 8125
63: stonewalling pairs accessed: 8125
393: stonewalling pairs accessed: 8125
157: stonewalling pairs accessed: 8125
28: stonewalling pairs accessed: 8125
345: stonewalling pairs accessed: 8125
343: stonewalling pairs accessed: 8125
200: stonewalling pairs accessed: 8125
76: stonewalling pairs accessed: 8125
279: stonewalling pairs accessed: 8125
134: stonewalling pairs accessed: 8125
259: stonewalling pairs accessed: 8125
148: stonewalling pairs accessed: 8125
352: stonewalling pairs accessed: 8125
336: stonewalling pairs accessed: 8125
25: stonewalling pairs accessed: 8125
204: stonewalling pairs accessed: 8125
304: stonewalling pairs accessed: 8125
233: stonewalling pairs accessed: 8125
20: stonewalling pairs accessed: 8125
209: stonewalling pairs accessed: 8125
268: stonewalling pairs accessed: 8125
31: stonewalling pairs accessed: 8125
380: stonewalling pairs accessed: 8125
24: stonewalling pairs accessed: 8125
70: stonewalling pairs accessed: 8125
175: stonewalling pairs accessed: 8125
146: stonewalling pairs accessed: 8125
260: stonewalling pairs accessed: 8125
80: stonewalling pairs accessed: 8125
321: stonewalling pairs accessed: 8125
235: stonewalling pairs accessed: 8125
231: stonewalling pairs accessed: 8125
269: stonewalling pairs accessed: 8125
167: stonewalling pairs accessed: 8125
311: stonewalling pairs accessed: 8125
389: stonewalling pairs accessed: 8125
7: stonewalling pairs accessed: 8125
355: stonewalling pairs accessed: 8125
257: stonewalling pairs accessed: 8125
251: stonewalling pairs accessed: 8125
102: stonewalling pairs accessed: 8125
191: stonewalling pairs accessed: 8125
188: stonewalling pairs accessed: 8125
172: stonewalling pairs accessed: 8125
282: stonewalling pairs accessed: 8125
337: stonewalling pairs accessed: 8125
32: stonewalling pairs accessed: 8125
21: stonewalling pairs accessed: 8125
11: stonewalling pairs accessed: 8125
319: stonewalling pairs accessed: 7810
276: stonewalling pairs accessed: 5957
388: stonewalling pairs accessed: 7831
39: stonewalling pairs accessed: 8024
280: stonewalling pairs accessed: 5710
6: stonewalling pairs accessed: 7863
14: stonewalling pairs accessed: 6629
15: stonewalling pairs accessed: 6558
22: stonewalling pairs accessed: 5239
36: stonewalling pairs accessed: 7340
95: stonewalling pairs accessed: 8091
390: stonewalling pairs accessed: 7266
275: stonewalling pairs accessed: 7923
295: stonewalling pairs accessed: 7344
399: stonewalling pairs accessed: 6057
248: stonewalling pairs accessed: 6325
318: stonewalling pairs accessed: 5693
375: stonewalling pairs accessed: 6392
264: stonewalling pairs accessed: 6453
288: stonewalling pairs accessed: 5755
364: stonewalling pairs accessed: 7752
272: stonewalling pairs accessed: 6339
302: stonewalling pairs accessed: 5553
334: stonewalling pairs accessed: 8110
377: stonewalling pairs accessed: 6280
273: stonewalling pairs accessed: 6143
381: stonewalling pairs accessed: 6225
247: stonewalling pairs accessed: 6241
255: stonewalling pairs accessed: 5957
250: stonewalling pairs accessed: 6466
335: stonewalling pairs accessed: 7550
30: stonewalling pairs accessed: 7885
243: stonewalling pairs accessed: 8003
5: stonewalling pairs accessed: 6618
27: stonewalling pairs accessed: 6505
353: stonewalling pairs accessed: 5225
252: stonewalling pairs accessed: 6217
306: stonewalling pairs accessed: 5968
263: stonewalling pairs accessed: 5930
382: stonewalling pairs accessed: 7431
0: stonewalling pairs accessed: 6365
256: stonewalling pairs accessed: 6083
315: stonewalling pairs accessed: 7353
270: stonewalling pairs accessed: 8080
325: stonewalling pairs accessed: 8121
371: stonewalling pairs accessed: 7246
395: stonewalling pairs accessed: 7134
271: stonewalling pairs accessed: 6260
181: stonewalling pairs accessed: 5709
332: stonewalling pairs accessed: 6028
16: stonewalling pairs accessed: 7776
340: stonewalling pairs accessed: 7280
363: stonewalling pairs accessed: 7218
196: stonewalling pairs accessed: 7090
34: stonewalling pairs accessed: 5199
240: stonewalling pairs accessed: 6047
357: stonewalling pairs accessed: 7897
4: stonewalling pairs accessed: 5122
326: stonewalling pairs accessed: 6184
338: stonewalling pairs accessed: 6269
354: stonewalling pairs accessed: 6233
341: stonewalling pairs accessed: 8014
368: stonewalling pairs accessed: 7188
2: stonewalling pairs accessed: 6711
23: stonewalling pairs accessed: 6546
398: stonewalling pairs accessed: 7284
296: stonewalling pairs accessed: 7929
298: stonewalling pairs accessed: 5635
309: stonewalling pairs accessed: 7226
284: stonewalling pairs accessed: 5774
38: stonewalling pairs accessed: 5098
333: stonewalling pairs accessed: 5105
254: stonewalling pairs accessed: 7914
278: stonewalling pairs accessed: 6329
173: stonewalling pairs accessed: 5976
249: stonewalling pairs accessed: 6095
329: stonewalling pairs accessed: 7464
168: stonewalling pairs accessed: 7253
174: stonewalling pairs accessed: 7366
176: stonewalling pairs accessed: 7005
184: stonewalling pairs accessed: 8077
369: stonewalling pairs accessed: 7440
10: stonewalling pairs accessed: 7349
266: stonewalling pairs accessed: 6100
37: stonewalling pairs accessed: 6551
40: stonewalling pairs accessed: 7791
48: stonewalling pairs accessed: 6516
49: stonewalling pairs accessed: 7417
51: stonewalling pairs accessed: 7975
52: stonewalling pairs accessed: 4974
54: stonewalling pairs accessed: 7797
274: stonewalling pairs accessed: 6317
1: stonewalling pairs accessed: 6644
126: stonewalling pairs accessed: 7227
120: stonewalling pairs accessed: 4981
128: stonewalling pairs accessed: 7093
130: stonewalling pairs accessed: 7335
131: stonewalling pairs accessed: 5692
132: stonewalling pairs accessed: 5145
136: stonewalling pairs accessed: 5170
139: stonewalling pairs accessed: 5904
143: stonewalling pairs accessed: 6216
144: stonewalling pairs accessed: 4879
152: stonewalling pairs accessed: 4971
153: stonewalling pairs accessed: 6206
26: stonewalling pairs accessed: 7986
261: stonewalling pairs accessed: 6175
253: stonewalling pairs accessed: 7687
365: stonewalling pairs accessed: 7554
88: stonewalling pairs accessed: 7679
90: stonewalling pairs accessed: 7446
108: stonewalling pairs accessed: 6536
110: stonewalling pairs accessed: 7985
112: stonewalling pairs accessed: 7206
116: stonewalling pairs accessed: 7271
118: stonewalling pairs accessed: 7370
86: stonewalling pairs accessed: 6295
89: stonewalling pairs accessed: 7970
93: stonewalling pairs accessed: 8115
105: stonewalling pairs accessed: 7318
106: stonewalling pairs accessed: 6438
109: stonewalling pairs accessed: 6294
113: stonewalling pairs accessed: 8056
114: stonewalling pairs accessed: 7963
149: stonewalling pairs accessed: 5718
170: stonewalling pairs accessed: 7404
347: stonewalling pairs accessed: 5101
178: stonewalling pairs accessed: 7420
194: stonewalling pairs accessed: 8049
351: stonewalling pairs accessed: 5063
324: stonewalling pairs accessed: 7634
330: stonewalling pairs accessed: 7635
331: stonewalling pairs accessed: 7941
346: stonewalling pairs accessed: 8108
241: stonewalling pairs accessed: 6242
9: stonewalling pairs accessed: 6565
29: stonewalling pairs accessed: 6673
33: stonewalling pairs accessed: 6600
323: stonewalling pairs accessed: 7910
197: stonewalling pairs accessed: 5692
314: stonewalling pairs accessed: 5899
317: stonewalling pairs accessed: 7714
55: stonewalling pairs accessed: 4766
56: stonewalling pairs accessed: 6266
57: stonewalling pairs accessed: 6144
59: stonewalling pairs accessed: 6201
60: stonewalling pairs accessed: 7548
62: stonewalling pairs accessed: 7365
71: stonewalling pairs accessed: 5720
72: stonewalling pairs accessed: 6414
75: stonewalling pairs accessed: 7841
79: stonewalling pairs accessed: 7913
43: stonewalling pairs accessed: 6441
44: stonewalling pairs accessed: 6207
47: stonewalling pairs accessed: 6247
53: stonewalling pairs accessed: 7748
58: stonewalling pairs accessed: 6073
69: stonewalling pairs accessed: 5940
74: stonewalling pairs accessed: 7567
42: stonewalling pairs accessed: 7340
78: stonewalling pairs accessed: 4983
41: stonewalling pairs accessed: 6083
61: stonewalling pairs accessed: 6285
65: stonewalling pairs accessed: 6409
67: stonewalling pairs accessed: 6182
68: stonewalling pairs accessed: 4892
73: stonewalling pairs accessed: 7666
77: stonewalling pairs accessed: 8027
385: stonewalling pairs accessed: 7033
258: stonewalling pairs accessed: 6018
277: stonewalling pairs accessed: 6242
66: stonewalling pairs accessed: 8056
169: stonewalling pairs accessed: 5806
stonewalling pairs accessed min: 4766 max: 8125 -- min data: 74.5 GiB mean data: 117.6 GiB time: 300.1s
write     138274     8645       0.041303    133120000  16384      0.065982   375.95     0.085531   376.06     0   
Max Write: 138274.04 MiB/sec (144990.84 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      138274.04  138274.04  138274.04       0.00    8642.13    8642.13    8642.13       0.00  376.06480     300.09     160461.44     0    400  40    1   1     1        1         0    0      1 136314880000 16777216 52000000.0 POSIX      0
Finished            : Tue Jul 14 15:15:20 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jul 14 16:07:00 2020
Command line        : /scratch2/leh015/io500-app/bin/ior -r -R -s 200000 -a POSIX -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datafiles/2020.07.14-13.32.08-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datafiles/2020.07.14-13.32.08-scr/ior_hard/stonewall
Machine             : Linux bss022
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jul 14 16:07:00 2020
Path                : /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr/ior_hard
FS                  : 457.2 TiB   Used FS: 11.6%   Inodes: 3984.8 Mi   Used Inodes: 7.1%
Participating tasks: 400
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datafiles/2020.07.14-13.32.08-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 200000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 3.42 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Tue Jul 14 16:07:00 2020

read      13781      307538     260.05      45.91      45.91      0.072556   260.13     0.071688   260.24     0   
Max Read:  13781.48 MiB/sec (14450.92 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        13781.48   13781.48   13781.48       0.00  307414.16  307414.16  307414.16       0.00  260.23525         NA            NA     0    400  40    1   0     1        1         0    0 200000    47008    47008 3586425.8 POSIX      0
Finished            : Tue Jul 14 16:11:20 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jul 14 15:30:22 2020
Command line        : /scratch2/leh015/io500-app/bin/ior -w -s 200000 -a POSIX -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datafiles/2020.07.14-13.32.08-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datafiles/2020.07.14-13.32.08-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux bss022
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Tue Jul 14 15:30:22 2020
Path                : /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr/ior_hard
FS                  : 457.2 TiB   Used FS: 10.8%   Inodes: 3909.2 Mi   Used Inodes: 5.3%
Participating tasks: 400
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datafiles/2020.07.14-13.32.08-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 200000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 3.42 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Tue Jul 14 15:30:22 2020
384: stonewalling pairs accessed: 200000
155: stonewalling pairs accessed: 200000
368: stonewalling pairs accessed: 200000
32: stonewalling pairs accessed: 200000
96: stonewalling pairs accessed: 200000
139: stonewalling pairs accessed: 200000
128: stonewalling pairs accessed: 200000
144: stonewalling pairs accessed: 200000
151: stonewalling pairs accessed: 200000
132: stonewalling pairs accessed: 200000
135: stonewalling pairs accessed: 200000
153: stonewalling pairs accessed: 200000
192: stonewalling pairs accessed: 200000
228: stonewalling pairs accessed: 200000
137: stonewalling pairs accessed: 200000
145: stonewalling pairs accessed: 200000
148: stonewalling pairs accessed: 200000
156: stonewalling pairs accessed: 200000
157: stonewalling pairs accessed: 200000
160: stonewalling pairs accessed: 200000
167: stonewalling pairs accessed: 200000
196: stonewalling pairs accessed: 200000
208: stonewalling pairs accessed: 200000
210: stonewalling pairs accessed: 200000
212: stonewalling pairs accessed: 200000
220: stonewalling pairs accessed: 200000
216: stonewalling pairs accessed: 200000
224: stonewalling pairs accessed: 200000
226: stonewalling pairs accessed: 200000
229: stonewalling pairs accessed: 200000
221: stonewalling pairs accessed: 200000
231: stonewalling pairs accessed: 200000
233: stonewalling pairs accessed: 200000
235: stonewalling pairs accessed: 200000
237: stonewalling pairs accessed: 200000
240: stonewalling pairs accessed: 200000
272: stonewalling pairs accessed: 200000
288: stonewalling pairs accessed: 200000
304: stonewalling pairs accessed: 200000
312: stonewalling pairs accessed: 200000
280: stonewalling pairs accessed: 200000
292: stonewalling pairs accessed: 200000
308: stonewalling pairs accessed: 200000
316: stonewalling pairs accessed: 200000
284: stonewalling pairs accessed: 200000
276: stonewalling pairs accessed: 200000
320: stonewalling pairs accessed: 200000
204: stonewalling pairs accessed: 200000
324: stonewalling pairs accessed: 200000
327: stonewalling pairs accessed: 200000
329: stonewalling pairs accessed: 200000
331: stonewalling pairs accessed: 200000
352: stonewalling pairs accessed: 200000
223: stonewalling pairs accessed: 200000
239: stonewalling pairs accessed: 200000
244: stonewalling pairs accessed: 200000
247: stonewalling pairs accessed: 200000
249: stonewalling pairs accessed: 200000
251: stonewalling pairs accessed: 200000
267: stonewalling pairs accessed: 200000
295: stonewalling pairs accessed: 200000
297: stonewalling pairs accessed: 200000
299: stonewalling pairs accessed: 200000
315: stonewalling pairs accessed: 200000
171: stonewalling pairs accessed: 200000
332: stonewalling pairs accessed: 200000
336: stonewalling pairs accessed: 200000
344: stonewalling pairs accessed: 200000
340: stonewalling pairs accessed: 200000
348: stonewalling pairs accessed: 200000
354: stonewalling pairs accessed: 200000
351: stonewalling pairs accessed: 200000
356: stonewalling pairs accessed: 200000
175: stonewalling pairs accessed: 200000
184: stonewalling pairs accessed: 200000
188: stonewalling pairs accessed: 200000
159: stonewalling pairs accessed: 200000
164: stonewalling pairs accessed: 200000
357: stonewalling pairs accessed: 200000
359: stonewalling pairs accessed: 200000
201: stonewalling pairs accessed: 200000
361: stonewalling pairs accessed: 200000
364: stonewalling pairs accessed: 200000
372: stonewalling pairs accessed: 200000
176: stonewalling pairs accessed: 200000
376: stonewalling pairs accessed: 200000
169: stonewalling pairs accessed: 200000
161: stonewalling pairs accessed: 200000
166: stonewalling pairs accessed: 200000
177: stonewalling pairs accessed: 200000
180: stonewalling pairs accessed: 200000
187: stonewalling pairs accessed: 200000
197: stonewalling pairs accessed: 200000
199: stonewalling pairs accessed: 200000
179: stonewalling pairs accessed: 200000
189: stonewalling pairs accessed: 200000
191: stonewalling pairs accessed: 200000
193: stonewalling pairs accessed: 200000
194: stonewalling pairs accessed: 200000
203: stonewalling pairs accessed: 200000
205: stonewalling pairs accessed: 200000
213: stonewalling pairs accessed: 200000
207: stonewalling pairs accessed: 200000
215: stonewalling pairs accessed: 200000
217: stonewalling pairs accessed: 200000
219: stonewalling pairs accessed: 200000
222: stonewalling pairs accessed: 200000
230: stonewalling pairs accessed: 200000
232: stonewalling pairs accessed: 200000
234: stonewalling pairs accessed: 200000
236: stonewalling pairs accessed: 200000
252: stonewalling pairs accessed: 200000
333: stonewalling pairs accessed: 200000
341: stonewalling pairs accessed: 200000
335: stonewalling pairs accessed: 200000
339: stonewalling pairs accessed: 200000
347: stonewalling pairs accessed: 200000
349: stonewalling pairs accessed: 200000
343: stonewalling pairs accessed: 200000
363: stonewalling pairs accessed: 200000
365: stonewalling pairs accessed: 200000
367: stonewalling pairs accessed: 200000
369: stonewalling pairs accessed: 200000
380: stonewalling pairs accessed: 200000
388: stonewalling pairs accessed: 200000
371: stonewalling pairs accessed: 200000
389: stonewalling pairs accessed: 200000
391: stonewalling pairs accessed: 200000
238: stonewalling pairs accessed: 200000
393: stonewalling pairs accessed: 200000
395: stonewalling pairs accessed: 200000
0: stonewalling pairs accessed: 200000
8: stonewalling pairs accessed: 200000
16: stonewalling pairs accessed: 200000
12: stonewalling pairs accessed: 200000
20: stonewalling pairs accessed: 200000
4: stonewalling pairs accessed: 200000
375: stonewalling pairs accessed: 200000
241: stonewalling pairs accessed: 200000
377: stonewalling pairs accessed: 200000
23: stonewalling pairs accessed: 200000
25: stonewalling pairs accessed: 200000
33: stonewalling pairs accessed: 200000
243: stonewalling pairs accessed: 200000
275: stonewalling pairs accessed: 200000
27: stonewalling pairs accessed: 200000
36: stonewalling pairs accessed: 200000
381: stonewalling pairs accessed: 200000
383: stonewalling pairs accessed: 200000
246: stonewalling pairs accessed: 200000
385: stonewalling pairs accessed: 200000
277: stonewalling pairs accessed: 200000
253: stonewalling pairs accessed: 200000
255: stonewalling pairs accessed: 200000
279: stonewalling pairs accessed: 200000
257: stonewalling pairs accessed: 200000
265: stonewalling pairs accessed: 200000
271: stonewalling pairs accessed: 200000
273: stonewalling pairs accessed: 200000
258: stonewalling pairs accessed: 200000
260: stonewalling pairs accessed: 200000
121: stonewalling pairs accessed: 200000
123: stonewalling pairs accessed: 200000
396: stonewalling pairs accessed: 200000
289: stonewalling pairs accessed: 200000
124: stonewalling pairs accessed: 200000
290: stonewalling pairs accessed: 200000
298: stonewalling pairs accessed: 200000
127: stonewalling pairs accessed: 200000
300: stonewalling pairs accessed: 200000
129: stonewalling pairs accessed: 200000
261: stonewalling pairs accessed: 200000
263: stonewalling pairs accessed: 200000
346: stonewalling pairs accessed: 200000
268: stonewalling pairs accessed: 200000
397: stonewalling pairs accessed: 200000
350: stonewalling pairs accessed: 200000
358: stonewalling pairs accessed: 200000
399: stonewalling pairs accessed: 200000
379: stonewalling pairs accessed: 200000
130: stonewalling pairs accessed: 200000
133: stonewalling pairs accessed: 200000
149: stonewalling pairs accessed: 200000
9: stonewalling pairs accessed: 200000
125: stonewalling pairs accessed: 200000
307: stonewalling pairs accessed: 190918
319: stonewalling pairs accessed: 190867
287: stonewalling pairs accessed: 182954
303: stonewalling pairs accessed: 188075
311: stonewalling pairs accessed: 188075
3: stonewalling pairs accessed: 194660
35: stonewalling pairs accessed: 198801
31: stonewalling pairs accessed: 179568
39: stonewalling pairs accessed: 195177
7: stonewalling pairs accessed: 193282
15: stonewalling pairs accessed: 195178
283: stonewalling pairs accessed: 188398
11: stonewalling pairs accessed: 198803
28: stonewalling pairs accessed: 181657
338: stonewalling pairs accessed: 189473
17: stonewalling pairs accessed: 185163
1: stonewalling pairs accessed: 199112
274: stonewalling pairs accessed: 186048
242: stonewalling pairs accessed: 193770
250: stonewalling pairs accessed: 191634
269: stonewalling pairs accessed: 188849
370: stonewalling pairs accessed: 171415
386: stonewalling pairs accessed: 199937
13: stonewalling pairs accessed: 195732
394: stonewalling pairs accessed: 199938
325: stonewalling pairs accessed: 190443
337: stonewalling pairs accessed: 175604
378: stonewalling pairs accessed: 171412
248: stonewalling pairs accessed: 192887
362: stonewalling pairs accessed: 175252
256: stonewalling pairs accessed: 190082
5: stonewalling pairs accessed: 193830
18: stonewalling pairs accessed: 184807
26: stonewalling pairs accessed: 181906
34: stonewalling pairs accessed: 173365
2: stonewalling pairs accessed: 168704
10: stonewalling pairs accessed: 170695
21: stonewalling pairs accessed: 184515
266: stonewalling pairs accessed: 191913
387: stonewalling pairs accessed: 186647
259: stonewalling pairs accessed: 189003
264: stonewalling pairs accessed: 192271
254: stonewalling pairs accessed: 190486
262: stonewalling pairs accessed: 192653
278: stonewalling pairs accessed: 188738
366: stonewalling pairs accessed: 173755
37: stonewalling pairs accessed: 172089
374: stonewalling pairs accessed: 173757
19: stonewalling pairs accessed: 173845
382: stonewalling pairs accessed: 170540
390: stonewalling pairs accessed: 173755
398: stonewalling pairs accessed: 175284
24: stonewalling pairs accessed: 182766
30: stonewalling pairs accessed: 173500
38: stonewalling pairs accessed: 179261
6: stonewalling pairs accessed: 168455
14: stonewalling pairs accessed: 185310
22: stonewalling pairs accessed: 182946
245: stonewalling pairs accessed: 193029
29: stonewalling pairs accessed: 179906
360: stonewalling pairs accessed: 175491
392: stonewalling pairs accessed: 169052
373: stonewalling pairs accessed: 170698
270: stonewalling pairs accessed: 188730
44: stonewalling pairs accessed: 198204
68: stonewalling pairs accessed: 198203
40: stonewalling pairs accessed: 198514
41: stonewalling pairs accessed: 179202
42: stonewalling pairs accessed: 198360
43: stonewalling pairs accessed: 171382
45: stonewalling pairs accessed: 197888
140: stonewalling pairs accessed: 187905
46: stonewalling pairs accessed: 171178
47: stonewalling pairs accessed: 197754
48: stonewalling pairs accessed: 198519
49: stonewalling pairs accessed: 191618
50: stonewalling pairs accessed: 198360
143: stonewalling pairs accessed: 192803
51: stonewalling pairs accessed: 190325
146: stonewalling pairs accessed: 192806
52: stonewalling pairs accessed: 198209
183: stonewalling pairs accessed: 197044
147: stonewalling pairs accessed: 185798
53: stonewalling pairs accessed: 197913
150: stonewalling pairs accessed: 199188
54: stonewalling pairs accessed: 179058
152: stonewalling pairs accessed: 185706
55: stonewalling pairs accessed: 197756
154: stonewalling pairs accessed: 185701
56: stonewalling pairs accessed: 177552
158: stonewalling pairs accessed: 199188
57: stonewalling pairs accessed: 197633
120: stonewalling pairs accessed: 187908
58: stonewalling pairs accessed: 177259
122: stonewalling pairs accessed: 187896
59: stonewalling pairs accessed: 197493
126: stonewalling pairs accessed: 187886
60: stonewalling pairs accessed: 197122
131: stonewalling pairs accessed: 187881
61: stonewalling pairs accessed: 196832
134: stonewalling pairs accessed: 187887
62: stonewalling pairs accessed: 187237
136: stonewalling pairs accessed: 187881
63: stonewalling pairs accessed: 196564
138: stonewalling pairs accessed: 187879
64: stonewalling pairs accessed: 196808
141: stonewalling pairs accessed: 187222
65: stonewalling pairs accessed: 179202
162: stonewalling pairs accessed: 199145
142: stonewalling pairs accessed: 185886
66: stonewalling pairs accessed: 196539
163: stonewalling pairs accessed: 185547
67: stonewalling pairs accessed: 179058
165: stonewalling pairs accessed: 182933
69: stonewalling pairs accessed: 178911
168: stonewalling pairs accessed: 181958
70: stonewalling pairs accessed: 178502
170: stonewalling pairs accessed: 199146
345: stonewalling pairs accessed: 175603
71: stonewalling pairs accessed: 197921
172: stonewalling pairs accessed: 199094
72: stonewalling pairs accessed: 177914
173: stonewalling pairs accessed: 198138
73: stonewalling pairs accessed: 197783
174: stonewalling pairs accessed: 197195
353: stonewalling pairs accessed: 189125
74: stonewalling pairs accessed: 177727
178: stonewalling pairs accessed: 197136
321: stonewalling pairs accessed: 190718
75: stonewalling pairs accessed: 196453
181: stonewalling pairs accessed: 189014
322: stonewalling pairs accessed: 190549
76: stonewalling pairs accessed: 177578
182: stonewalling pairs accessed: 196569
323: stonewalling pairs accessed: 182359
77: stonewalling pairs accessed: 197640
185: stonewalling pairs accessed: 196407
326: stonewalling pairs accessed: 175805
78: stonewalling pairs accessed: 189861
186: stonewalling pairs accessed: 193786
328: stonewalling pairs accessed: 175721
79: stonewalling pairs accessed: 196304
190: stonewalling pairs accessed: 193655
330: stonewalling pairs accessed: 190231
195: stonewalling pairs accessed: 192321
334: stonewalling pairs accessed: 189647
198: stonewalling pairs accessed: 196574
342: stonewalling pairs accessed: 175491
355: stonewalling pairs accessed: 188388
281: stonewalling pairs accessed: 191267
306: stonewalling pairs accessed: 187938
309: stonewalling pairs accessed: 176243
310: stonewalling pairs accessed: 187803
313: stonewalling pairs accessed: 187627
314: stonewalling pairs accessed: 176053
317: stonewalling pairs accessed: 187262
318: stonewalling pairs accessed: 182653
282: stonewalling pairs accessed: 183751
285: stonewalling pairs accessed: 183189
286: stonewalling pairs accessed: 180828
291: stonewalling pairs accessed: 178396
293: stonewalling pairs accessed: 178285
294: stonewalling pairs accessed: 178128
296: stonewalling pairs accessed: 177964
301: stonewalling pairs accessed: 188237
302: stonewalling pairs accessed: 178130
305: stonewalling pairs accessed: 176520
116: stonewalling pairs accessed: 192973
80: stonewalling pairs accessed: 197502
81: stonewalling pairs accessed: 189286
82: stonewalling pairs accessed: 189109
83: stonewalling pairs accessed: 183909
84: stonewalling pairs accessed: 197326
85: stonewalling pairs accessed: 183598
86: stonewalling pairs accessed: 177371
87: stonewalling pairs accessed: 197123
88: stonewalling pairs accessed: 176306
89: stonewalling pairs accessed: 196920
90: stonewalling pairs accessed: 176058
91: stonewalling pairs accessed: 196753
92: stonewalling pairs accessed: 175886
93: stonewalling pairs accessed: 169936
94: stonewalling pairs accessed: 175517
95: stonewalling pairs accessed: 175759
97: stonewalling pairs accessed: 196331
98: stonewalling pairs accessed: 196001
99: stonewalling pairs accessed: 169599
100: stonewalling pairs accessed: 195905
101: stonewalling pairs accessed: 174747
102: stonewalling pairs accessed: 195462
103: stonewalling pairs accessed: 195736
104: stonewalling pairs accessed: 174286
105: stonewalling pairs accessed: 196331
106: stonewalling pairs accessed: 174005
107: stonewalling pairs accessed: 196753
108: stonewalling pairs accessed: 196190
109: stonewalling pairs accessed: 195956
110: stonewalling pairs accessed: 173697
111: stonewalling pairs accessed: 196313
112: stonewalling pairs accessed: 193251
113: stonewalling pairs accessed: 196043
114: stonewalling pairs accessed: 193077
115: stonewalling pairs accessed: 188752
117: stonewalling pairs accessed: 188065
118: stonewalling pairs accessed: 192882
119: stonewalling pairs accessed: 196328
209: stonewalling pairs accessed: 195113
225: stonewalling pairs accessed: 195113
200: stonewalling pairs accessed: 196412
202: stonewalling pairs accessed: 195878
206: stonewalling pairs accessed: 195361
211: stonewalling pairs accessed: 194755
214: stonewalling pairs accessed: 194516
218: stonewalling pairs accessed: 194235
227: stonewalling pairs accessed: 194756
stonewalling pairs accessed min: 168455 max: 200000 -- min data: 7.4 GiB mean data: 8.5 GiB time: 300.1s
write     10690      238526     300.27      45.91      45.91      0.097762   335.39     0.091775   335.48     0   
Max Write: 10690.29 MiB/sec (11209.58 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       10690.29   10690.29   10690.29       0.00  238461.20  238461.20  238461.20       0.00  335.48434     300.12      11550.15     0    400  40    1   0     1        1         0    0 200000    47008    47008 3586425.8 POSIX      0
Finished            : Tue Jul 14 15:35:58 2020
mdtest_easy_delete
-- started at 07/14/2020 16:17:13 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-app/bin/mdtest '-r' '-F' '-P' '-d' './datafiles/2020.07.14-13.32.08-scr/mdt_easy' '-n' '340000' '-u' '-L' '-a' 'POSIX' '-x' './datafiles/2020.07.14-13.32.08-scr/mdt_easy-stonewall' '-N' '1'
Path: /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr
FS: 457.2 TiB   Used FS: 11.6%   Inodes: 3984.8 Mi   Used Inodes: 7.1%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 40 for each phase.
400 tasks, 136000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     186397.831     186380.185     186392.186          3.579
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.836          0.836          0.836          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        729.691        729.622        729.644          0.014
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.197          1.197          1.197          0.000
-- finished at 07/14/2020 16:29:25 --

mdtest_easy_stat
-- started at 07/14/2020 16:02:16 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-app/bin/mdtest '-T' '-F' '-P' '-d' './datafiles/2020.07.14-13.32.08-scr/mdt_easy' '-n' '340000' '-u' '-L' '-a' 'POSIX' '-x' './datafiles/2020.07.14-13.32.08-scr/mdt_easy-stonewall' '-N' '1'
Path: /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr
FS: 457.2 TiB   Used FS: 11.6%   Inodes: 3984.8 Mi   Used Inodes: 7.1%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 40 for each phase.
400 tasks, 136000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     485453.377     485364.266     485415.095         15.361
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        280.202        280.150        280.173          0.009
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/14/2020 16:06:57 --

mdtest_easy_write
-- started at 07/14/2020 15:15:23 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-app/bin/mdtest '-Y' '-C' '-F' '-P' '-d' './datafiles/2020.07.14-13.32.08-scr/mdt_easy' '-n' '340000' '-u' '-L' '-a' 'POSIX' '-x' './datafiles/2020.07.14-13.32.08-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr
FS: 457.2 TiB   Used FS: 10.8%   Inodes: 3779.5 Mi   Used Inodes: 2.0%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 40 for each phase.
400 tasks, 136000000 files
Continue stonewall hit min: 243710 max: 340000 avg: 328286.1 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     206326.691     206320.209     206323.780          1.442
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     442418.271             NA
   Tree creation             :          0.321          0.321          0.321          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        659.170        659.149        659.158          0.005
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        296.811             NA
   Tree creation             :          3.115          3.115          3.115          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/14/2020 15:30:19 --

mdtest_hard_delete
-- started at 07/14/2020 16:35:50 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-app/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './datafiles/2020.07.14-13.32.08-scr/mdt_hard' '-n' '800000' '-x' './datafiles/2020.07.14-13.32.08-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr
FS: 457.2 TiB   Used FS: 11.6%   Inodes: 3855.1 Mi   Used Inodes: 4.0%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 40 for each phase.
400 tasks, 320000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     124151.808     124144.410     124150.117          1.038
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.209          0.209          0.209          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        638.282        638.244        638.253          0.005
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          4.781          4.781          4.781          0.000
-- finished at 07/14/2020 16:46:34 --

mdtest_hard_read
-- started at 07/14/2020 16:29:27 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './datafiles/2020.07.14-13.32.08-scr/mdt_hard' '-n' '800000' '-x' './datafiles/2020.07.14-13.32.08-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr
FS: 457.2 TiB   Used FS: 11.6%   Inodes: 3855.1 Mi   Used Inodes: 4.0%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 40 for each phase.
400 tasks, 320000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     208812.482     208792.911     208807.196          2.681
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        379.511        379.475        379.485          0.005
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/14/2020 16:35:47 --

mdtest_hard_stat
-- started at 07/14/2020 16:13:40 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-app/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './datafiles/2020.07.14-13.32.08-scr/mdt_hard' '-n' '800000' '-x' './datafiles/2020.07.14-13.32.08-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr
FS: 457.2 TiB   Used FS: 11.6%   Inodes: 3984.8 Mi   Used Inodes: 7.1%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 40 for each phase.
400 tasks, 320000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     378140.568     378015.331     378117.857         16.411
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        209.619        209.550        209.562          0.009
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/14/2020 16:17:10 --

mdtest_hard_write
-- started at 07/14/2020 15:36:01 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-app/bin/mdtest '-Y' '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './datafiles/2020.07.14-13.32.08-scr/mdt_hard' '-n' '800000' '-x' './datafiles/2020.07.14-13.32.08-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /scratch2/leh015/io500-app/datafiles/2020.07.14-13.32.08-scr
FS: 457.2 TiB   Used FS: 11.6%   Inodes: 3909.2 Mi   Used Inodes: 5.3%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 40 for each phase.
400 tasks, 320000000 files
Continue stonewall hit min: 141209 max: 198098 avg: 164669.8 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     150710.483     150699.175     150706.922          1.978
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     220021.404             NA
   Tree creation             :          1.546          1.546          1.546          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        525.810        525.771        525.783          0.007
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.371             NA
   Tree creation             :          0.647          0.647          0.647          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/14/2020 15:49:06 --

result_summary
[RESULT] BW   phase 1            ior_easy_write              135.033 GiB/s : time 375.95 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              206.327 kiops : time 659.17 seconds
[RESULT] BW   phase 2            ior_hard_write               10.440 GiB/s : time 335.39 seconds
[RESULT] IOPS phase 2         mdtest_hard_write              150.710 kiops : time 525.81 seconds
[RESULT] IOPS phase 3                      find              532.460 kiops : time 404.23 seconds
[RESULT] BW   phase 3             ior_easy_read              133.621 GiB/s : time 379.93 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              485.453 kiops : time 280.20 seconds
[RESULT] BW   phase 4             ior_hard_read               13.458 GiB/s : time 260.13 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat              378.141 kiops : time 209.62 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              186.398 kiops : time 729.69 seconds
[RESULT] IOPS phase 7          mdtest_hard_read              208.812 kiops : time 379.51 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete              124.152 kiops : time 646.20 seconds
[SCORE] Bandwidth 39.9019 GiB/s : IOPS 248.81 kiops : TOTAL 99.6393