SpectrumScale on Oracle Cloud

Institution Oracle Cloud Infrastructure
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.8
Client Kernel Version 3.10.0-1127.10.1.el7.x86_64

DATA SERVER

Storage Type NVMe
Volatile Memory 768 GB
Storage Interface iSCSI
Network Ethernet
Software Version 5.0.5.0
OS Version CentOS78

INFORMATION

Client Nodes 48
Client Total Procs 768
Metadata Nodes 40
Metadata Storage Devices 10
Data Nodes 40
Data Storage Devices 10

METADATA

Easy Write 990.88 kIOP/s
Easy Stat 791.68 kIOP/s
Easy Delete 403.88 kIOP/s
Hard Write 18.47 kIOP/s
Hard Read 28.25 kIOP/s
Hard Stat 49.58 kIOP/s
Hard Delete 9.83 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="/gpfs/fs1/openmpi-4.0.4/bin/mpirun"
  io500_mpiargs="--allow-run-as-root -np 768 -npernode 16 --hostfile /gpfs/fs1/hostsfile.cn"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 17:55:16 2020
Command line        : /gpfs/fs1/io500-app-master/bin/ior -r -R -a POSIX -t 2m -v -b 45056m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datafiles/2020.07.09-15.51.08-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datafiles/2020.07.09-15.51.08-scr/ior_easy/stonewall
Machine             : Linux ss-compute-1
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Thu Jul  9 17:55:16 2020
Path                : /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr/ior_easy
FS                  : 390.6 TiB   Used FS: 9.8%   Inodes: 600.0 Mi   Used Inodes: 53.6%
Participating tasks: 768
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datafiles/2020.07.09-15.51.08-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 48
tasks               : 768
clients per node    : 16
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 44 GiB
aggregate filesize  : 33 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Thu Jul  9 17:55:16 2020

read      84851      42429      0.018101    46137344   2048.00    0.024904   407.77     0.014480   407.81     0   
Max Read:  84850.53 MiB/sec (88972.23 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        84850.53   84850.53   84850.53       0.00   42425.27   42425.27   42425.27       0.00  407.81133         NA            NA     0    768  16    1   1     1        1         0    0      1 47244640256  2097152 34603008.0 POSIX      0
Finished            : Thu Jul  9 18:02:04 2020
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 17:17:53 2020
Command line        : /gpfs/fs1/io500-app-master/bin/ior -w -a POSIX -t 2m -v -b 45056m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./datafiles/2020.07.09-15.51.08-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./datafiles/2020.07.09-15.51.08-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux ss-compute-1
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Thu Jul  9 17:17:53 2020
Path                : /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr/ior_easy
FS                  : 390.6 TiB   Used FS: 9.7%   Inodes: 600.0 Mi   Used Inodes: 0.0%
Participating tasks: 768
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datafiles/2020.07.09-15.51.08-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 48
tasks               : 768
clients per node    : 16
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 44 GiB
aggregate filesize  : 33 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Thu Jul  9 17:17:54 2020
303: stonewalling pairs accessed: 22528
299: stonewalling pairs accessed: 22528
289: stonewalling pairs accessed: 22528
298: stonewalling pairs accessed: 22528
290: stonewalling pairs accessed: 22528
40: stonewalling pairs accessed: 22528
43: stonewalling pairs accessed: 22528
39: stonewalling pairs accessed: 22528
41: stonewalling pairs accessed: 22528
44: stonewalling pairs accessed: 22528
36: stonewalling pairs accessed: 22528
35: stonewalling pairs accessed: 22528
723: stonewalling pairs accessed: 22528
726: stonewalling pairs accessed: 22528
491: stonewalling pairs accessed: 22528
486: stonewalling pairs accessed: 22528
488: stonewalling pairs accessed: 22528
489: stonewalling pairs accessed: 22528
482: stonewalling pairs accessed: 22528
480: stonewalling pairs accessed: 22528
734: stonewalling pairs accessed: 22528
359: stonewalling pairs accessed: 22528
353: stonewalling pairs accessed: 22528
361: stonewalling pairs accessed: 22528
363: stonewalling pairs accessed: 22528
364: stonewalling pairs accessed: 22528
354: stonewalling pairs accessed: 22528
352: stonewalling pairs accessed: 22528
365: stonewalling pairs accessed: 22528
360: stonewalling pairs accessed: 22528
722: stonewalling pairs accessed: 22528
735: stonewalling pairs accessed: 22528
730: stonewalling pairs accessed: 22528
732: stonewalling pairs accessed: 22528
731: stonewalling pairs accessed: 22528
720: stonewalling pairs accessed: 22528
727: stonewalling pairs accessed: 22528
531: stonewalling pairs accessed: 22528
539: stonewalling pairs accessed: 22528
529: stonewalling pairs accessed: 22528
541: stonewalling pairs accessed: 22528
534: stonewalling pairs accessed: 22528
535: stonewalling pairs accessed: 22528
530: stonewalling pairs accessed: 22528
542: stonewalling pairs accessed: 22528
538: stonewalling pairs accessed: 22528
532: stonewalling pairs accessed: 22528
99: stonewalling pairs accessed: 22528
101: stonewalling pairs accessed: 22528
102: stonewalling pairs accessed: 22528
105: stonewalling pairs accessed: 22528
109: stonewalling pairs accessed: 22528
104: stonewalling pairs accessed: 22528
110: stonewalling pairs accessed: 22528
98: stonewalling pairs accessed: 22528
627: stonewalling pairs accessed: 22528
635: stonewalling pairs accessed: 22528
629: stonewalling pairs accessed: 22528
636: stonewalling pairs accessed: 22528
631: stonewalling pairs accessed: 22528
628: stonewalling pairs accessed: 22528
624: stonewalling pairs accessed: 22528
106: stonewalling pairs accessed: 22528
103: stonewalling pairs accessed: 22528
108: stonewalling pairs accessed: 22528
96: stonewalling pairs accessed: 22528
495: stonewalling pairs accessed: 22528
222: stonewalling pairs accessed: 22528
211: stonewalling pairs accessed: 22528
215: stonewalling pairs accessed: 22528
217: stonewalling pairs accessed: 22528
223: stonewalling pairs accessed: 22528
208: stonewalling pairs accessed: 22528
210: stonewalling pairs accessed: 22528
218: stonewalling pairs accessed: 22528
209: stonewalling pairs accessed: 22528
214: stonewalling pairs accessed: 22528
640: stonewalling pairs accessed: 22528
646: stonewalling pairs accessed: 22528
220: stonewalling pairs accessed: 22528
370: stonewalling pairs accessed: 22528
213: stonewalling pairs accessed: 22528
307: stonewalling pairs accessed: 22528
315: stonewalling pairs accessed: 22528
309: stonewalling pairs accessed: 22528
308: stonewalling pairs accessed: 22528
305: stonewalling pairs accessed: 22528
314: stonewalling pairs accessed: 22528
318: stonewalling pairs accessed: 22528
423: stonewalling pairs accessed: 22528
229: stonewalling pairs accessed: 22528
227: stonewalling pairs accessed: 22528
262: stonewalling pairs accessed: 22528
233: stonewalling pairs accessed: 22528
235: stonewalling pairs accessed: 22528
236: stonewalling pairs accessed: 22528
231: stonewalling pairs accessed: 22528
675: stonewalling pairs accessed: 22528
239: stonewalling pairs accessed: 22528
678: stonewalling pairs accessed: 22528
228: stonewalling pairs accessed: 22528
232: stonewalling pairs accessed: 22528
234: stonewalling pairs accessed: 22528
226: stonewalling pairs accessed: 22528
357: stonewalling pairs accessed: 22528
224: stonewalling pairs accessed: 22528
630: stonewalling pairs accessed: 22528
268: stonewalling pairs accessed: 22528
568: stonewalling pairs accessed: 22528
111: stonewalling pairs accessed: 22528
300: stonewalling pairs accessed: 22528
537: stonewalling pairs accessed: 22528
254: stonewalling pairs accessed: 22528
243: stonewalling pairs accessed: 22528
249: stonewalling pairs accessed: 22528
246: stonewalling pairs accessed: 22528
251: stonewalling pairs accessed: 22528
255: stonewalling pairs accessed: 22528
241: stonewalling pairs accessed: 22528
244: stonewalling pairs accessed: 22528
245: stonewalling pairs accessed: 22528
247: stonewalling pairs accessed: 22528
250: stonewalling pairs accessed: 22528
252: stonewalling pairs accessed: 22528
248: stonewalling pairs accessed: 22528
242: stonewalling pairs accessed: 22528
2: stonewalling pairs accessed: 20492
10: stonewalling pairs accessed: 20401
11: stonewalling pairs accessed: 20480
100: stonewalling pairs accessed: 11427
48: stonewalling pairs accessed: 20419
114: stonewalling pairs accessed: 20735
70: stonewalling pairs accessed: 20937
336: stonewalling pairs accessed: 20280
533: stonewalling pairs accessed: 14395
82: stonewalling pairs accessed: 20349
721: stonewalling pairs accessed: 16554
49: stonewalling pairs accessed: 20441
72: stonewalling pairs accessed: 20919
337: stonewalling pairs accessed: 20433
83: stonewalling pairs accessed: 20373
724: stonewalling pairs accessed: 15839
50: stonewalling pairs accessed: 20417
79: stonewalling pairs accessed: 20962
338: stonewalling pairs accessed: 20425
91: stonewalling pairs accessed: 20406
725: stonewalling pairs accessed: 16886
51: stonewalling pairs accessed: 20487
339: stonewalling pairs accessed: 20269
94: stonewalling pairs accessed: 20429
728: stonewalling pairs accessed: 16818
12: stonewalling pairs accessed: 20390
52: stonewalling pairs accessed: 20402
340: stonewalling pairs accessed: 20197
729: stonewalling pairs accessed: 14212
53: stonewalling pairs accessed: 20441
344: stonewalling pairs accessed: 20292
369: stonewalling pairs accessed: 20507
416: stonewalling pairs accessed: 19597
465: stonewalling pairs accessed: 21543
355: stonewalling pairs accessed: 17281
128: stonewalling pairs accessed: 20267
16: stonewalling pairs accessed: 20783
161: stonewalling pairs accessed: 20807
544: stonewalling pairs accessed: 21258
625: stonewalling pairs accessed: 19958
481: stonewalling pairs accessed: 13679
54: stonewalling pairs accessed: 20485
346: stonewalling pairs accessed: 20275
371: stonewalling pairs accessed: 19827
417: stonewalling pairs accessed: 21315
466: stonewalling pairs accessed: 21501
356: stonewalling pairs accessed: 13274
129: stonewalling pairs accessed: 20309
17: stonewalling pairs accessed: 20909
166: stonewalling pairs accessed: 20853
547: stonewalling pairs accessed: 21342
632: stonewalling pairs accessed: 19975
484: stonewalling pairs accessed: 13686
55: stonewalling pairs accessed: 20473
347: stonewalling pairs accessed: 20290
372: stonewalling pairs accessed: 20479
419: stonewalling pairs accessed: 21324
467: stonewalling pairs accessed: 21547
358: stonewalling pairs accessed: 17150
130: stonewalling pairs accessed: 20243
18: stonewalling pairs accessed: 20788
168: stonewalling pairs accessed: 20756
550: stonewalling pairs accessed: 21370
633: stonewalling pairs accessed: 12963
485: stonewalling pairs accessed: 20249
56: stonewalling pairs accessed: 20423
373: stonewalling pairs accessed: 20542
420: stonewalling pairs accessed: 19934
471: stonewalling pairs accessed: 21542
362: stonewalling pairs accessed: 18620
131: stonewalling pairs accessed: 20306
20: stonewalling pairs accessed: 20824
169: stonewalling pairs accessed: 20828
552: stonewalling pairs accessed: 21243
634: stonewalling pairs accessed: 21542
487: stonewalling pairs accessed: 21372
57: stonewalling pairs accessed: 20462
374: stonewalling pairs accessed: 19994
422: stonewalling pairs accessed: 21924
472: stonewalling pairs accessed: 21526
132: stonewalling pairs accessed: 20265
23: stonewalling pairs accessed: 20881
174: stonewalling pairs accessed: 20812
554: stonewalling pairs accessed: 21246
637: stonewalling pairs accessed: 21635
490: stonewalling pairs accessed: 19700
58: stonewalling pairs accessed: 20438
375: stonewalling pairs accessed: 20384
424: stonewalling pairs accessed: 21220
473: stonewalling pairs accessed: 21553
133: stonewalling pairs accessed: 20267
24: stonewalling pairs accessed: 20833
175: stonewalling pairs accessed: 20801
555: stonewalling pairs accessed: 21352
638: stonewalling pairs accessed: 21623
492: stonewalling pairs accessed: 19618
59: stonewalling pairs accessed: 20459
376: stonewalling pairs accessed: 20471
321: stonewalling pairs accessed: 20861
425: stonewalling pairs accessed: 21396
474: stonewalling pairs accessed: 21532
134: stonewalling pairs accessed: 20344
25: stonewalling pairs accessed: 20884
556: stonewalling pairs accessed: 21223
626: stonewalling pairs accessed: 11318
493: stonewalling pairs accessed: 20392
60: stonewalling pairs accessed: 20422
377: stonewalling pairs accessed: 19851
322: stonewalling pairs accessed: 20776
427: stonewalling pairs accessed: 21355
475: stonewalling pairs accessed: 21536
135: stonewalling pairs accessed: 20282
26: stonewalling pairs accessed: 20855
558: stonewalling pairs accessed: 21272
494: stonewalling pairs accessed: 20394
61: stonewalling pairs accessed: 20482
378: stonewalling pairs accessed: 20458
323: stonewalling pairs accessed: 20904
428: stonewalling pairs accessed: 19501
477: stonewalling pairs accessed: 21566
136: stonewalling pairs accessed: 20296
29: stonewalling pairs accessed: 20906
551: stonewalling pairs accessed: 21323
62: stonewalling pairs accessed: 20410
379: stonewalling pairs accessed: 20366
324: stonewalling pairs accessed: 20811
429: stonewalling pairs accessed: 19574
478: stonewalling pairs accessed: 21594
137: stonewalling pairs accessed: 20283
30: stonewalling pairs accessed: 20815
63: stonewalling pairs accessed: 20399
380: stonewalling pairs accessed: 20361
325: stonewalling pairs accessed: 20882
431: stonewalling pairs accessed: 21356
138: stonewalling pairs accessed: 20250
382: stonewalling pairs accessed: 20490
326: stonewalling pairs accessed: 20856
139: stonewalling pairs accessed: 20301
31: stonewalling pairs accessed: 20894
383: stonewalling pairs accessed: 20481
327: stonewalling pairs accessed: 20906
140: stonewalling pairs accessed: 20291
329: stonewalling pairs accessed: 20858
141: stonewalling pairs accessed: 20287
330: stonewalling pairs accessed: 20808
142: stonewalling pairs accessed: 20226
331: stonewalling pairs accessed: 20877
143: stonewalling pairs accessed: 20287
332: stonewalling pairs accessed: 20792
333: stonewalling pairs accessed: 20902
334: stonewalling pairs accessed: 20802
335: stonewalling pairs accessed: 20866
320: stonewalling pairs accessed: 20780
432: stonewalling pairs accessed: 21723
437: stonewalling pairs accessed: 21769
439: stonewalling pairs accessed: 21808
440: stonewalling pairs accessed: 21711
441: stonewalling pairs accessed: 21707
443: stonewalling pairs accessed: 21750
28: stonewalling pairs accessed: 20822
704: stonewalling pairs accessed: 20722
348: stonewalling pairs accessed: 20456
444: stonewalling pairs accessed: 21721
705: stonewalling pairs accessed: 20802
445: stonewalling pairs accessed: 21728
707: stonewalling pairs accessed: 20794
447: stonewalling pairs accessed: 21651
708: stonewalling pairs accessed: 20739
709: stonewalling pairs accessed: 20785
710: stonewalling pairs accessed: 20799
711: stonewalling pairs accessed: 20803
712: stonewalling pairs accessed: 20773
713: stonewalling pairs accessed: 20808
714: stonewalling pairs accessed: 20769
310: stonewalling pairs accessed: 19818
311: stonewalling pairs accessed: 19861
312: stonewalling pairs accessed: 18262
715: stonewalling pairs accessed: 20807
313: stonewalling pairs accessed: 18294
716: stonewalling pairs accessed: 20722
317: stonewalling pairs accessed: 19855
717: stonewalling pairs accessed: 20771
319: stonewalling pairs accessed: 19906
718: stonewalling pairs accessed: 20630
112: stonewalling pairs accessed: 20716
304: stonewalling pairs accessed: 18295
719: stonewalling pairs accessed: 20782
118: stonewalling pairs accessed: 20779
316: stonewalling pairs accessed: 18314
706: stonewalling pairs accessed: 20779
119: stonewalling pairs accessed: 20766
127: stonewalling pairs accessed: 20737
4: stonewalling pairs accessed: 20423
5: stonewalling pairs accessed: 20411
7: stonewalling pairs accessed: 20420
13: stonewalling pairs accessed: 20395
32: stonewalling pairs accessed: 18054
0: stonewalling pairs accessed: 20391
3: stonewalling pairs accessed: 20418
6: stonewalling pairs accessed: 20380
33: stonewalling pairs accessed: 18104
69: stonewalling pairs accessed: 20970
349: stonewalling pairs accessed: 20449
274: stonewalling pairs accessed: 20926
368: stonewalling pairs accessed: 20499
434: stonewalling pairs accessed: 21679
403: stonewalling pairs accessed: 20968
421: stonewalling pairs accessed: 20054
81: stonewalling pairs accessed: 20396
513: stonewalling pairs accessed: 20872
468: stonewalling pairs accessed: 21541
451: stonewalling pairs accessed: 20303
367: stonewalling pairs accessed: 17730
386: stonewalling pairs accessed: 20928
144: stonewalling pairs accessed: 19298
176: stonewalling pairs accessed: 20557
192: stonewalling pairs accessed: 20821
22: stonewalling pairs accessed: 20869
658: stonewalling pairs accessed: 21515
164: stonewalling pairs accessed: 20746
592: stonewalling pairs accessed: 20850
306: stonewalling pairs accessed: 18258
608: stonewalling pairs accessed: 20943
745: stonewalling pairs accessed: 21959
639: stonewalling pairs accessed: 19957
576: stonewalling pairs accessed: 20821
689: stonewalling pairs accessed: 21474
753: stonewalling pairs accessed: 20805
34: stonewalling pairs accessed: 17441
71: stonewalling pairs accessed: 20939
350: stonewalling pairs accessed: 20465
275: stonewalling pairs accessed: 20987
381: stonewalling pairs accessed: 20428
435: stonewalling pairs accessed: 21722
407: stonewalling pairs accessed: 20955
84: stonewalling pairs accessed: 20362
516: stonewalling pairs accessed: 20825
470: stonewalling pairs accessed: 21548
454: stonewalling pairs accessed: 20722
389: stonewalling pairs accessed: 21053
145: stonewalling pairs accessed: 19108
178: stonewalling pairs accessed: 20539
194: stonewalling pairs accessed: 20804
19: stonewalling pairs accessed: 20881
659: stonewalling pairs accessed: 21546
165: stonewalling pairs accessed: 20799
596: stonewalling pairs accessed: 20837
545: stonewalling pairs accessed: 21324
611: stonewalling pairs accessed: 21018
748: stonewalling pairs accessed: 21993
578: stonewalling pairs accessed: 20769
690: stonewalling pairs accessed: 21397
755: stonewalling pairs accessed: 20838
37: stonewalling pairs accessed: 20560
73: stonewalling pairs accessed: 20926
351: stonewalling pairs accessed: 20463
276: stonewalling pairs accessed: 20956
446: stonewalling pairs accessed: 21711
408: stonewalling pairs accessed: 20945
88: stonewalling pairs accessed: 20359
517: stonewalling pairs accessed: 20854
476: stonewalling pairs accessed: 21563
455: stonewalling pairs accessed: 20391
390: stonewalling pairs accessed: 21041
147: stonewalling pairs accessed: 19035
179: stonewalling pairs accessed: 20538
195: stonewalling pairs accessed: 20868
21: stonewalling pairs accessed: 20871
660: stonewalling pairs accessed: 21507
171: stonewalling pairs accessed: 20839
597: stonewalling pairs accessed: 20913
548: stonewalling pairs accessed: 21236
613: stonewalling pairs accessed: 21046
749: stonewalling pairs accessed: 22029
579: stonewalling pairs accessed: 20867
691: stonewalling pairs accessed: 21513
757: stonewalling pairs accessed: 20813
38: stonewalling pairs accessed: 18126
67: stonewalling pairs accessed: 20941
341: stonewalling pairs accessed: 20463
279: stonewalling pairs accessed: 20996
411: stonewalling pairs accessed: 20993
89: stonewalling pairs accessed: 20409
518: stonewalling pairs accessed: 20880
464: stonewalling pairs accessed: 21521
456: stonewalling pairs accessed: 20443
391: stonewalling pairs accessed: 21039
148: stonewalling pairs accessed: 19268
180: stonewalling pairs accessed: 20532
196: stonewalling pairs accessed: 20826
27: stonewalling pairs accessed: 20867
662: stonewalling pairs accessed: 21578
172: stonewalling pairs accessed: 20797
603: stonewalling pairs accessed: 20915
557: stonewalling pairs accessed: 21336
614: stonewalling pairs accessed: 21012
750: stonewalling pairs accessed: 21986
580: stonewalling pairs accessed: 20813
692: stonewalling pairs accessed: 21466
758: stonewalling pairs accessed: 20824
42: stonewalling pairs accessed: 17866
77: stonewalling pairs accessed: 20913
342: stonewalling pairs accessed: 20247
280: stonewalling pairs accessed: 20953
412: stonewalling pairs accessed: 20944
90: stonewalling pairs accessed: 20356
519: stonewalling pairs accessed: 20872
458: stonewalling pairs accessed: 20246
393: stonewalling pairs accessed: 21015
151: stonewalling pairs accessed: 20065
183: stonewalling pairs accessed: 20535
197: stonewalling pairs accessed: 20894
665: stonewalling pairs accessed: 21526
173: stonewalling pairs accessed: 20822
605: stonewalling pairs accessed: 20914
546: stonewalling pairs accessed: 21310
615: stonewalling pairs accessed: 21030
751: stonewalling pairs accessed: 22034
582: stonewalling pairs accessed: 20818
694: stonewalling pairs accessed: 21480
759: stonewalling pairs accessed: 20832
45: stonewalling pairs accessed: 17517
343: stonewalling pairs accessed: 20460
281: stonewalling pairs accessed: 20999
413: stonewalling pairs accessed: 20975
93: stonewalling pairs accessed: 20391
520: stonewalling pairs accessed: 20848
459: stonewalling pairs accessed: 20403
394: stonewalling pairs accessed: 20974
152: stonewalling pairs accessed: 20192
185: stonewalling pairs accessed: 20364
199: stonewalling pairs accessed: 20882
668: stonewalling pairs accessed: 21482
606: stonewalling pairs accessed: 20919
549: stonewalling pairs accessed: 21362
616: stonewalling pairs accessed: 21000
736: stonewalling pairs accessed: 21998
583: stonewalling pairs accessed: 20884
695: stonewalling pairs accessed: 21488
760: stonewalling pairs accessed: 20774
46: stonewalling pairs accessed: 18092
345: stonewalling pairs accessed: 20387
284: stonewalling pairs accessed: 20950
415: stonewalling pairs accessed: 20960
95: stonewalling pairs accessed: 20398
521: stonewalling pairs accessed: 20871
460: stonewalling pairs accessed: 20512
398: stonewalling pairs accessed: 20958
154: stonewalling pairs accessed: 19338
187: stonewalling pairs accessed: 20531
200: stonewalling pairs accessed: 20813
671: stonewalling pairs accessed: 21549
607: stonewalling pairs accessed: 20905
553: stonewalling pairs accessed: 21332
617: stonewalling pairs accessed: 21052
737: stonewalling pairs accessed: 22039
586: stonewalling pairs accessed: 20824
696: stonewalling pairs accessed: 21497
761: stonewalling pairs accessed: 20799
47: stonewalling pairs accessed: 18119
285: stonewalling pairs accessed: 21018
400: stonewalling pairs accessed: 20917
80: stonewalling pairs accessed: 20380
525: stonewalling pairs accessed: 20905
479: stonewalling pairs accessed: 21553
461: stonewalling pairs accessed: 20117
399: stonewalling pairs accessed: 20998
155: stonewalling pairs accessed: 18799
189: stonewalling pairs accessed: 20607
201: stonewalling pairs accessed: 20889
657: stonewalling pairs accessed: 21546
594: stonewalling pairs accessed: 20858
618: stonewalling pairs accessed: 20921
739: stonewalling pairs accessed: 22046
587: stonewalling pairs accessed: 20878
697: stonewalling pairs accessed: 21517
762: stonewalling pairs accessed: 20719
287: stonewalling pairs accessed: 21014
402: stonewalling pairs accessed: 20923
86: stonewalling pairs accessed: 20373
526: stonewalling pairs accessed: 20848
462: stonewalling pairs accessed: 20173
384: stonewalling pairs accessed: 20952
156: stonewalling pairs accessed: 18692
177: stonewalling pairs accessed: 20778
202: stonewalling pairs accessed: 20793
667: stonewalling pairs accessed: 21515
600: stonewalling pairs accessed: 20825
619: stonewalling pairs accessed: 21023
740: stonewalling pairs accessed: 21978
588: stonewalling pairs accessed: 20834
698: stonewalling pairs accessed: 21488
763: stonewalling pairs accessed: 20800
272: stonewalling pairs accessed: 20942
406: stonewalling pairs accessed: 20956
87: stonewalling pairs accessed: 20398
515: stonewalling pairs accessed: 20872
463: stonewalling pairs accessed: 20512
385: stonewalling pairs accessed: 21012
157: stonewalling pairs accessed: 19955
184: stonewalling pairs accessed: 20778
203: stonewalling pairs accessed: 20877
656: stonewalling pairs accessed: 21477
601: stonewalling pairs accessed: 20932
620: stonewalling pairs accessed: 21004
741: stonewalling pairs accessed: 22036
589: stonewalling pairs accessed: 20843
699: stonewalling pairs accessed: 21510
764: stonewalling pairs accessed: 20758
277: stonewalling pairs accessed: 21030
410: stonewalling pairs accessed: 20927
92: stonewalling pairs accessed: 20359
522: stonewalling pairs accessed: 20872
448: stonewalling pairs accessed: 20513
387: stonewalling pairs accessed: 21016
158: stonewalling pairs accessed: 18961
190: stonewalling pairs accessed: 20472
204: stonewalling pairs accessed: 20794
661: stonewalling pairs accessed: 21518
593: stonewalling pairs accessed: 20907
621: stonewalling pairs accessed: 20992
747: stonewalling pairs accessed: 22034
590: stonewalling pairs accessed: 20828
700: stonewalling pairs accessed: 21466
765: stonewalling pairs accessed: 20725
278: stonewalling pairs accessed: 20955
449: stonewalling pairs accessed: 20421
395: stonewalling pairs accessed: 21036
159: stonewalling pairs accessed: 18941
191: stonewalling pairs accessed: 20552
205: stonewalling pairs accessed: 20877
666: stonewalling pairs accessed: 21503
598: stonewalling pairs accessed: 20854
623: stonewalling pairs accessed: 21032
743: stonewalling pairs accessed: 22028
591: stonewalling pairs accessed: 20856
701: stonewalling pairs accessed: 21486
767: stonewalling pairs accessed: 20792
457: stonewalling pairs accessed: 20539
397: stonewalling pairs accessed: 21018
146: stonewalling pairs accessed: 18945
206: stonewalling pairs accessed: 20835
670: stonewalling pairs accessed: 21493
599: stonewalling pairs accessed: 20921
610: stonewalling pairs accessed: 21020
584: stonewalling pairs accessed: 20875
702: stonewalling pairs accessed: 21471
754: stonewalling pairs accessed: 20763
450: stonewalling pairs accessed: 20068
149: stonewalling pairs accessed: 19189
207: stonewalling pairs accessed: 20895
602: stonewalling pairs accessed: 20893
609: stonewalling pairs accessed: 21035
577: stonewalling pairs accessed: 20852
688: stonewalling pairs accessed: 21452
766: stonewalling pairs accessed: 20737
452: stonewalling pairs accessed: 20066
153: stonewalling pairs accessed: 16907
193: stonewalling pairs accessed: 20898
595: stonewalling pairs accessed: 20919
622: stonewalling pairs accessed: 20997
585: stonewalling pairs accessed: 20893
756: stonewalling pairs accessed: 20721
453: stonewalling pairs accessed: 20343
752: stonewalling pairs accessed: 20685
1: stonewalling pairs accessed: 20442
8: stonewalling pairs accessed: 20408
15: stonewalling pairs accessed: 20478
328: stonewalling pairs accessed: 20824
121: stonewalling pairs accessed: 20753
66: stonewalling pairs accessed: 20767
273: stonewalling pairs accessed: 21015
433: stonewalling pairs accessed: 21725
404: stonewalling pairs accessed: 20878
85: stonewalling pairs accessed: 20384
663: stonewalling pairs accessed: 21525
162: stonewalling pairs accessed: 20806
122: stonewalling pairs accessed: 20712
282: stonewalling pairs accessed: 20971
442: stonewalling pairs accessed: 21717
409: stonewalling pairs accessed: 20968
116: stonewalling pairs accessed: 20712
283: stonewalling pairs accessed: 21012
117: stonewalling pairs accessed: 20755
286: stonewalling pairs accessed: 20935
124: stonewalling pairs accessed: 20703
9: stonewalling pairs accessed: 20417
14: stonewalling pairs accessed: 20401
703: stonewalling pairs accessed: 21494
120: stonewalling pairs accessed: 20694
64: stonewalling pairs accessed: 20864
436: stonewalling pairs accessed: 21694
414: stonewalling pairs accessed: 20845
523: stonewalling pairs accessed: 20888
388: stonewalling pairs accessed: 20954
160: stonewalling pairs accessed: 20772
527: stonewalling pairs accessed: 20867
392: stonewalling pairs accessed: 20985
163: stonewalling pairs accessed: 20840
170: stonewalling pairs accessed: 20763
126: stonewalling pairs accessed: 20705
604: stonewalling pairs accessed: 20863
738: stonewalling pairs accessed: 22002
612: stonewalling pairs accessed: 20946
581: stonewalling pairs accessed: 20859
524: stonewalling pairs accessed: 20793
405: stonewalling pairs accessed: 20973
167: stonewalling pairs accessed: 20787
396: stonewalling pairs accessed: 20957
115: stonewalling pairs accessed: 20783
512: stonewalling pairs accessed: 20789
68: stonewalling pairs accessed: 20899
76: stonewalling pairs accessed: 20855
198: stonewalling pairs accessed: 20878
664: stonewalling pairs accessed: 21492
560: stonewalling pairs accessed: 20340
123: stonewalling pairs accessed: 20739
125: stonewalling pairs accessed: 20742
65: stonewalling pairs accessed: 20953
74: stonewalling pairs accessed: 20875
113: stonewalling pairs accessed: 20739
514: stonewalling pairs accessed: 20824
536: stonewalling pairs accessed: 16108
733: stonewalling pairs accessed: 16632
564: stonewalling pairs accessed: 20667
75: stonewalling pairs accessed: 20920
78: stonewalling pairs accessed: 20828
496: stonewalling pairs accessed: 20977
497: stonewalling pairs accessed: 21024
498: stonewalling pairs accessed: 20976
499: stonewalling pairs accessed: 20925
500: stonewalling pairs accessed: 20981
501: stonewalling pairs accessed: 21031
502: stonewalling pairs accessed: 20610
503: stonewalling pairs accessed: 21067
504: stonewalling pairs accessed: 21020
505: stonewalling pairs accessed: 20983
506: stonewalling pairs accessed: 21020
507: stonewalling pairs accessed: 21025
508: stonewalling pairs accessed: 21004
509: stonewalling pairs accessed: 21048
511: stonewalling pairs accessed: 21030
543: stonewalling pairs accessed: 14414
186: stonewalling pairs accessed: 20789
563: stonewalling pairs accessed: 20745
188: stonewalling pairs accessed: 20748
744: stonewalling pairs accessed: 22007
571: stonewalling pairs accessed: 20803
510: stonewalling pairs accessed: 20909
540: stonewalling pairs accessed: 14863
212: stonewalling pairs accessed: 11334
219: stonewalling pairs accessed: 11848
673: stonewalling pairs accessed: 19238
677: stonewalling pairs accessed: 19269
679: stonewalling pairs accessed: 19064
680: stonewalling pairs accessed: 19116
681: stonewalling pairs accessed: 19074
683: stonewalling pairs accessed: 19117
685: stonewalling pairs accessed: 19172
687: stonewalling pairs accessed: 18498
216: stonewalling pairs accessed: 11216
569: stonewalling pairs accessed: 20743
221: stonewalling pairs accessed: 5267
107: stonewalling pairs accessed: 9701
295: stonewalling pairs accessed: 20360
97: stonewalling pairs accessed: 11339
182: stonewalling pairs accessed: 20839
238: stonewalling pairs accessed: 14030
237: stonewalling pairs accessed: 13506
230: stonewalling pairs accessed: 5714
225: stonewalling pairs accessed: 5924
561: stonewalling pairs accessed: 20797
293: stonewalling pairs accessed: 21642
294: stonewalling pairs accessed: 4705
301: stonewalling pairs accessed: 9300
291: stonewalling pairs accessed: 15552
297: stonewalling pairs accessed: 19786
288: stonewalling pairs accessed: 21209
302: stonewalling pairs accessed: 20285
292: stonewalling pairs accessed: 21015
296: stonewalling pairs accessed: 15393
641: stonewalling pairs accessed: 19456
642: stonewalling pairs accessed: 19436
643: stonewalling pairs accessed: 19502
644: stonewalling pairs accessed: 19436
645: stonewalling pairs accessed: 19484
647: stonewalling pairs accessed: 19516
648: stonewalling pairs accessed: 19352
649: stonewalling pairs accessed: 19494
650: stonewalling pairs accessed: 19406
651: stonewalling pairs accessed: 19460
652: stonewalling pairs accessed: 19432
653: stonewalling pairs accessed: 19499
654: stonewalling pairs accessed: 19331
655: stonewalling pairs accessed: 19397
256: stonewalling pairs accessed: 18786
257: stonewalling pairs accessed: 17896
258: stonewalling pairs accessed: 17530
259: stonewalling pairs accessed: 20303
260: stonewalling pairs accessed: 21529
261: stonewalling pairs accessed: 19667
263: stonewalling pairs accessed: 18312
264: stonewalling pairs accessed: 17704
265: stonewalling pairs accessed: 18126
266: stonewalling pairs accessed: 17672
267: stonewalling pairs accessed: 18871
269: stonewalling pairs accessed: 18722
270: stonewalling pairs accessed: 17947
271: stonewalling pairs accessed: 18424
240: stonewalling pairs accessed: 7239
672: stonewalling pairs accessed: 19303
674: stonewalling pairs accessed: 19123
676: stonewalling pairs accessed: 19044
682: stonewalling pairs accessed: 18967
684: stonewalling pairs accessed: 19147
686: stonewalling pairs accessed: 19293
562: stonewalling pairs accessed: 19998
565: stonewalling pairs accessed: 19123
746: stonewalling pairs accessed: 11974
573: stonewalling pairs accessed: 22316
742: stonewalling pairs accessed: 5996
559: stonewalling pairs accessed: 5713
253: stonewalling pairs accessed: 5680
693: stonewalling pairs accessed: 5622
181: stonewalling pairs accessed: 17251
483: stonewalling pairs accessed: 12055
401: stonewalling pairs accessed: 15244
469: stonewalling pairs accessed: 5541
572: stonewalling pairs accessed: 19060
438: stonewalling pairs accessed: 2447
570: stonewalling pairs accessed: 19073
574: stonewalling pairs accessed: 19072
575: stonewalling pairs accessed: 19135
566: stonewalling pairs accessed: 19104
426: stonewalling pairs accessed: 18909
528: stonewalling pairs accessed: 16320
366: stonewalling pairs accessed: 10348
418: stonewalling pairs accessed: 18857
567: stonewalling pairs accessed: 20522
430: stonewalling pairs accessed: 18876
669: stonewalling pairs accessed: 5645
150: stonewalling pairs accessed: 18627
stonewalling pairs accessed min: 2447 max: 22528 -- min data: 4.8 GiB mean data: 39.8 GiB time: 322.3s
write     93159      46662      0.015847    46137344   2048.00    0.646402   370.78     0.013704   371.44     0   
Max Write: 93159.47 MiB/sec (97684.79 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       93159.47   93159.47   93159.47       0.00   46579.74   46579.74   46579.74       0.00  371.43842     322.30      97105.22     0    768  16    1   1     1        1         0    0      1 47244640256  2097152 34603008.0 POSIX      0
Finished            : Thu Jul  9 17:24:04 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 18:09:00 2020
Command line        : /gpfs/fs1/io500-app-master/bin/ior -r -R -s 95000 -a POSIX -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datafiles/2020.07.09-15.51.08-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datafiles/2020.07.09-15.51.08-scr/ior_hard/stonewall
Machine             : Linux ss-compute-1
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Thu Jul  9 18:09:00 2020
Path                : /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr/ior_hard
FS                  : 390.6 TiB   Used FS: 9.8%   Inodes: 600.0 Mi   Used Inodes: 53.6%
Participating tasks: 768
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datafiles/2020.07.09-15.51.08-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 95000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 48
tasks               : 768
clients per node    : 16
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 3.12 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Thu Jul  9 18:09:00 2020

read      34656      774177     94.24       45.91      45.91      0.007661   94.24      0.129293   94.38      0   
Max Read:  34656.44 MiB/sec (36339.91 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        34656.44   34656.44   34656.44       0.00  773057.94  773057.94  773057.94       0.00   94.37844         NA            NA     0    768  16    1   0     1        1         0    0  95000    47008    47008 3270820.2 POSIX      0
Finished            : Thu Jul  9 18:10:35 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 17:30:05 2020
Command line        : /gpfs/fs1/io500-app-master/bin/ior -w -s 95000 -a POSIX -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./datafiles/2020.07.09-15.51.08-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./datafiles/2020.07.09-15.51.08-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux ss-compute-1
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Thu Jul  9 17:30:05 2020
Path                : /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr/ior_hard
FS                  : 390.6 TiB   Used FS: 9.0%   Inodes: 600.0 Mi   Used Inodes: 52.1%
Participating tasks: 768
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./datafiles/2020.07.09-15.51.08-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 95000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 48
tasks               : 768
clients per node    : 16
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 3.12 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Thu Jul  9 17:30:05 2020
13: stonewalling pairs accessed: 95000
1: stonewalling pairs accessed: 95000
2: stonewalling pairs accessed: 95000
3: stonewalling pairs accessed: 95000
4: stonewalling pairs accessed: 95000
5: stonewalling pairs accessed: 95000
6: stonewalling pairs accessed: 95000
7: stonewalling pairs accessed: 95000
8: stonewalling pairs accessed: 95000
9: stonewalling pairs accessed: 95000
10: stonewalling pairs accessed: 95000
11: stonewalling pairs accessed: 95000
12: stonewalling pairs accessed: 95000
0: stonewalling pairs accessed: 95000
14: stonewalling pairs accessed: 95000
15: stonewalling pairs accessed: 95000
767: stonewalling pairs accessed: 95000
752: stonewalling pairs accessed: 95000
753: stonewalling pairs accessed: 95000
754: stonewalling pairs accessed: 95000
755: stonewalling pairs accessed: 95000
756: stonewalling pairs accessed: 95000
757: stonewalling pairs accessed: 95000
758: stonewalling pairs accessed: 95000
759: stonewalling pairs accessed: 95000
760: stonewalling pairs accessed: 95000
761: stonewalling pairs accessed: 95000
762: stonewalling pairs accessed: 95000
763: stonewalling pairs accessed: 95000
764: stonewalling pairs accessed: 95000
765: stonewalling pairs accessed: 95000
766: stonewalling pairs accessed: 95000
326: stonewalling pairs accessed: 95000
320: stonewalling pairs accessed: 95000
321: stonewalling pairs accessed: 95000
322: stonewalling pairs accessed: 95000
323: stonewalling pairs accessed: 95000
324: stonewalling pairs accessed: 95000
325: stonewalling pairs accessed: 95000
327: stonewalling pairs accessed: 95000
328: stonewalling pairs accessed: 95000
329: stonewalling pairs accessed: 95000
330: stonewalling pairs accessed: 95000
331: stonewalling pairs accessed: 95000
332: stonewalling pairs accessed: 95000
333: stonewalling pairs accessed: 95000
334: stonewalling pairs accessed: 95000
335: stonewalling pairs accessed: 95000
638: stonewalling pairs accessed: 95000
628: stonewalling pairs accessed: 95000
633: stonewalling pairs accessed: 95000
635: stonewalling pairs accessed: 95000
624: stonewalling pairs accessed: 95000
625: stonewalling pairs accessed: 95000
626: stonewalling pairs accessed: 95000
627: stonewalling pairs accessed: 95000
629: stonewalling pairs accessed: 95000
630: stonewalling pairs accessed: 95000
631: stonewalling pairs accessed: 95000
632: stonewalling pairs accessed: 95000
634: stonewalling pairs accessed: 95000
636: stonewalling pairs accessed: 95000
637: stonewalling pairs accessed: 95000
639: stonewalling pairs accessed: 95000
108: stonewalling pairs accessed: 95000
96: stonewalling pairs accessed: 95000
107: stonewalling pairs accessed: 95000
104: stonewalling pairs accessed: 95000
105: stonewalling pairs accessed: 95000
109: stonewalling pairs accessed: 95000
110: stonewalling pairs accessed: 95000
111: stonewalling pairs accessed: 95000
99: stonewalling pairs accessed: 95000
100: stonewalling pairs accessed: 95000
101: stonewalling pairs accessed: 95000
102: stonewalling pairs accessed: 95000
103: stonewalling pairs accessed: 95000
106: stonewalling pairs accessed: 95000
97: stonewalling pairs accessed: 95000
98: stonewalling pairs accessed: 95000
592: stonewalling pairs accessed: 25700
593: stonewalling pairs accessed: 25702
560: stonewalling pairs accessed: 40262
594: stonewalling pairs accessed: 25696
561: stonewalling pairs accessed: 40262
595: stonewalling pairs accessed: 25706
562: stonewalling pairs accessed: 40269
596: stonewalling pairs accessed: 25696
563: stonewalling pairs accessed: 40266
597: stonewalling pairs accessed: 25696
564: stonewalling pairs accessed: 40265
598: stonewalling pairs accessed: 25696
565: stonewalling pairs accessed: 40264
599: stonewalling pairs accessed: 25698
566: stonewalling pairs accessed: 40264
600: stonewalling pairs accessed: 25700
567: stonewalling pairs accessed: 40273
601: stonewalling pairs accessed: 25704
568: stonewalling pairs accessed: 40265
569: stonewalling pairs accessed: 40282
570: stonewalling pairs accessed: 40265
602: stonewalling pairs accessed: 25698
571: stonewalling pairs accessed: 40281
603: stonewalling pairs accessed: 25696
573: stonewalling pairs accessed: 40272
604: stonewalling pairs accessed: 25709
574: stonewalling pairs accessed: 40261
605: stonewalling pairs accessed: 25702
575: stonewalling pairs accessed: 40268
606: stonewalling pairs accessed: 25702
607: stonewalling pairs accessed: 25704
672: stonewalling pairs accessed: 41857
673: stonewalling pairs accessed: 41864
674: stonewalling pairs accessed: 41880
675: stonewalling pairs accessed: 41891
48: stonewalling pairs accessed: 43006
676: stonewalling pairs accessed: 41875
52: stonewalling pairs accessed: 43005
677: stonewalling pairs accessed: 41851
55: stonewalling pairs accessed: 43004
352: stonewalling pairs accessed: 23426
678: stonewalling pairs accessed: 41858
58: stonewalling pairs accessed: 43003
353: stonewalling pairs accessed: 23430
679: stonewalling pairs accessed: 41871
59: stonewalling pairs accessed: 43005
354: stonewalling pairs accessed: 23426
680: stonewalling pairs accessed: 41924
60: stonewalling pairs accessed: 43000
355: stonewalling pairs accessed: 23431
681: stonewalling pairs accessed: 41959
63: stonewalling pairs accessed: 43002
356: stonewalling pairs accessed: 23430
358: stonewalling pairs accessed: 23428
400: stonewalling pairs accessed: 22767
682: stonewalling pairs accessed: 41921
360: stonewalling pairs accessed: 23428
401: stonewalling pairs accessed: 22760
686: stonewalling pairs accessed: 42016
365: stonewalling pairs accessed: 23429
402: stonewalling pairs accessed: 22769
403: stonewalling pairs accessed: 22760
404: stonewalling pairs accessed: 22779
405: stonewalling pairs accessed: 22778
406: stonewalling pairs accessed: 22772
407: stonewalling pairs accessed: 22772
408: stonewalling pairs accessed: 22772
409: stonewalling pairs accessed: 22776
410: stonewalling pairs accessed: 22764
411: stonewalling pairs accessed: 22770
412: stonewalling pairs accessed: 22760
413: stonewalling pairs accessed: 22772
414: stonewalling pairs accessed: 22764
415: stonewalling pairs accessed: 22760
176: stonewalling pairs accessed: 27480
720: stonewalling pairs accessed: 32873
177: stonewalling pairs accessed: 27478
721: stonewalling pairs accessed: 32880
178: stonewalling pairs accessed: 27482
722: stonewalling pairs accessed: 32874
180: stonewalling pairs accessed: 27478
723: stonewalling pairs accessed: 32866
181: stonewalling pairs accessed: 27478
724: stonewalling pairs accessed: 32869
182: stonewalling pairs accessed: 27478
725: stonewalling pairs accessed: 32876
184: stonewalling pairs accessed: 27478
726: stonewalling pairs accessed: 32874
185: stonewalling pairs accessed: 27478
727: stonewalling pairs accessed: 32873
186: stonewalling pairs accessed: 27479
728: stonewalling pairs accessed: 32874
187: stonewalling pairs accessed: 27483
729: stonewalling pairs accessed: 32887
188: stonewalling pairs accessed: 27484
730: stonewalling pairs accessed: 32866
731: stonewalling pairs accessed: 32876
732: stonewalling pairs accessed: 32886
733: stonewalling pairs accessed: 32887
734: stonewalling pairs accessed: 32895
735: stonewalling pairs accessed: 32887
129: stonewalling pairs accessed: 25447
130: stonewalling pairs accessed: 25444
131: stonewalling pairs accessed: 25445
132: stonewalling pairs accessed: 25439
133: stonewalling pairs accessed: 25438
134: stonewalling pairs accessed: 25445
135: stonewalling pairs accessed: 25441
136: stonewalling pairs accessed: 25439
256: stonewalling pairs accessed: 25275
137: stonewalling pairs accessed: 25444
257: stonewalling pairs accessed: 25264
138: stonewalling pairs accessed: 25444
258: stonewalling pairs accessed: 25255
259: stonewalling pairs accessed: 25244
139: stonewalling pairs accessed: 25445
260: stonewalling pairs accessed: 25243
140: stonewalling pairs accessed: 25443
261: stonewalling pairs accessed: 25260
141: stonewalling pairs accessed: 25443
262: stonewalling pairs accessed: 25243
142: stonewalling pairs accessed: 25444
263: stonewalling pairs accessed: 25243
143: stonewalling pairs accessed: 25446
264: stonewalling pairs accessed: 25240
265: stonewalling pairs accessed: 25256
266: stonewalling pairs accessed: 25270
267: stonewalling pairs accessed: 25254
268: stonewalling pairs accessed: 25243
269: stonewalling pairs accessed: 25265
270: stonewalling pairs accessed: 25271
271: stonewalling pairs accessed: 25255
272: stonewalling pairs accessed: 26999
368: stonewalling pairs accessed: 23604
273: stonewalling pairs accessed: 26997
369: stonewalling pairs accessed: 23604
274: stonewalling pairs accessed: 26997
370: stonewalling pairs accessed: 23606
275: stonewalling pairs accessed: 26998
371: stonewalling pairs accessed: 23610
276: stonewalling pairs accessed: 26998
372: stonewalling pairs accessed: 23610
277: stonewalling pairs accessed: 26998
373: stonewalling pairs accessed: 23615
278: stonewalling pairs accessed: 26997
374: stonewalling pairs accessed: 23611
279: stonewalling pairs accessed: 26997
375: stonewalling pairs accessed: 23604
280: stonewalling pairs accessed: 26999
376: stonewalling pairs accessed: 23604
281: stonewalling pairs accessed: 26998
377: stonewalling pairs accessed: 23603
282: stonewalling pairs accessed: 26997
283: stonewalling pairs accessed: 26999
378: stonewalling pairs accessed: 23603
284: stonewalling pairs accessed: 26997
379: stonewalling pairs accessed: 23605
285: stonewalling pairs accessed: 26997
380: stonewalling pairs accessed: 23605
286: stonewalling pairs accessed: 26997
381: stonewalling pairs accessed: 23606
287: stonewalling pairs accessed: 26998
382: stonewalling pairs accessed: 23605
384: stonewalling pairs accessed: 26638
383: stonewalling pairs accessed: 23605
385: stonewalling pairs accessed: 26638
386: stonewalling pairs accessed: 26638
387: stonewalling pairs accessed: 26639
388: stonewalling pairs accessed: 26638
389: stonewalling pairs accessed: 26638
390: stonewalling pairs accessed: 26638
391: stonewalling pairs accessed: 26637
392: stonewalling pairs accessed: 26638
393: stonewalling pairs accessed: 26638
394: stonewalling pairs accessed: 26638
395: stonewalling pairs accessed: 26638
396: stonewalling pairs accessed: 26638
397: stonewalling pairs accessed: 26637
398: stonewalling pairs accessed: 26638
399: stonewalling pairs accessed: 26638
512: stonewalling pairs accessed: 28787
513: stonewalling pairs accessed: 28787
514: stonewalling pairs accessed: 28789
515: stonewalling pairs accessed: 28789
160: stonewalling pairs accessed: 27497
516: stonewalling pairs accessed: 28788
161: stonewalling pairs accessed: 27497
517: stonewalling pairs accessed: 28789
162: stonewalling pairs accessed: 27497
518: stonewalling pairs accessed: 28789
163: stonewalling pairs accessed: 27497
519: stonewalling pairs accessed: 28788
164: stonewalling pairs accessed: 27497
520: stonewalling pairs accessed: 28788
165: stonewalling pairs accessed: 27497
521: stonewalling pairs accessed: 28789
166: stonewalling pairs accessed: 27496
167: stonewalling pairs accessed: 27497
522: stonewalling pairs accessed: 28788
168: stonewalling pairs accessed: 27497
523: stonewalling pairs accessed: 28788
169: stonewalling pairs accessed: 27497
524: stonewalling pairs accessed: 28788
525: stonewalling pairs accessed: 28788
526: stonewalling pairs accessed: 28790
170: stonewalling pairs accessed: 27499
527: stonewalling pairs accessed: 28788
171: stonewalling pairs accessed: 27495
172: stonewalling pairs accessed: 27496
173: stonewalling pairs accessed: 27496
174: stonewalling pairs accessed: 27496
175: stonewalling pairs accessed: 27497
464: stonewalling pairs accessed: 26281
465: stonewalling pairs accessed: 26280
466: stonewalling pairs accessed: 26282
467: stonewalling pairs accessed: 26283
468: stonewalling pairs accessed: 26278
469: stonewalling pairs accessed: 26281
470: stonewalling pairs accessed: 26282
471: stonewalling pairs accessed: 26281
472: stonewalling pairs accessed: 26281
473: stonewalling pairs accessed: 26281
474: stonewalling pairs accessed: 26281
475: stonewalling pairs accessed: 26281
476: stonewalling pairs accessed: 26283
477: stonewalling pairs accessed: 26281
478: stonewalling pairs accessed: 26281
479: stonewalling pairs accessed: 26279
49: stonewalling pairs accessed: 43006
50: stonewalling pairs accessed: 43006
51: stonewalling pairs accessed: 43006
53: stonewalling pairs accessed: 43006
54: stonewalling pairs accessed: 43006
56: stonewalling pairs accessed: 43006
57: stonewalling pairs accessed: 43006
61: stonewalling pairs accessed: 43006
62: stonewalling pairs accessed: 43006
683: stonewalling pairs accessed: 42017
684: stonewalling pairs accessed: 42017
685: stonewalling pairs accessed: 42026
687: stonewalling pairs accessed: 42025
572: stonewalling pairs accessed: 41357
608: stonewalling pairs accessed: 24234
432: stonewalling pairs accessed: 23754
433: stonewalling pairs accessed: 23754
434: stonewalling pairs accessed: 23754
435: stonewalling pairs accessed: 23754
436: stonewalling pairs accessed: 23754
437: stonewalling pairs accessed: 23754
438: stonewalling pairs accessed: 23754
439: stonewalling pairs accessed: 23754
440: stonewalling pairs accessed: 23755
441: stonewalling pairs accessed: 23754
442: stonewalling pairs accessed: 23754
443: stonewalling pairs accessed: 23755
444: stonewalling pairs accessed: 23754
445: stonewalling pairs accessed: 23754
446: stonewalling pairs accessed: 23754
447: stonewalling pairs accessed: 23755
288: stonewalling pairs accessed: 22866
289: stonewalling pairs accessed: 22866
290: stonewalling pairs accessed: 22866
291: stonewalling pairs accessed: 22866
292: stonewalling pairs accessed: 22866
293: stonewalling pairs accessed: 22866
294: stonewalling pairs accessed: 22866
295: stonewalling pairs accessed: 22866
296: stonewalling pairs accessed: 22866
297: stonewalling pairs accessed: 22866
298: stonewalling pairs accessed: 22866
299: stonewalling pairs accessed: 22866
300: stonewalling pairs accessed: 22866
301: stonewalling pairs accessed: 22866
302: stonewalling pairs accessed: 22866
303: stonewalling pairs accessed: 22866
192: stonewalling pairs accessed: 23225
193: stonewalling pairs accessed: 23225
194: stonewalling pairs accessed: 23225
195: stonewalling pairs accessed: 23225
196: stonewalling pairs accessed: 23225
197: stonewalling pairs accessed: 23225
198: stonewalling pairs accessed: 23225
199: stonewalling pairs accessed: 23225
200: stonewalling pairs accessed: 23225
201: stonewalling pairs accessed: 23225
202: stonewalling pairs accessed: 23225
203: stonewalling pairs accessed: 23225
204: stonewalling pairs accessed: 23225
205: stonewalling pairs accessed: 23225
206: stonewalling pairs accessed: 23225
207: stonewalling pairs accessed: 23225
547: stonewalling pairs accessed: 22135
550: stonewalling pairs accessed: 22135
551: stonewalling pairs accessed: 22135
552: stonewalling pairs accessed: 22135
553: stonewalling pairs accessed: 22135
554: stonewalling pairs accessed: 22135
555: stonewalling pairs accessed: 22135
556: stonewalling pairs accessed: 22135
557: stonewalling pairs accessed: 22135
558: stonewalling pairs accessed: 22135
559: stonewalling pairs accessed: 22135
544: stonewalling pairs accessed: 22135
545: stonewalling pairs accessed: 22135
546: stonewalling pairs accessed: 22135
548: stonewalling pairs accessed: 22135
549: stonewalling pairs accessed: 22135
189: stonewalling pairs accessed: 27485
190: stonewalling pairs accessed: 27485
191: stonewalling pairs accessed: 27485
336: stonewalling pairs accessed: 22327
337: stonewalling pairs accessed: 22326
338: stonewalling pairs accessed: 22326
339: stonewalling pairs accessed: 22326
340: stonewalling pairs accessed: 22326
341: stonewalling pairs accessed: 22327
342: stonewalling pairs accessed: 22326
343: stonewalling pairs accessed: 22327
344: stonewalling pairs accessed: 22326
345: stonewalling pairs accessed: 22327
346: stonewalling pairs accessed: 22326
347: stonewalling pairs accessed: 22326
348: stonewalling pairs accessed: 22326
349: stonewalling pairs accessed: 22326
350: stonewalling pairs accessed: 22326
351: stonewalling pairs accessed: 22327
577: stonewalling pairs accessed: 23154
578: stonewalling pairs accessed: 23154
579: stonewalling pairs accessed: 23154
580: stonewalling pairs accessed: 23154
581: stonewalling pairs accessed: 23154
582: stonewalling pairs accessed: 23154
583: stonewalling pairs accessed: 23154
584: stonewalling pairs accessed: 23154
585: stonewalling pairs accessed: 23154
586: stonewalling pairs accessed: 23154
587: stonewalling pairs accessed: 23154
588: stonewalling pairs accessed: 23154
589: stonewalling pairs accessed: 23154
590: stonewalling pairs accessed: 23154
591: stonewalling pairs accessed: 23154
576: stonewalling pairs accessed: 23154
416: stonewalling pairs accessed: 22575
417: stonewalling pairs accessed: 22575
418: stonewalling pairs accessed: 22575
419: stonewalling pairs accessed: 22576
420: stonewalling pairs accessed: 22575
421: stonewalling pairs accessed: 22576
422: stonewalling pairs accessed: 22576
423: stonewalling pairs accessed: 22576
424: stonewalling pairs accessed: 22575
425: stonewalling pairs accessed: 22575
359: stonewalling pairs accessed: 23432
361: stonewalling pairs accessed: 23432
426: stonewalling pairs accessed: 22576
362: stonewalling pairs accessed: 23432
427: stonewalling pairs accessed: 22576
363: stonewalling pairs accessed: 23432
428: stonewalling pairs accessed: 22576
429: stonewalling pairs accessed: 22576
430: stonewalling pairs accessed: 22576
431: stonewalling pairs accessed: 22576
128: stonewalling pairs accessed: 25451
144: stonewalling pairs accessed: 25903
150: stonewalling pairs accessed: 25903
151: stonewalling pairs accessed: 25903
152: stonewalling pairs accessed: 25903
153: stonewalling pairs accessed: 25903
154: stonewalling pairs accessed: 25903
155: stonewalling pairs accessed: 25903
156: stonewalling pairs accessed: 25903
157: stonewalling pairs accessed: 25903
158: stonewalling pairs accessed: 25903
159: stonewalling pairs accessed: 25903
145: stonewalling pairs accessed: 25903
146: stonewalling pairs accessed: 25903
147: stonewalling pairs accessed: 25903
148: stonewalling pairs accessed: 25903
149: stonewalling pairs accessed: 25903
480: stonewalling pairs accessed: 22879
481: stonewalling pairs accessed: 22879
482: stonewalling pairs accessed: 22879
308: stonewalling pairs accessed: 23342
483: stonewalling pairs accessed: 22879
484: stonewalling pairs accessed: 22880
485: stonewalling pairs accessed: 22879
486: stonewalling pairs accessed: 22880
315: stonewalling pairs accessed: 23342
487: stonewalling pairs accessed: 22879
316: stonewalling pairs accessed: 23342
488: stonewalling pairs accessed: 22879
317: stonewalling pairs accessed: 23342
489: stonewalling pairs accessed: 22879
318: stonewalling pairs accessed: 23342
319: stonewalling pairs accessed: 23342
305: stonewalling pairs accessed: 23342
490: stonewalling pairs accessed: 22879
306: stonewalling pairs accessed: 23342
491: stonewalling pairs accessed: 22880
307: stonewalling pairs accessed: 23342
492: stonewalling pairs accessed: 22880
309: stonewalling pairs accessed: 23342
493: stonewalling pairs accessed: 22880
311: stonewalling pairs accessed: 23342
494: stonewalling pairs accessed: 22880
704: stonewalling pairs accessed: 24731
312: stonewalling pairs accessed: 23342
495: stonewalling pairs accessed: 22880
705: stonewalling pairs accessed: 24731
240: stonewalling pairs accessed: 24515
313: stonewalling pairs accessed: 23342
706: stonewalling pairs accessed: 24731
242: stonewalling pairs accessed: 24515
314: stonewalling pairs accessed: 23342
707: stonewalling pairs accessed: 24731
246: stonewalling pairs accessed: 24515
708: stonewalling pairs accessed: 24731
247: stonewalling pairs accessed: 24515
304: stonewalling pairs accessed: 23342
709: stonewalling pairs accessed: 24731
248: stonewalling pairs accessed: 24515
310: stonewalling pairs accessed: 23342
710: stonewalling pairs accessed: 24731
249: stonewalling pairs accessed: 24515
711: stonewalling pairs accessed: 24731
250: stonewalling pairs accessed: 24515
712: stonewalling pairs accessed: 24731
251: stonewalling pairs accessed: 24515
714: stonewalling pairs accessed: 24731
252: stonewalling pairs accessed: 24515
253: stonewalling pairs accessed: 24515
715: stonewalling pairs accessed: 24731
716: stonewalling pairs accessed: 24731
254: stonewalling pairs accessed: 24515
717: stonewalling pairs accessed: 24731
255: stonewalling pairs accessed: 24515
718: stonewalling pairs accessed: 24731
241: stonewalling pairs accessed: 24515
719: stonewalling pairs accessed: 24731
243: stonewalling pairs accessed: 24515
244: stonewalling pairs accessed: 24515
245: stonewalling pairs accessed: 24515
65: stonewalling pairs accessed: 25857
67: stonewalling pairs accessed: 25857
85: stonewalling pairs accessed: 24041
68: stonewalling pairs accessed: 25857
69: stonewalling pairs accessed: 25857
70: stonewalling pairs accessed: 25857
86: stonewalling pairs accessed: 24041
87: stonewalling pairs accessed: 24041
92: stonewalling pairs accessed: 24041
94: stonewalling pairs accessed: 24041
95: stonewalling pairs accessed: 24041
82: stonewalling pairs accessed: 24041
83: stonewalling pairs accessed: 24041
84: stonewalling pairs accessed: 24041
88: stonewalling pairs accessed: 24041
89: stonewalling pairs accessed: 24041
90: stonewalling pairs accessed: 24041
91: stonewalling pairs accessed: 24041
93: stonewalling pairs accessed: 24041
80: stonewalling pairs accessed: 24041
81: stonewalling pairs accessed: 24041
22: stonewalling pairs accessed: 26723
24: stonewalling pairs accessed: 26723
26: stonewalling pairs accessed: 26723
27: stonewalling pairs accessed: 26723
28: stonewalling pairs accessed: 26723
29: stonewalling pairs accessed: 26723
30: stonewalling pairs accessed: 26723
31: stonewalling pairs accessed: 26723
16: stonewalling pairs accessed: 26723
17: stonewalling pairs accessed: 26723
18: stonewalling pairs accessed: 26723
19: stonewalling pairs accessed: 26723
20: stonewalling pairs accessed: 26723
21: stonewalling pairs accessed: 26723
23: stonewalling pairs accessed: 26723
25: stonewalling pairs accessed: 26723
656: stonewalling pairs accessed: 24987
657: stonewalling pairs accessed: 24987
658: stonewalling pairs accessed: 24987
659: stonewalling pairs accessed: 24987
660: stonewalling pairs accessed: 24987
661: stonewalling pairs accessed: 24987
662: stonewalling pairs accessed: 24987
663: stonewalling pairs accessed: 24987
664: stonewalling pairs accessed: 24987
665: stonewalling pairs accessed: 24987
666: stonewalling pairs accessed: 24987
667: stonewalling pairs accessed: 24987
668: stonewalling pairs accessed: 24987
669: stonewalling pairs accessed: 24987
670: stonewalling pairs accessed: 24987
671: stonewalling pairs accessed: 24987
500: stonewalling pairs accessed: 23927
117: stonewalling pairs accessed: 24330
501: stonewalling pairs accessed: 23927
118: stonewalling pairs accessed: 24330
502: stonewalling pairs accessed: 23927
119: stonewalling pairs accessed: 24330
503: stonewalling pairs accessed: 23927
120: stonewalling pairs accessed: 24330
504: stonewalling pairs accessed: 23927
122: stonewalling pairs accessed: 24330
505: stonewalling pairs accessed: 23927
123: stonewalling pairs accessed: 24330
506: stonewalling pairs accessed: 23927
124: stonewalling pairs accessed: 24330
507: stonewalling pairs accessed: 23927
125: stonewalling pairs accessed: 24330
508: stonewalling pairs accessed: 23927
126: stonewalling pairs accessed: 24330
509: stonewalling pairs accessed: 23927
127: stonewalling pairs accessed: 24330
510: stonewalling pairs accessed: 23927
511: stonewalling pairs accessed: 23927
112: stonewalling pairs accessed: 24330
496: stonewalling pairs accessed: 23927
113: stonewalling pairs accessed: 24330
497: stonewalling pairs accessed: 23927
114: stonewalling pairs accessed: 24330
32: stonewalling pairs accessed: 24813
498: stonewalling pairs accessed: 23927
115: stonewalling pairs accessed: 24330
34: stonewalling pairs accessed: 24813
499: stonewalling pairs accessed: 23927
116: stonewalling pairs accessed: 24330
35: stonewalling pairs accessed: 24813
121: stonewalling pairs accessed: 24330
36: stonewalling pairs accessed: 24813
37: stonewalling pairs accessed: 24813
38: stonewalling pairs accessed: 24813
39: stonewalling pairs accessed: 24813
40: stonewalling pairs accessed: 24813
41: stonewalling pairs accessed: 24813
42: stonewalling pairs accessed: 24813
43: stonewalling pairs accessed: 24813
44: stonewalling pairs accessed: 24813
45: stonewalling pairs accessed: 24813
46: stonewalling pairs accessed: 24813
47: stonewalling pairs accessed: 24813
33: stonewalling pairs accessed: 24813
736: stonewalling pairs accessed: 25787
740: stonewalling pairs accessed: 25787
713: stonewalling pairs accessed: 24732
744: stonewalling pairs accessed: 25787
747: stonewalling pairs accessed: 25787
748: stonewalling pairs accessed: 25787
749: stonewalling pairs accessed: 25787
750: stonewalling pairs accessed: 25787
751: stonewalling pairs accessed: 25787
737: stonewalling pairs accessed: 25787
738: stonewalling pairs accessed: 25787
739: stonewalling pairs accessed: 25787
741: stonewalling pairs accessed: 25787
742: stonewalling pairs accessed: 25787
743: stonewalling pairs accessed: 25787
745: stonewalling pairs accessed: 25787
746: stonewalling pairs accessed: 25787
690: stonewalling pairs accessed: 25076
694: stonewalling pairs accessed: 25076
695: stonewalling pairs accessed: 25076
699: stonewalling pairs accessed: 25076
700: stonewalling pairs accessed: 25076
701: stonewalling pairs accessed: 25076
702: stonewalling pairs accessed: 25076
703: stonewalling pairs accessed: 25076
688: stonewalling pairs accessed: 25076
689: stonewalling pairs accessed: 25076
691: stonewalling pairs accessed: 25076
692: stonewalling pairs accessed: 25076
693: stonewalling pairs accessed: 25076
696: stonewalling pairs accessed: 25076
697: stonewalling pairs accessed: 25076
698: stonewalling pairs accessed: 25076
71: stonewalling pairs accessed: 25857
72: stonewalling pairs accessed: 25857
640: stonewalling pairs accessed: 24671
73: stonewalling pairs accessed: 25857
643: stonewalling pairs accessed: 24671
216: stonewalling pairs accessed: 23272
74: stonewalling pairs accessed: 25857
647: stonewalling pairs accessed: 24671
75: stonewalling pairs accessed: 25857
650: stonewalling pairs accessed: 24671
217: stonewalling pairs accessed: 23272
226: stonewalling pairs accessed: 24198
76: stonewalling pairs accessed: 25857
651: stonewalling pairs accessed: 24671
218: stonewalling pairs accessed: 23272
227: stonewalling pairs accessed: 24198
77: stonewalling pairs accessed: 25857
652: stonewalling pairs accessed: 24671
219: stonewalling pairs accessed: 23272
228: stonewalling pairs accessed: 24198
78: stonewalling pairs accessed: 25857
653: stonewalling pairs accessed: 24671
220: stonewalling pairs accessed: 23272
229: stonewalling pairs accessed: 24198
79: stonewalling pairs accessed: 25857
654: stonewalling pairs accessed: 24671
221: stonewalling pairs accessed: 23272
230: stonewalling pairs accessed: 24198
66: stonewalling pairs accessed: 25857
655: stonewalling pairs accessed: 24671
222: stonewalling pairs accessed: 23272
231: stonewalling pairs accessed: 24198
64: stonewalling pairs accessed: 25857
641: stonewalling pairs accessed: 24671
223: stonewalling pairs accessed: 23272
232: stonewalling pairs accessed: 24198
642: stonewalling pairs accessed: 24671
208: stonewalling pairs accessed: 23272
233: stonewalling pairs accessed: 24198
644: stonewalling pairs accessed: 24671
209: stonewalling pairs accessed: 23272
234: stonewalling pairs accessed: 24198
645: stonewalling pairs accessed: 24671
235: stonewalling pairs accessed: 24198
646: stonewalling pairs accessed: 24671
210: stonewalling pairs accessed: 23272
648: stonewalling pairs accessed: 24671
211: stonewalling pairs accessed: 23272
236: stonewalling pairs accessed: 24198
649: stonewalling pairs accessed: 24671
212: stonewalling pairs accessed: 23272
237: stonewalling pairs accessed: 24198
213: stonewalling pairs accessed: 23272
238: stonewalling pairs accessed: 24198
214: stonewalling pairs accessed: 23272
239: stonewalling pairs accessed: 24198
215: stonewalling pairs accessed: 23272
224: stonewalling pairs accessed: 24198
225: stonewalling pairs accessed: 24198
179: stonewalling pairs accessed: 27485
183: stonewalling pairs accessed: 27485
529: stonewalling pairs accessed: 25523
357: stonewalling pairs accessed: 23432
364: stonewalling pairs accessed: 23432
609: stonewalling pairs accessed: 24234
366: stonewalling pairs accessed: 23431
611: stonewalling pairs accessed: 24235
532: stonewalling pairs accessed: 25523
367: stonewalling pairs accessed: 23431
612: stonewalling pairs accessed: 24234
534: stonewalling pairs accessed: 25523
613: stonewalling pairs accessed: 24234
535: stonewalling pairs accessed: 25523
448: stonewalling pairs accessed: 26243
614: stonewalling pairs accessed: 24234
539: stonewalling pairs accessed: 25523
449: stonewalling pairs accessed: 26243
615: stonewalling pairs accessed: 24234
540: stonewalling pairs accessed: 25523
450: stonewalling pairs accessed: 26243
616: stonewalling pairs accessed: 24234
542: stonewalling pairs accessed: 25523
451: stonewalling pairs accessed: 26243
617: stonewalling pairs accessed: 24234
543: stonewalling pairs accessed: 25523
452: stonewalling pairs accessed: 26243
618: stonewalling pairs accessed: 24234
531: stonewalling pairs accessed: 25523
453: stonewalling pairs accessed: 26243
619: stonewalling pairs accessed: 24234
537: stonewalling pairs accessed: 25523
454: stonewalling pairs accessed: 26243
620: stonewalling pairs accessed: 24235
455: stonewalling pairs accessed: 26243
621: stonewalling pairs accessed: 24234
538: stonewalling pairs accessed: 25523
456: stonewalling pairs accessed: 26243
622: stonewalling pairs accessed: 24234
541: stonewalling pairs accessed: 25523
457: stonewalling pairs accessed: 26243
623: stonewalling pairs accessed: 24234
528: stonewalling pairs accessed: 25523
610: stonewalling pairs accessed: 24234
530: stonewalling pairs accessed: 25523
458: stonewalling pairs accessed: 26242
533: stonewalling pairs accessed: 25523
459: stonewalling pairs accessed: 26242
536: stonewalling pairs accessed: 25523
460: stonewalling pairs accessed: 26242
461: stonewalling pairs accessed: 26242
462: stonewalling pairs accessed: 26242
463: stonewalling pairs accessed: 26242
stonewalling pairs accessed min: 22135 max: 95000 -- min data: 1.0 GiB mean data: 1.5 GiB time: 300.1s
write     4094       91339      302.04      45.91      45.91      0.013636   798.78     0.050564   798.84     0   
Max Write: 4094.45 MiB/sec (4293.34 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        4094.45    4094.45    4094.45       0.00   91332.13   91332.13   91332.13       0.00  798.84260     300.08       3819.96     0    768  16    1   0     1        1         0    0  95000    47008    47008 3270820.2 POSIX      0
Finished            : Thu Jul  9 17:43:24 2020
mdtest_easy_delete
-- started at 07/09/2020 18:13:44 --

mdtest-3.3.0+dev was launched with 768 total task(s) on 48 node(s)
Command line used: /gpfs/fs1/io500-app-master/bin/mdtest '-r' '-F' '-P' '-d' './datafiles/2020.07.09-15.51.08-scr/mdt_easy' '-n' '1000000' '-u' '-L' '-a' 'POSIX' '-x' './datafiles/2020.07.09-15.51.08-scr/mdt_easy-stonewall' '-N' '1'
Path: /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr
FS: 390.6 TiB   Used FS: 9.8%   Inodes: 600.0 Mi   Used Inodes: 53.6%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
768 tasks, 768000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     403883.641     403883.368     403883.494          0.046
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.085          0.085          0.085          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        811.318        811.318        811.318          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         11.800         11.800         11.800          0.000
-- finished at 07/09/2020 18:27:28 --

mdtest_easy_stat
-- started at 07/09/2020 18:02:05 --

mdtest-3.3.0+dev was launched with 768 total task(s) on 48 node(s)
Command line used: /gpfs/fs1/io500-app-master/bin/mdtest '-T' '-F' '-P' '-d' './datafiles/2020.07.09-15.51.08-scr/mdt_easy' '-n' '1000000' '-u' '-L' '-a' 'POSIX' '-x' './datafiles/2020.07.09-15.51.08-scr/mdt_easy-stonewall' '-N' '1'
Path: /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr
FS: 390.6 TiB   Used FS: 9.8%   Inodes: 600.0 Mi   Used Inodes: 53.6%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
768 tasks, 768000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     791681.823     791680.447     791681.290          0.211
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        413.902        413.901        413.901          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 18:08:59 --

mdtest_easy_write
-- started at 07/09/2020 17:24:05 --

mdtest-3.3.0+dev was launched with 768 total task(s) on 48 node(s)
Command line used: /gpfs/fs1/io500-app-master/bin/mdtest '-Y' '-C' '-F' '-P' '-d' './datafiles/2020.07.09-15.51.08-scr/mdt_easy' '-n' '1000000' '-u' '-L' '-a' 'POSIX' '-x' './datafiles/2020.07.09-15.51.08-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr
FS: 390.6 TiB   Used FS: 8.3%   Inodes: 600.0 Mi   Used Inodes: 0.0%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
768 tasks, 768000000 files
Continue stonewall hit min: 372106 max: 426664 avg: 401529.3 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     990875.938     990874.103     990875.099          0.357
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    1047506.404             NA
   Tree creation             :          0.178          0.178          0.178          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        330.696        330.695        330.696          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        294.389             NA
   Tree creation             :          5.617          5.617          5.617          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 17:30:03 --

mdtest_hard_delete
-- started at 07/09/2020 18:32:59 --

mdtest-3.3.0+dev was launched with 768 total task(s) on 48 node(s)
Command line used: /gpfs/fs1/io500-app-master/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './datafiles/2020.07.09-15.51.08-scr/mdt_hard' '-n' '30000' '-x' './datafiles/2020.07.09-15.51.08-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr
FS: 390.6 TiB   Used FS: 9.8%   Inodes: 600.0 Mi   Used Inodes: 1.5%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
768 tasks, 23040000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       9833.994       9833.987       9833.990          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         11.815         11.815         11.815          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        946.295        946.295        946.295          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.085          0.085          0.085          0.000
-- finished at 07/09/2020 18:48:46 --

mdtest_hard_read
-- started at 07/09/2020 18:27:29 --

mdtest-3.3.0+dev was launched with 768 total task(s) on 48 node(s)
Command line used: /gpfs/fs1/io500-app-master/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './datafiles/2020.07.09-15.51.08-scr/mdt_hard' '-n' '30000' '-x' './datafiles/2020.07.09-15.51.08-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr
FS: 390.6 TiB   Used FS: 9.8%   Inodes: 600.0 Mi   Used Inodes: 1.5%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
768 tasks, 23040000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      28253.302      28253.256      28253.279          0.009
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        329.373        329.372        329.373          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 18:32:58 --

mdtest_hard_stat
-- started at 07/09/2020 18:10:36 --

mdtest-3.3.0+dev was launched with 768 total task(s) on 48 node(s)
Command line used: /gpfs/fs1/io500-app-master/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './datafiles/2020.07.09-15.51.08-scr/mdt_hard' '-n' '30000' '-x' './datafiles/2020.07.09-15.51.08-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr
FS: 390.6 TiB   Used FS: 9.8%   Inodes: 600.0 Mi   Used Inodes: 53.6%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
768 tasks, 23040000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      49574.648      49574.508      49574.547          0.020
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        187.715        187.714        187.714          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 18:13:43 --

mdtest_hard_write
-- started at 07/09/2020 17:43:25 --

mdtest-3.3.0+dev was launched with 768 total task(s) on 48 node(s)
Command line used: /gpfs/fs1/io500-app-master/bin/mdtest '-Y' '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './datafiles/2020.07.09-15.51.08-scr/mdt_hard' '-n' '30000' '-x' './datafiles/2020.07.09-15.51.08-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /gpfs/fs1/io500-app-master/datafiles/2020.07.09-15.51.08-scr
FS: 390.6 TiB   Used FS: 9.8%   Inodes: 600.0 Mi   Used Inodes: 52.1%

Nodemap: 111111111111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
768 tasks, 23040000 files
Continue stonewall hit min: 7729 max: 12117 avg: 9374.6 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      18473.222      18473.199      18473.212          0.004
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      23993.113             NA
   Tree creation             :       2896.678       2896.678       2896.678          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        503.749        503.748        503.749          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.072             NA
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 17:52:11 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               90.976 GiB/s : time 370.78 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              990.876 kiops : time 330.70 seconds
[RESULT] BW   phase 2            ior_hard_write                3.998 GiB/s : time 798.78 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               18.473 kiops : time 503.75 seconds
[RESULT] IOPS phase 3                      find             1826.850 kiops : time 184.46 seconds
[RESULT] BW   phase 3             ior_easy_read               82.862 GiB/s : time 407.77 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              791.682 kiops : time 413.90 seconds
[RESULT] BW   phase 4             ior_hard_read               33.844 GiB/s : time  94.24 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               49.575 kiops : time 187.71 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              403.884 kiops : time 811.32 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               28.253 kiops : time 329.37 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                9.834 kiops : time 947.34 seconds
[SCORE] Bandwidth 31.7799 GiB/s : IOPS 139.964 kiops : TOTAL 66.6936