HAL

Institution National Center for Supercomputing Applications
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.7
Client Kernel Version 4.18.0-147.0.3.el7.ppc64le

DATA SERVER

Storage Type NVMe
Volatile Memory 32
Storage Interface Infiniband
Network Infiniband
Software Version 5.0.5.0
OS Version 7.6

INFORMATION

Client Nodes 10
Client Total Procs 100
Metadata Nodes 8
Metadata Storage Devices 2
Data Nodes 8
Data Storage Devices 2

METADATA

Easy Write 71.46 kIOP/s
Easy Stat 105.83 kIOP/s
Easy Delete 33.26 kIOP/s
Hard Write 2.39 kIOP/s
Hard Read 65.37 kIOP/s
Hard Stat 104.91 kIOP/s
Hard Delete 0.46 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpiexec"
  io500_mpiargs="-np 100"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Jun 27 10:37:55 2020
Command line        : /home/malone12/benchmark/io500-app/bin/ior -r -R -a POSIX -t 4m -v -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_easy/stonewall
Machine             : Linux hal01
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Sat Jun 27 10:37:55 2020
Path                : /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_easy
FS                  : 223.2 TiB   Used FS: 52.1%   Inodes: 194.5 Mi   Used Inodes: 95.9%
Participating tasks: 100
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 100
clients per node    : 10
repetitions         : 1
xfersize            : 4 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 946.04 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Sat Jun 27 10:37:55 2020

WARNING: Expected aggregate file size       = 1040187392000000.
WARNING: Stat() of aggregate file size      = 5979399782400.
WARNING: Using actual aggregate bytes moved = 5979399782400.
read      13043      3260.95    0.000176    10158080000 4096       0.034485   437.17     0.002172   437.21     0   
Max Read:  13042.72 MiB/sec (13676.29 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        13042.72   13042.72   13042.72       0.00    3260.68    3260.68    3260.68       0.00  437.20930         NA            NA     0    100  10    1   1     1        1         0    0      1 10401873920000  4194304 5702400.0 POSIX      0
Finished            : Sat Jun 27 10:45:12 2020
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Jun 27 09:56:10 2020
Command line        : /home/malone12/benchmark/io500-app/bin/ior -w -a POSIX -t 4m -v -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux hal01
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Sat Jun 27 09:56:10 2020
Path                : /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_easy
FS                  : 223.2 TiB   Used FS: 52.0%   Inodes: 194.5 Mi   Used Inodes: 81.4%
Participating tasks: 100
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 100
clients per node    : 10
repetitions         : 1
xfersize            : 4 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 946.04 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Sat Jun 27 09:56:11 2020
23: stonewalling pairs accessed: 11120
30: stonewalling pairs accessed: 10654
31: stonewalling pairs accessed: 10931
32: stonewalling pairs accessed: 10945
34: stonewalling pairs accessed: 10905
39: stonewalling pairs accessed: 10842
35: stonewalling pairs accessed: 10958
36: stonewalling pairs accessed: 10839
77: stonewalling pairs accessed: 10996
22: stonewalling pairs accessed: 11650
38: stonewalling pairs accessed: 10882
90: stonewalling pairs accessed: 11807
37: stonewalling pairs accessed: 10900
65: stonewalling pairs accessed: 10751
11: stonewalling pairs accessed: 11846
17: stonewalling pairs accessed: 11937
71: stonewalling pairs accessed: 11060
19: stonewalling pairs accessed: 12002
47: stonewalling pairs accessed: 12509
12: stonewalling pairs accessed: 12164
14: stonewalling pairs accessed: 12043
16: stonewalling pairs accessed: 12020
48: stonewalling pairs accessed: 12256
41: stonewalling pairs accessed: 12413
45: stonewalling pairs accessed: 12278
15: stonewalling pairs accessed: 12078
83: stonewalling pairs accessed: 12250
78: stonewalling pairs accessed: 10919
75: stonewalling pairs accessed: 10724
40: stonewalling pairs accessed: 12500
46: stonewalling pairs accessed: 12963
79: stonewalling pairs accessed: 11324
42: stonewalling pairs accessed: 12261
28: stonewalling pairs accessed: 11387
18: stonewalling pairs accessed: 11993
64: stonewalling pairs accessed: 10952
74: stonewalling pairs accessed: 10984
66: stonewalling pairs accessed: 10836
67: stonewalling pairs accessed: 10949
69: stonewalling pairs accessed: 10696
60: stonewalling pairs accessed: 11145
62: stonewalling pairs accessed: 10631
61: stonewalling pairs accessed: 10854
76: stonewalling pairs accessed: 10783
13: stonewalling pairs accessed: 12278
20: stonewalling pairs accessed: 11302
98: stonewalling pairs accessed: 12073
81: stonewalling pairs accessed: 12343
57: stonewalling pairs accessed: 9854
63: stonewalling pairs accessed: 10719
53: stonewalling pairs accessed: 10416
54: stonewalling pairs accessed: 10744
58: stonewalling pairs accessed: 10353
59: stonewalling pairs accessed: 10128
51: stonewalling pairs accessed: 9884
55: stonewalling pairs accessed: 10308
50: stonewalling pairs accessed: 9852
85: stonewalling pairs accessed: 12184
5: stonewalling pairs accessed: 14251
49: stonewalling pairs accessed: 12279
33: stonewalling pairs accessed: 10945
10: stonewalling pairs accessed: 11920
26: stonewalling pairs accessed: 11454
91: stonewalling pairs accessed: 11905
68: stonewalling pairs accessed: 10928
87: stonewalling pairs accessed: 12252
56: stonewalling pairs accessed: 9887
73: stonewalling pairs accessed: 10709
43: stonewalling pairs accessed: 12889
24: stonewalling pairs accessed: 11388
94: stonewalling pairs accessed: 11888
80: stonewalling pairs accessed: 12121
52: stonewalling pairs accessed: 10129
44: stonewalling pairs accessed: 12256
27: stonewalling pairs accessed: 11489
96: stonewalling pairs accessed: 11901
82: stonewalling pairs accessed: 12115
29: stonewalling pairs accessed: 11429
97: stonewalling pairs accessed: 11733
89: stonewalling pairs accessed: 12193
84: stonewalling pairs accessed: 12166
21: stonewalling pairs accessed: 11431
92: stonewalling pairs accessed: 11860
25: stonewalling pairs accessed: 11401
95: stonewalling pairs accessed: 11852
86: stonewalling pairs accessed: 12259
88: stonewalling pairs accessed: 12269
93: stonewalling pairs accessed: 11712
99: stonewalling pairs accessed: 11892
0: stonewalling pairs accessed: 13954
2: stonewalling pairs accessed: 14106
8: stonewalling pairs accessed: 14090
3: stonewalling pairs accessed: 13923
9: stonewalling pairs accessed: 14254
1: stonewalling pairs accessed: 14201
6: stonewalling pairs accessed: 14156
7: stonewalling pairs accessed: 13845
4: stonewalling pairs accessed: 14256
70: stonewalling pairs accessed: 11015
stonewalling pairs accessed min: 9852 max: 14256 -- min data: 38.5 GiB mean data: 45.7 GiB time: 301.3s
72: stonewalling pairs accessed: 10975
WARNING: Expected aggregate file size       = 1040187392000000.
WARNING: Stat() of aggregate file size      = 5979399782400.
WARNING: Using actual aggregate bytes moved = 5979399782400.
WARNING: maybe caused by deadlineForStonewalling
write     15217      3806       0.000123    10158080000 4096       0.192096   374.54     0.013982   374.75     0   
Max Write: 15216.67 MiB/sec (15955.84 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       15216.67   15216.67   15216.67       0.00    3804.17    3804.17    3804.17       0.00  374.74690     301.29      15520.55     0    100  10    1   1     1        1         0    0      1 10401873920000  4194304 5702400.0 POSIX      0
Finished            : Sat Jun 27 10:02:25 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Jun 27 10:49:39 2020
Command line        : /home/malone12/benchmark/io500-app/bin/ior -r -R -s 10000000 -a POSIX -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_hard/stonewall
Machine             : Linux hal01
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Sat Jun 27 10:49:39 2020
Path                : /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_hard
FS                  : 223.2 TiB   Used FS: 52.1%   Inodes: 194.5 Mi   Used Inodes: 95.9%
Participating tasks: 100
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 100
clients per node    : 10
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 42.75 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing read performance test: Sat Jun 27 10:49:39 2020

WARNING: Expected aggregate file size       = 47008000000000.
WARNING: Stat() of aggregate file size      = 210986006400.
WARNING: Using actual aggregate bytes moved = 210986006400.
read      2145.14    47864      93.77       45.91      45.91      0.019181   93.77      0.007354   93.80      0   
Max Read:  2145.14 MiB/sec (2249.34 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         2145.14    2145.14    2145.14       0.00   47850.12   47850.12   47850.12       0.00   93.79914         NA            NA     0    100  10    1   0     1        1         0    0 10000000    47008    47008  201211.9 POSIX      0
Finished            : Sat Jun 27 10:51:13 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Jun 27 10:08:58 2020
Command line        : /home/malone12/benchmark/io500-app/bin/ior -w -s 10000000 -a POSIX -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux hal01
Start time skew across all tasks: 0.00 sec
TestID              : 0
StartTime           : Sat Jun 27 10:08:58 2020
Path                : /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_hard
FS                  : 223.2 TiB   Used FS: 52.1%   Inodes: 194.5 Mi   Used Inodes: 94.5%
Participating tasks: 100
Using reorderTasks '-C' (useful to avoid read cache in client)

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 100
clients per node    : 10
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 42.75 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
Commencing write performance test: Sat Jun 27 10:08:58 2020
5: stonewalling pairs accessed: 44834
33: stonewalling pairs accessed: 34158
73: stonewalling pairs accessed: 39222
34: stonewalling pairs accessed: 34155
79: stonewalling pairs accessed: 39220
88: stonewalling pairs accessed: 40004
90: stonewalling pairs accessed: 34511
40: stonewalling pairs accessed: 30534
83: stonewalling pairs accessed: 40004
98: stonewalling pairs accessed: 34513
80: stonewalling pairs accessed: 40006
85: stonewalling pairs accessed: 40007
86: stonewalling pairs accessed: 40007
89: stonewalling pairs accessed: 40006
36: stonewalling pairs accessed: 34158
97: stonewalling pairs accessed: 34516
41: stonewalling pairs accessed: 30536
71: stonewalling pairs accessed: 39231
82: stonewalling pairs accessed: 40005
84: stonewalling pairs accessed: 40005
92: stonewalling pairs accessed: 34519
44: stonewalling pairs accessed: 30532
8: stonewalling pairs accessed: 44815
46: stonewalling pairs accessed: 30531
3: stonewalling pairs accessed: 44872
95: stonewalling pairs accessed: 34524
4: stonewalling pairs accessed: 44864
91: stonewalling pairs accessed: 34528
23: stonewalling pairs accessed: 37173
25: stonewalling pairs accessed: 37173
0: stonewalling pairs accessed: 44851
12: stonewalling pairs accessed: 35754
49: stonewalling pairs accessed: 30530
14: stonewalling pairs accessed: 35753
51: stonewalling pairs accessed: 30519
47: stonewalling pairs accessed: 30538
67: stonewalling pairs accessed: 33253
43: stonewalling pairs accessed: 30540
64: stonewalling pairs accessed: 33253
45: stonewalling pairs accessed: 30537
66: stonewalling pairs accessed: 33254
96: stonewalling pairs accessed: 34545
48: stonewalling pairs accessed: 30540
94: stonewalling pairs accessed: 34542
6: stonewalling pairs accessed: 44883
42: stonewalling pairs accessed: 30550
68: stonewalling pairs accessed: 33256
55: stonewalling pairs accessed: 30519
93: stonewalling pairs accessed: 34543
99: stonewalling pairs accessed: 34538
18: stonewalling pairs accessed: 35757
16: stonewalling pairs accessed: 35760
28: stonewalling pairs accessed: 37185
35: stonewalling pairs accessed: 34193
24: stonewalling pairs accessed: 37184
39: stonewalling pairs accessed: 34194
37: stonewalling pairs accessed: 34193
19: stonewalling pairs accessed: 35757
31: stonewalling pairs accessed: 34196
62: stonewalling pairs accessed: 33261
10: stonewalling pairs accessed: 35754
50: stonewalling pairs accessed: 30519
9: stonewalling pairs accessed: 44813
11: stonewalling pairs accessed: 35757
1: stonewalling pairs accessed: 44879
7: stonewalling pairs accessed: 44838
60: stonewalling pairs accessed: 33261
2: stonewalling pairs accessed: 44873
81: stonewalling pairs accessed: 40032
87: stonewalling pairs accessed: 40033
29: stonewalling pairs accessed: 37211
32: stonewalling pairs accessed: 34218
22: stonewalling pairs accessed: 37199
13: stonewalling pairs accessed: 35763
72: stonewalling pairs accessed: 39258
75: stonewalling pairs accessed: 39258
70: stonewalling pairs accessed: 39258
61: stonewalling pairs accessed: 33262
76: stonewalling pairs accessed: 39258
77: stonewalling pairs accessed: 39258
53: stonewalling pairs accessed: 30519
54: stonewalling pairs accessed: 30519
56: stonewalling pairs accessed: 30519
17: stonewalling pairs accessed: 35771
15: stonewalling pairs accessed: 35767
30: stonewalling pairs accessed: 34242
78: stonewalling pairs accessed: 39261
58: stonewalling pairs accessed: 30524
74: stonewalling pairs accessed: 39270
38: stonewalling pairs accessed: 34231
52: stonewalling pairs accessed: 30521
21: stonewalling pairs accessed: 37229
27: stonewalling pairs accessed: 37229
26: stonewalling pairs accessed: 37230
57: stonewalling pairs accessed: 30519
59: stonewalling pairs accessed: 30519
63: stonewalling pairs accessed: 33266
20: stonewalling pairs accessed: 37230
stonewalling pairs accessed min: 30519 max: 44883 -- min data: 1.3 GiB mean data: 1.6 GiB time: 300.1s
65: stonewalling pairs accessed: 33267
69: stonewalling pairs accessed: 33269
WARNING: Expected aggregate file size       = 47008000000000.
WARNING: Stat() of aggregate file size      = 210986006400.
WARNING: Using actual aggregate bytes moved = 210986006400.
WARNING: maybe caused by deadlineForStonewalling
write     442.81     9879       300.40      45.91      45.91      0.046340   454.33     0.022424   454.40     0   
Max Write: 442.81 MiB/sec (464.32 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write         442.81     442.81     442.81       0.00    9877.47    9877.47    9877.47       0.00  454.39796     300.08        537.99     0    100  10    1   0     1        1         0    0 10000000    47008    47008  201211.9 POSIX      0
Finished            : Sat Jun 27 10:16:32 2020
mdtest_easy_delete
-- started at 06/27/2020 10:51:52 --

mdtest-3.3.0+dev was launched with 100 total task(s) on 10 node(s)
Command line used: /home/malone12/benchmark/io500-app/bin/mdtest '-r' '-F' '-P' '-Y' '-d' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_easy' '-n' '2300000' '-u' '-L' '-a' 'POSIX' '-x' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_easy-stonewall' '-N' '1'
Path: /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr
FS: 223.2 TiB   Used FS: 52.1%   Inodes: 194.5 Mi   Used Inodes: 95.9%

Nodemap: 1111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 10 for each phase.
100 tasks, 230000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      33257.918      33257.912      33257.914          0.002
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.211          0.211          0.211          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        814.284        814.284        814.284          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          4.731          4.731          4.731          0.000
-- finished at 06/27/2020 11:05:31 --

mdtest_easy_stat
-- started at 06/27/2020 10:45:17 --

mdtest-3.3.0+dev was launched with 100 total task(s) on 10 node(s)
Command line used: /home/malone12/benchmark/io500-app/bin/mdtest '-T' '-F' '-P' '-Y' '-d' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_easy' '-n' '2300000' '-u' '-L' '-a' 'POSIX' '-x' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_easy-stonewall' '-N' '1'
Path: /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr
FS: 223.2 TiB   Used FS: 52.1%   Inodes: 194.5 Mi   Used Inodes: 95.9%

Nodemap: 1111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 10 for each phase.
100 tasks, 230000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     105832.252     105832.147     105832.220          0.034
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        255.890        255.890        255.890          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/27/2020 10:49:33 --

mdtest_easy_write
-- started at 06/27/2020 10:02:31 --

mdtest-3.3.0+dev was launched with 100 total task(s) on 10 node(s)
Command line used: /home/malone12/benchmark/io500-app/bin/mdtest '-C' '-F' '-P' '-Y' '-d' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_easy' '-n' '2300000' '-u' '-L' '-a' 'POSIX' '-x' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr
FS: 223.2 TiB   Used FS: 51.8%   Inodes: 194.5 Mi   Used Inodes: 81.4%

Nodemap: 1111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 10 for each phase.
100 tasks, 230000000 files
Continue stonewall hit min: 214169 max: 270814 avg: 245788.3 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      71462.576      71462.489      71462.554          0.029
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      82639.971             NA
   Tree creation             :          0.383          0.383          0.383          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        378.960        378.959        378.959          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        297.421             NA
   Tree creation             :          2.613          2.613          2.613          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/27/2020 10:08:52 --

mdtest_hard_delete
-- started at 06/27/2020 11:06:28 --

mdtest-3.3.0+dev was launched with 100 total task(s) on 10 node(s)
Command line used: /home/malone12/benchmark/io500-app/bin/mdtest '-r' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_hard' '-n' '60000' '-x' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr
FS: 223.2 TiB   Used FS: 52.1%   Inodes: 194.5 Mi   Used Inodes: 82.8%

Nodemap: 1111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 10 for each phase.
100 tasks, 6000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        457.064        457.064        457.064          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.117          3.117          3.117          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       6526.876       6526.875       6526.875          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.321          0.321          0.321          0.000
-- finished at 06/27/2020 12:55:15 --

mdtest_hard_read
-- started at 06/27/2020 11:05:37 --

mdtest-3.3.0+dev was launched with 100 total task(s) on 10 node(s)
Command line used: /home/malone12/benchmark/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_hard' '-n' '60000' '-x' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr
FS: 223.2 TiB   Used FS: 52.1%   Inodes: 194.5 Mi   Used Inodes: 82.9%

Nodemap: 1111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 10 for each phase.
100 tasks, 6000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      65371.910      65371.827      65371.868          0.025
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :         45.634         45.634         45.634          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/27/2020 11:06:23 --

mdtest_hard_stat
-- started at 06/27/2020 10:51:18 --

mdtest-3.3.0+dev was launched with 100 total task(s) on 10 node(s)
Command line used: /home/malone12/benchmark/io500-app/bin/mdtest '-T' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_hard' '-n' '60000' '-x' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr
FS: 223.2 TiB   Used FS: 52.1%   Inodes: 194.5 Mi   Used Inodes: 95.9%

Nodemap: 1111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 10 for each phase.
100 tasks, 6000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     104914.285     104914.180     104914.233          0.032
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         28.435         28.435         28.435          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/27/2020 10:51:47 --

mdtest_hard_write
-- started at 06/27/2020 10:16:38 --

mdtest-3.3.0+dev was launched with 100 total task(s) on 10 node(s)
Command line used: /home/malone12/benchmark/io500-app/bin/mdtest '-C' '-t' '-F' '-Y' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_hard' '-n' '60000' '-x' '/gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /gpfs/hal/io500/datafiles/2020.06.27-06.53.38-scr
FS: 223.2 TiB   Used FS: 52.1%   Inodes: 194.5 Mi   Used Inodes: 94.5%

Nodemap: 1111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 10 for each phase.
100 tasks, 6000000 files
Continue stonewall hit min: 4394 max: 29832 avg: 8872.6 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       2388.929       2388.928       2388.929          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA       2956.780             NA
   Tree creation             :        429.535        429.535        429.535          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       1248.761       1248.761       1248.761          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.076             NA
   Tree creation             :          0.002          0.002          0.002          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 06/27/2020 10:37:26 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               14.860 GiB/s : time 374.54 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               71.463 kiops : time 378.96 seconds
[RESULT] BW   phase 2            ior_hard_write                0.432 GiB/s : time 454.33 seconds
[RESULT] IOPS phase 2         mdtest_hard_write                2.389 kiops : time 1248.76 seconds
[RESULT] IOPS phase 3                      find             1300.120 kiops : time  23.12 seconds
[RESULT] BW   phase 3             ior_easy_read               12.737 GiB/s : time 437.17 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              105.832 kiops : time 255.89 seconds
[RESULT] BW   phase 4             ior_hard_read                2.095 GiB/s : time  93.77 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat              104.914 kiops : time  28.43 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               33.258 kiops : time 814.28 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               65.372 kiops : time  45.63 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                0.457 kiops : time 6532.34 seconds
[SCORE] Bandwidth 3.61864 GiB/s : IOPS 35.369 kiops : TOTAL 11.3132