BeeGFS on Oracle Cloud

Institution Oracle Cloud Infrastructure
Client Procs Per Node
Client Operating System Oracle Linux UEK
Client Operating System Version 7.7
Client Kernel Version 4.14.35-1902.6.6.el7uek.x86_64

DATA SERVER

Storage Type NVMe
Volatile Memory 384 GB
Storage Interface NVMe
Network Ethernet
Software Version 7.1.5
OS Version OracleLinuxUEK77

INFORMATION

Client Nodes 10
Client Total Procs 160
Metadata Nodes 10
Metadata Storage Devices 1
Data Nodes 10
Data Storage Devices 10

METADATA

Easy Write 163.20 kIOP/s
Easy Stat 720.73 kIOP/s
Easy Delete 327.70 kIOP/s
Hard Write 3.02 kIOP/s
Hard Read 17.66 kIOP/s
Hard Stat 61.23 kIOP/s
Hard Delete 9.61 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpiexec"
io500_mpiargs="--allow-run-as-root -mca btl self -x UCX_TLS=rc,self,sm -x HCOLL_ENABLE_MCAST_ALL=0 -mca coll_hcoll_enable 0 -x UCX_IB_TRAFFIC_CLASS=105 -x UCX_IB_GID_INDEX=3 -n 160 -npernode 16 --hostfile /mnt/beeond/hostsfile.cn"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}

  mkdir -p $workdir-{scr,app}/{ior_easy,ior_hard,mdt_easy,mdt_hard}

# for ior_easy.
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=4 $PWD/$workdir-scr/ior_easy
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=4 $PWD/$workdir-app/ior_easy

# stripe across all OSTs for ior_hard, 256k chunksize
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=10  $PWD/$workdir-scr/ior_hard
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=10  $PWD/$workdir-app/ior_hard

# for ior_easy.
##beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=4 $PWD/$workdir-scr/ior_easy
##beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=4 $PWD/$workdir-app/ior_easy

# stripe across all OSTs for ior_hard, 256k chunksize
#beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=256k --numtargets=3 $PWD/$workdir-scr/ior_hard
#beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=256k --numtargets=3 $PWD/$workdir-app/ior_hard

# turn off striping and use small chunks for mdtest
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=64k --numtargets=1 $PWD/$workdir-scr/mdt_easy
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=64k --numtargets=1 $PWD/$workdir-scr/mdt_hard

beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=64k --numtargets=1 $PWD/$workdir-app/mdt_easy
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=64k --numtargets=1 $PWD/$workdir-app/mdt_hard


}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 13:52:30 2020
Command line        : /mnt/beeond/io500-app/bin/ior -r -R -a POSIX -t 1m -b 3600000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./out//2020.07.09-12.23.20-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./out//2020.07.09-12.23.20-scr/ior_easy/stonewall
Machine             : Linux inst-052fy-united-hermit
TestID              : 0
StartTime           : Thu Jul  9 13:52:30 2020
Path                : /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr/ior_easy
FS                  : 58.2 TiB   Used FS: 47.0%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.07.09-12.23.20-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 160
clients per node    : 16
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 3.43 TiB
aggregate filesize  : 549.32 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 603979776000000.
WARNING: Stat() of aggregate file size      = 13522100551680.
WARNING: Using actual aggregate bytes moved = 13522100551680.
read      27874      27875      0.000129    3686400000 1024.00    0.020637   462.62     0.016468   462.64     0   
Max Read:  27873.83 MiB/sec (29227.83 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        27873.83   27873.83   27873.83       0.00   27873.83   27873.83   27873.83       0.00  462.64464         NA            NA     0    160  16    1   1     1        1         0    0      1 3774873600000  1048576 12895680.0 POSIX      0
Finished            : Thu Jul  9 14:00:12 2020
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 13:18:39 2020
Command line        : /mnt/beeond/io500-app/bin/ior -w -a POSIX -t 1m -b 3600000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./out//2020.07.09-12.23.20-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./out//2020.07.09-12.23.20-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux inst-052fy-united-hermit
TestID              : 0
StartTime           : Thu Jul  9 13:18:39 2020
Path                : /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr/ior_easy
FS                  : 58.2 TiB   Used FS: 29.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.07.09-12.23.20-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 160
clients per node    : 16
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 3.43 TiB
aggregate filesize  : 549.32 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 56920 max: 80598 -- min data: 55.6 GiB mean data: 63.1 GiB time: 300.1s
WARNING: Expected aggregate file size       = 603979776000000.
WARNING: Stat() of aggregate file size      = 13522100551680.
WARNING: Using actual aggregate bytes moved = 13522100551680.
WARNING: maybe caused by deadlineForStonewalling
write     26683      26685      0.000131    3686400000 1024.00    0.047907   483.26     0.038639   483.29     0   
Max Write: 26683.11 MiB/sec (27979.27 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       26683.11   26683.11   26683.11       0.00   26683.11   26683.11   26683.11       0.00  483.28996     300.08      34447.68     0    160  16    1   1     1        1         0    0      1 3774873600000  1048576 12895680.0 POSIX      0
Finished            : Thu Jul  9 13:26:43 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 14:02:32 2020
Command line        : /mnt/beeond/io500-app/bin/ior -r -R -s 40000000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./out//2020.07.09-12.23.20-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./out//2020.07.09-12.23.20-scr/ior_hard/stonewall
Machine             : Linux inst-052fy-united-hermit
TestID              : 0
StartTime           : Thu Jul  9 14:02:32 2020
Path                : /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr/ior_hard
FS                  : 58.2 TiB   Used FS: 47.0%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.07.09-12.23.20-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 40000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 160
clients per node    : 16
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 273.62 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 300851200000000.
WARNING: Stat() of aggregate file size      = 1505647436800.
WARNING: Using actual aggregate bytes moved = 1505647436800.
read      18613      415336     77.10       45.91      45.91      0.037462   77.12      0.013561   77.14      0   
Max Read:  18613.03 MiB/sec (19517.18 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        18613.03   18613.03   18613.03       0.00  415188.43  415188.43  415188.43       0.00   77.14473         NA            NA     0    160  16    1   0     1        1         0    0 40000000    47008    47008 1435897.2 POSIX      0
Finished            : Thu Jul  9 14:03:50 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Jul  9 13:36:54 2020
Command line        : /mnt/beeond/io500-app/bin/ior -w -s 40000000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./out//2020.07.09-12.23.20-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./out//2020.07.09-12.23.20-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux inst-052fy-united-hermit
TestID              : 0
StartTime           : Thu Jul  9 13:36:54 2020
Path                : /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr/ior_hard
FS                  : 58.2 TiB   Used FS: 44.7%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.07.09-12.23.20-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 40000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 160
clients per node    : 16
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 273.62 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 122766 max: 200185 -- min data: 5.4 GiB mean data: 6.5 GiB time: 312.3s
WARNING: Expected aggregate file size       = 300851200000000.
WARNING: Stat() of aggregate file size      = 1505647436800.
WARNING: Using actual aggregate bytes moved = 1505647436800.
WARNING: maybe caused by deadlineForStonewalling
write     3071.03    68508      457.32      45.91      45.91      1.00       467.53     0.973843   467.56     0   
Max Write: 3071.03 MiB/sec (3220.20 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        3071.03    3071.03    3071.03       0.00   68503.32   68503.32   68503.32       0.00  467.56273     312.26       3427.47     0    160  16    1   0     1        1         0    0 40000000    47008    47008 1435897.2 POSIX      0
Finished            : Thu Jul  9 13:44:41 2020
mdtest_easy_delete
-- started at 07/09/2020 14:04:19 --

mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-r' '-F' '-P' '-d' './out//2020.07.09-12.23.20-scr/mdt_easy' '-n' '50000000' '-u' '-L' '-a' 'POSIX' '-x' './out//2020.07.09-12.23.20-scr/mdt_easy-stonewall' '-N' '1'
Path: /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr
FS: 58.2 TiB   Used FS: 47.0%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
160 tasks, 8000000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     327699.547     327699.254     327699.419          0.060
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.040          0.040          0.040          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        285.740        285.740        285.740          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         25.138         25.138         25.138          0.000
-- finished at 07/09/2020 14:09:32 --

mdtest_easy_stat
-- started at 07/09/2020 14:00:17 --

mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-T' '-F' '-P' '-d' './out//2020.07.09-12.23.20-scr/mdt_easy' '-n' '50000000' '-u' '-L' '-a' 'POSIX' '-x' './out//2020.07.09-12.23.20-scr/mdt_easy-stonewall' '-N' '1'
Path: /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr
FS: 58.2 TiB   Used FS: 47.0%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
160 tasks, 8000000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     720732.440     720731.046     720731.755          0.369
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        129.919        129.919        129.919          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 14:02:27 --

mdtest_easy_write
-- started at 07/09/2020 13:26:48 --

mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-Y' '-C' '-F' '-P' '-d' './out//2020.07.09-12.23.20-scr/mdt_easy' '-n' '50000000' '-u' '-L' '-a' 'POSIX' '-x' './out//2020.07.09-12.23.20-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr
FS: 58.2 TiB   Used FS: 44.0%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
160 tasks, 8000000000 files
Continue stonewall hit min: 349554 max: 585230 avg: 417069.2 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     163196.534     163196.466     163196.508          0.014
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     220902.356             NA
   Tree creation             :         16.022         16.022         16.022          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        573.767        573.767        573.767          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        302.084             NA
   Tree creation             :          0.062          0.062          0.062          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 13:36:22 --

mdtest_hard_delete
-- started at 07/09/2020 14:10:38 --

mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './out//2020.07.09-12.23.20-scr/mdt_hard' '-n' '20000000' '-x' './out//2020.07.09-12.23.20-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr
FS: 58.2 TiB   Used FS: 46.4%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
160 tasks, 3200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       9612.585       9612.574       9612.578          0.002
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.746          0.746          0.746          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        115.066        115.066        115.066          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.340          1.340          1.340          0.000
-- finished at 07/09/2020 14:12:35 --

mdtest_hard_read
-- started at 07/09/2020 14:09:34 --

mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './out//2020.07.09-12.23.20-scr/mdt_hard' '-n' '20000000' '-x' './out//2020.07.09-12.23.20-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr
FS: 58.2 TiB   Used FS: 46.4%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
160 tasks, 3200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      17659.771      17659.673      17659.708          0.033
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :         62.633         62.633         62.633          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 14:10:37 --

mdtest_hard_stat
-- started at 07/09/2020 14:03:56 --

mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './out//2020.07.09-12.23.20-scr/mdt_hard' '-n' '20000000' '-x' './out//2020.07.09-12.23.20-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr
FS: 58.2 TiB   Used FS: 47.0%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
160 tasks, 3200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      61228.159      61227.804      61228.048          0.066
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         18.065         18.065         18.065          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 14:04:14 --

mdtest_hard_write
-- started at 07/09/2020 13:44:44 --

mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-Y' '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './out//2020.07.09-12.23.20-scr/mdt_hard' '-n' '20000000' '-x' './out//2020.07.09-12.23.20-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /mnt/beeond/io500-app/out/2020.07.09-12.23.20-scr
FS: 58.2 TiB   Used FS: 47.0%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 16 for each phase.
160 tasks, 3200000000 files
Continue stonewall hit min: 5648 max: 6913 avg: 6141.4 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       3020.908       3020.906       3020.907          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA       3274.681             NA
   Tree creation             :        540.538        540.538        540.538          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        366.142        366.142        366.142          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.068             NA
   Tree creation             :          0.002          0.002          0.002          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/09/2020 13:50:50 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               26.058 GiB/s : time 483.26 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              163.197 kiops : time 573.77 seconds
[RESULT] BW   phase 2            ior_hard_write                2.999 GiB/s : time 467.53 seconds
[RESULT] IOPS phase 2         mdtest_hard_write                3.021 kiops : time 366.14 seconds
[RESULT] IOPS phase 3                      find              987.000 kiops : time  95.99 seconds
[RESULT] BW   phase 3             ior_easy_read               27.221 GiB/s : time 462.62 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              720.732 kiops : time 129.92 seconds
[RESULT] BW   phase 4             ior_hard_read               18.177 GiB/s : time  77.12 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               61.228 kiops : time  18.07 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              327.700 kiops : time 285.74 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               17.660 kiops : time  62.63 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                9.613 kiops : time 118.22 seconds
[SCORE] Bandwidth 14.0228 GiB/s : IOPS 76.6742 kiops : TOTAL 32.79