BeeGFS on Oracle Cloud

Institution Oracle Cloud Infrastructure
Client Procs Per Node
Client Operating System Oracle Linux UEK
Client Operating System Version 7.7
Client Kernel Version 4.14.35-1902.6.6.el7uek.x86_64

DATA SERVER

Storage Type NVMe
Volatile Memory 384 GB
Storage Interface NVMe
Network Ethernet
Software Version 7.1.5
OS Version OracleLinuxUEK77

INFORMATION

Client Nodes 170
Client Total Procs 2,040
Metadata Nodes 150
Metadata Storage Devices 1
Data Nodes 170
Data Storage Devices 1

METADATA

Easy Write 1,913.30 kIOP/s
Easy Stat 8,261.12 kIOP/s
Easy Delete 3,602.59 kIOP/s
Hard Write 2.42 kIOP/s
Hard Read 18.24 kIOP/s
Hard Stat 25.90 kIOP/s
Hard Delete 8.34 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpiexec"
  io500_mpiargs="--allow-run-as-root -mca btl self -x UCX_TLS=rc,self,sm -x HCOLL_ENABLE_MCAST_ALL=0 -mca coll_hcoll_enable 0 -x UCX_IB_TRAFFIC_CLASS=105 -x UCX_IB_GID_INDEX=3 -n 2040 -npernode 12 --hostfile /mnt/beeond/hostsfile.cn"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
  mkdir -p $workdir-{scr,app}/{ior_easy,ior_hard,mdt_easy,mdt_hard}

# for ior_easy.
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=4 $PWD/$workdir-scr/ior_easy
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=4 $PWD/$workdir-app/ior_easy

# stripe across all OSTs for ior_hard, 256k chunksize
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=170 $PWD/$workdir-scr/ior_hard
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=170 $PWD/$workdir-app/ior_hard

# for ior_easy.
##beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=4 $PWD/$workdir-scr/ior_easy
##beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=1m --numtargets=4 $PWD/$workdir-app/ior_easy

# stripe across all OSTs for ior_hard, 256k chunksize
#beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=256k --numtargets=3 $PWD/$workdir-scr/ior_hard
#beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=256k --numtargets=3 $PWD/$workdir-app/ior_hard

# turn off striping and use small chunks for mdtest
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=64k --numtargets=1 $PWD/$workdir-scr/mdt_easy
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=64k --numtargets=1 $PWD/$workdir-scr/mdt_hard

beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=64k --numtargets=1 $PWD/$workdir-app/mdt_easy
beegfs-ctl --mount=/mnt/beeond --setpattern --chunksize=64k --numtargets=1 $PWD/$workdir-app/mdt_hard

}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Jul  6 18:43:14 2020
Command line        : /mnt/beeond/io500-app/bin/ior -r -R -a POSIX -t 1m -b 200000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./out//2020.07.06-17.56.35-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./out//2020.07.06-17.56.35-scr/ior_easy/stonewall
Machine             : Linux inst-0ne1g-guided-louse
TestID              : 0
StartTime           : Mon Jul  6 18:43:14 2020
Path                : /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr/ior_easy
FS                  : 989.4 TiB   Used FS: 21.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.07.06-17.56.35-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 170
tasks               : 2040
clients per node    : 12
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 195.31 GiB
aggregate filesize  : 389.10 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 427819008000000.
WARNING: Stat() of aggregate file size      = 212061325885440.
WARNING: Using actual aggregate bytes moved = 212061325885440.
read      432333     432761     0.002337    204800000  1024.00    0.195843   467.32     0.290874   467.78     0   
Max Read:  432333.17 MiB/sec (453334.18 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       432333.17  432333.17  432333.17       0.00  432333.17  432333.17  432333.17       0.00  467.78146         NA            NA     0   2040  12    1   1     1        1         0    0      1 209715200000  1048576 202237440.0 POSIX      0
Finished            : Mon Jul  6 18:51:02 2020
ior_easy_write
Warning: Permanently added the ECDSA host key for IP address '192.168.168.69' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.133' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.78' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.142' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.137' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.132' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.73' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.68' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.71' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.135' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.72' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.79' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.143' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.138' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.74' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.134' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.83' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.75' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.139' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.136' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.76' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.140' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.141' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.77' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.85' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.149' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.70' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.147' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.144' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.80' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.150' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.86' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.146' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.82' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.84' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.148' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.87' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.81' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.145' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.151' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.67' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.131' to the list of known hosts.
Warning: Permanently added the ECDSA host key for IP address '192.168.168.130' to the list of known hosts.
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Jul  6 17:57:01 2020
Command line        : /mnt/beeond/io500-app/bin/ior -w -a POSIX -t 1m -b 200000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o ./out//2020.07.06-17.56.35-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=./out//2020.07.06-17.56.35-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux inst-0ne1g-guided-louse
TestID              : 0
StartTime           : Mon Jul  6 17:57:01 2020
Path                : /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr/ior_easy
FS                  : 989.4 TiB   Used FS: 0.0%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.07.06-17.56.35-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 170
tasks               : 2040
clients per node    : 12
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 195.31 GiB
aggregate filesize  : 389.10 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 60752 max: 99136 -- min data: 59.3 GiB mean data: 84.1 GiB time: 300.1s
WARNING: Expected aggregate file size       = 427819008000000.
WARNING: Stat() of aggregate file size      = 212061325885440.
WARNING: Using actual aggregate bytes moved = 212061325885440.
WARNING: maybe caused by deadlineForStonewalling
write     341867     441276     0.002149    204800000  1024.00    132.84     458.30     0.454498   591.57     0   
Max Write: 341867.05 MiB/sec (358473.58 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      341867.05  341867.05  341867.05       0.00  341867.05  341867.05  341867.05       0.00  591.56752     300.10     585732.46     0   2040  12    1   1     1        1         0    0      1 209715200000  1048576 202237440.0 POSIX      0
Finished            : Mon Jul  6 18:10:23 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Jul  6 18:53:47 2020
Command line        : /mnt/beeond/io500-app/bin/ior -r -R -s 1600000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./out//2020.07.06-17.56.35-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./out//2020.07.06-17.56.35-scr/ior_hard/stonewall
Machine             : Linux inst-0ne1g-guided-louse
TestID              : 0
StartTime           : Mon Jul  6 18:53:47 2020
Path                : /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr/ior_hard
FS                  : 989.4 TiB   Used FS: 21.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.07.06-17.56.35-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1600000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 170
tasks               : 2040
clients per node    : 12
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 139.55 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 153434112000000.
WARNING: Stat() of aggregate file size      = 17620757007360.
WARNING: Using actual aggregate bytes moved = 17620757007360.
read      191839     4348054    85.47       45.91      45.91      2.12       86.21      0.737258   87.60      0   
Max Read:  191839.21 MiB/sec (201157.99 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       191839.21  191839.21  191839.21       0.00 4279228.92 4279228.92 4279228.92       0.00   87.59660         NA            NA     0   2040  12    1   0     1        1         0    0 1600000    47008    47008 16804464.0 POSIX      0
Finished            : Mon Jul  6 18:55:14 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Jul  6 18:22:21 2020
Command line        : /mnt/beeond/io500-app/bin/ior -w -s 1600000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o ./out//2020.07.06-17.56.35-scr/ior_hard/IOR_file -O stoneWallingStatusFile=./out//2020.07.06-17.56.35-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux inst-0ne1g-guided-louse
TestID              : 0
StartTime           : Mon Jul  6 18:22:21 2020
Path                : /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr/ior_hard
FS                  : 989.4 TiB   Used FS: 20.0%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.07.06-17.56.35-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1600000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 170
tasks               : 2040
clients per node    : 12
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 139.55 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 114614 max: 183748 -- min data: 5.0 GiB mean data: 6.8 GiB time: 305.7s
WARNING: Expected aggregate file size       = 153434112000000.
WARNING: Stat() of aggregate file size      = 17620757007360.
WARNING: Using actual aggregate bytes moved = 17620757007360.
WARNING: maybe caused by deadlineForStonewalling
write     36573      851726     367.12      45.91      45.91      20.25      440.10     1.36       459.47     0   
Max Write: 36573.28 MiB/sec (38349.87 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       36573.28   36573.28   36573.28       0.00  815815.73  815815.73  815815.73       0.00  459.47376     305.69      46723.65     0   2040  12    1   0     1        1         0    0 1600000    47008    47008 16804464.0 POSIX      0
Finished            : Mon Jul  6 18:32:31 2020
mdtest_easy_delete
-- started at 07/06/2020 18:56:06 --

mdtest-3.3.0+dev was launched with 2040 total task(s) on 170 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-r' '-F' '-P' '-d' './out//2020.07.06-17.56.35-scr/mdt_easy' '-n' '2000000' '-u' '-L' '-a' 'POSIX' '-x' './out//2020.07.06-17.56.35-scr/mdt_easy-stonewall' '-N' '1'
Path: /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr
FS: 989.4 TiB   Used FS: 21.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 12 for each phase.
2040 tasks, 4080000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    3602587.481    3602579.070    3602584.178          0.665
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.040          0.040          0.040          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        352.402        352.401        352.401          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         25.030         25.030         25.030          0.000
-- finished at 07/06/2020 19:02:26 --

mdtest_easy_stat
-- started at 07/06/2020 18:51:08 --

mdtest-3.3.0+dev was launched with 2040 total task(s) on 170 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-T' '-F' '-P' '-d' './out//2020.07.06-17.56.35-scr/mdt_easy' '-n' '2000000' '-u' '-L' '-a' 'POSIX' '-x' './out//2020.07.06-17.56.35-scr/mdt_easy-stonewall' '-N' '1'
Path: /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr
FS: 989.4 TiB   Used FS: 21.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 12 for each phase.
2040 tasks, 4080000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    8261120.928    8261096.517    8261109.655          2.990
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        153.679        153.678        153.679          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/06/2020 18:53:42 --

mdtest_easy_write
-- started at 07/06/2020 18:10:45 --

mdtest-3.3.0+dev was launched with 2040 total task(s) on 170 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-Y' '-C' '-F' '-P' '-d' './out//2020.07.06-17.56.35-scr/mdt_easy' '-n' '2000000' '-u' '-L' '-a' 'POSIX' '-x' './out//2020.07.06-17.56.35-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr
FS: 989.4 TiB   Used FS: 19.5%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 12 for each phase.
2040 tasks, 4080000000 files
Continue stonewall hit min: 208856 max: 622331 avg: 406316.0 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    1913301.546    1913298.908    1913300.334          0.470
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    2951274.670             NA
   Tree creation             :          0.052          0.052          0.052          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        663.543        663.542        663.542          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        280.857             NA
   Tree creation             :         19.140         19.140         19.140          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/06/2020 18:22:08 --

mdtest_hard_delete
-- started at 07/06/2020 19:03:33 --

mdtest-3.3.0+dev was launched with 2040 total task(s) on 170 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './out//2020.07.06-17.56.35-scr/mdt_hard' '-n' '130000' '-x' './out//2020.07.06-17.56.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr
FS: 989.4 TiB   Used FS: 21.1%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 12 for each phase.
2040 tasks, 265200000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       8338.210       8338.167       8338.201          0.003
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.888          0.888          0.888          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        127.222        127.222        127.222          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.126          1.126          1.126          0.000
-- finished at 07/06/2020 19:05:41 --

mdtest_hard_read
-- started at 07/06/2020 19:02:31 --

mdtest-3.3.0+dev was launched with 2040 total task(s) on 170 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './out//2020.07.06-17.56.35-scr/mdt_hard' '-n' '130000' '-x' './out//2020.07.06-17.56.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr
FS: 989.4 TiB   Used FS: 21.1%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 12 for each phase.
2040 tasks, 265200000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      18243.668      18243.475      18243.632          0.014
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :         58.147         58.146         58.146          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/06/2020 19:03:29 --

mdtest_hard_stat
-- started at 07/06/2020 18:55:19 --

mdtest-3.3.0+dev was launched with 2040 total task(s) on 170 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './out//2020.07.06-17.56.35-scr/mdt_hard' '-n' '130000' '-x' './out//2020.07.06-17.56.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr
FS: 989.4 TiB   Used FS: 21.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 12 for each phase.
2040 tasks, 265200000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      25896.766      25896.598      25896.723          0.015
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         40.963         40.963         40.963          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/06/2020 18:56:00 --

mdtest_hard_write
-- started at 07/06/2020 18:32:36 --

mdtest-3.3.0+dev was launched with 2040 total task(s) on 170 node(s)
Command line used: /mnt/beeond/io500-app/bin/mdtest '-Y' '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' './out//2020.07.06-17.56.35-scr/mdt_hard' '-n' '130000' '-x' './out//2020.07.06-17.56.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /mnt/beeond/io500-app/out/2020.07.06-17.56.35-scr
FS: 989.4 TiB   Used FS: 21.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 12 for each phase.
2040 tasks, 265200000 files
Continue stonewall hit min: 393 max: 520 avg: 448.1 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       2424.582       2424.577       2424.579          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA       3038.109             NA
   Tree creation             :        153.806        153.806        153.806          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        437.520        437.519        437.519          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.876             NA
   Tree creation             :          0.007          0.007          0.007          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/06/2020 18:39:54 --

result_summary
[RESULT] BW   phase 1            ior_easy_write              333.854 GiB/s : time 458.30 seconds
[RESULT] IOPS phase 1         mdtest_easy_write             1913.300 kiops : time 663.54 seconds
[RESULT] BW   phase 2            ior_hard_write               35.716 GiB/s : time 440.10 seconds
[RESULT] IOPS phase 2         mdtest_hard_write                2.425 kiops : time 437.52 seconds
[RESULT] IOPS phase 3                      find             6537.040 kiops : time 194.37 seconds
[RESULT] BW   phase 3             ior_easy_read              422.200 GiB/s : time 467.32 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             8261.120 kiops : time 153.68 seconds
[RESULT] BW   phase 4             ior_hard_read              187.343 GiB/s : time  86.21 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               25.897 kiops : time  40.96 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete             3602.590 kiops : time 352.40 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               18.244 kiops : time  58.15 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                8.338 kiops : time 132.45 seconds
[SCORE] Bandwidth 175.244 GiB/s : IOPS 208.382 kiops : TOTAL 191.096