AI400X

Institution DDN
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 8.1
Client Kernel Version 4.18.0-147.8.1.el8_1.x86_64

DATA SERVER

Storage Type NVMe
Volatile Memory 3.84TB
Storage Interface NVMe
Network IB-EDR
Software Version 2.13.54
OS Version CentOS7.7

INFORMATION

Client Nodes 40
Client Total Procs 1,280
Metadata Nodes 0
Metadata Storage Devices 0
Data Nodes 4
Data Storage Devices 10

METADATA

Easy Write 378.84 kIOP/s
Easy Stat 1,069.15 kIOP/s
Easy Delete 237.28 kIOP/s
Hard Write 131.45 kIOP/s
Hard Read 350.85 kIOP/s
Hard Stat 1,108.62 kIOP/s
Hard Delete 99.59 kIOP/s

Submitted Files

io500
#!/bin/bash
#SBATCH --job-name=io500-ai400
#SBATCH -P 32n
#SBATCH --nodes=10
#SBATCH --ntasks-per-node=16
#SBATCH -o io_500_out_%j
#SBATCH -e io_500_err_%J
#SBATCH --dependency=singleton

LUSTRE_MGS=10.0.11.224@o2ib10
LUSTRE_MDS=es400nv-vm[1-4],es400xmd-vm[1-4]
LUSTRE_OSS=es400nv-vm[1-4],es400xmd-vm[1-4]
#LUSTRE_MDS=es400nv-vm[1-4]
#LUSTRE_OSS=es400nv-vm[1-4]
#LUSTRE_CLIENT=${SLURM_JOB_NODELIST}
LUSTRE_CLIENT=ec[01-40]
FSNAME=/ai400
MNT=/ai400_0

ROOT=`pwd`
#module purge
#module load mpi/gcc/openmpi/4.0.4
export PATH=/usr/mpi/gcc/openmpi-4.0.3rc4/bin:$PATH
PDSH="pdsh"

# Lustre MDS/OSS Setting
$PDSH -w ${LUSTRE_MDS} "echo 128 > /sys/module/mdt/parameters/max_mod_rpcs_per_client"
$PDSH -w ${LUSTRE_OSS},${LUSTRE_MDS} "sysctl -w vm.min_free_kbytes=524288"
$PDSH -w ${LUSTRE_OSS} lctl set_param \
osd-ldiskfs.*.read_cache_enable=0 \
osd-ldiskfs.*.writethrough_cache_enable=0 \
obdfilter.*.brw_size=16 \
obdfilter.*.precreate_batch=1024

# ReMount Lustre Client
#$PDSH -w ${LUSTRE_CLIENT} umount -t lustre -a
#$PDSH -w ${LUSTRE_CLIENT} mount -t lustre ${LUSTRE_MGS}:${FSNAME} ${MNT}
$PDSH -w ${LUSTRE_CLIENT},${LUSTRE_MDS},${LUSTRE_OSS} lctl get_param version
sleep 2

# Lustre Client Setting
$PDSH -w ${LUSTRE_CLIENT} lctl set_param \
osc.*.max_pages_per_rpc=16M \
osc.*.max_rpcs_in_flight=16 \
osc.*.max_dirty_mb=512 \
osc.*.checksums=0 \
osc.*.short_io_bytes=65536 \
llite.*.max_read_ahead_mb=2048 \
llite.*.max_read_ahead_per_file_mb=16 \
llite.*.max_cached_mb=8192 \
ldlm.namespaces.*.lru_size=0 \
ldlm.namespaces.*.lru_max_age=5000 \
mdc.*.max_rpcs_in_flight=128 \
mdc.*.max_mod_rpcs_in_flight=127
sleep 2

# Cleanup & TRIM to all OSTs
$PDSH -w ${LUSTRE_CLIENT} lctl set_param ldlm.namespaces.*.lru_size=clear
#$PDSH -w ${LUSTRE_OSS} fstrim -av
$PDSH -w ${LUSTRE_MDS},${LUSTRE_OSS} "echo 3 > /proc/sys/vm/drop_caches"
$PDSH -w ${LUSTRE_CLIENT} "cpupower frequency-set -g performance"

#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  #io500_mpirun="/work/tools/mpi/gcc/openmpi/4.0.4/bin/mpirun"
  io500_mpirun="mpirun"
  #io500_mpiargs="--bind-to core --map-by node --allow-run-as-root -np 160 -hostfile ./hostfile -npernode 16 singularity.sh --bind /work -B /usr/mpi -B /usr/lib64 -B /sys/class/infiniband_verbs -B /bin -B /sbin -B /etc/libibverbs.d centos8.sif "
  #io500_mpiargs="--bind-to core --map-by node --allow-run-as-root -np 640 -hostfile ./hostfile -npernode 16 singularity.sh --bind /work -B /usr/mpi -B /usr/lib64 -B /sys/class/infiniband_verbs -B /bin -B /sbin -B /etc/libibverbs.d centos8.sif "
  io500_mpiargs=" --oversubscribe --bind-to core:overload-allowed --map-by core --allow-run-as-root -np 1280 -hostfile ./hostfile -npernode 32"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}

  lfs setdirstripe -c 16 -i 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 $workdir-app/mdtest-{easy,hard}
  lfs setdirstripe -c 16 -i 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 $workdir-scr/mdt_{easy,hard}
  #lfs setdirstripe -c 8 -i 0,1,2,3,4,5,6,7 $workdir-app/mdtest-{easy,hard}
  #lfs setdirstripe -c 8 -i 0,1,2,3,4,5,6,7 $workdir-scr/mdt_{easy,hard}
  #lfs setdirstripe -c 4 -i 0,1,2,3 $workdir-app/mdtest-{easy,hard}
  #lfs setdirstripe -c 4 -i 0,1,2,3 $workdir-scr/mdt_{easy,hard}
  lfs setdirstripe -c 16 -D $workdir-app/mdtest-{easy,hard}
  lfs setdirstripe -c 16 -D $workdir-scr/mdt_{easy,hard}
  lfs setstripe -L mdt -E 1M $workdir-app/mdtest-{easy,hard} 
  lfs setstripe -L mdt -E 1M $workdir-scr/mdt_{easy,hard}

  mkdir $workdir-app/ior-hard $workdir-scr/ior_hard
  lfs setstripe -C 640 -S 16m $workdir-app/ior-hard $workdir-scr/ior_hard
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jul 14 15:20:44 2020
Command line        : /work/BMLab/Lustre/io500_isc20/io500-app/bin/ior -r -R -a POSIX -t 16m --posix.odirect -b 70000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/stonewall
Machine             : Linux ec01
TestID              : 0
StartTime           : Tue Jul 14 15:20:44 2020
Path                : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy
FS                  : 94.9 TiB   Used FS: 33.1%   Inodes: 926.5 Mi   Used Inodes: 17.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 40
tasks               : 1280
clients per node    : 32
repetitions         : 1
xfersize            : 16 MiB
blocksize           : 68.36 GiB
aggregate filesize  : 85.45 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 93952409600000.
WARNING: Stat() of aggregate file size      = 31159987732480.
WARNING: Using actual aggregate bytes moved = 31159987732480.
read      91027      5690       0.074611    71680000   16384      0.026875   326.42     0.010509   326.46     0   
Max Read:  91026.62 MiB/sec (95448.33 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        91026.62   91026.62   91026.62       0.00    5689.16    5689.16    5689.16       0.00  326.45923         NA            NA     0   1280  32    1   1     1        1         0    0      1 73400320000 16777216 29716480.0 POSIX      0
Finished            : Tue Jul 14 15:26:11 2020
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jul 14 14:55:48 2020
Command line        : /work/BMLab/Lustre/io500_isc20/io500-app/bin/ior -w -a POSIX -t 16m --posix.odirect -b 70000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux ec01
TestID              : 0
StartTime           : Tue Jul 14 14:55:48 2020
Path                : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy
FS                  : 94.9 TiB   Used FS: 0.0%   Inodes: 768.0 Mi   Used Inodes: 0.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 40
tasks               : 1280
clients per node    : 32
repetitions         : 1
xfersize            : 16 MiB
blocksize           : 68.36 GiB
aggregate filesize  : 85.45 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 884 max: 1451 -- min data: 13.8 GiB mean data: 17.5 GiB time: 300.2s
WARNING: Expected aggregate file size       = 93952409600000.
WARNING: Stat() of aggregate file size      = 31159987732480.
WARNING: Using actual aggregate bytes moved = 31159987732480.
WARNING: maybe caused by deadlineForStonewalling
write     75334      4710       0.088906    71680000   16384      0.093032   394.35     0.020854   394.46     0   
Max Write: 75333.76 MiB/sec (78993.18 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       75333.76   75333.76   75333.76       0.00    4708.36    4708.36    4708.36       0.00  394.46430     300.23      76421.16     0   1280  32    1   1     1        1         0    0      1 73400320000 16777216 29716480.0 POSIX      0
Finished            : Tue Jul 14 15:02:22 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jul 14 15:28:12 2020
Command line        : /work/BMLab/Lustre/io500_isc20/io500-app/bin/ior -r -R -s 55000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/stonewall
Machine             : Linux ec01
TestID              : 0
StartTime           : Tue Jul 14 15:28:12 2020
Path                : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard
FS                  : 94.9 TiB   Used FS: 33.1%   Inodes: 926.5 Mi   Used Inodes: 17.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 55000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 40
tasks               : 1280
clients per node    : 32
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 3.01 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
read      73969      1650594    42.65       45.91      45.91      0.007877   42.65      0.008210   42.67      0   
Max Read:  73968.78 MiB/sec (77561.89 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        73968.78   73968.78   73968.78       0.00 1649972.06 1649972.06 1649972.06       0.00   42.66739         NA            NA     0   1280  32    1   0     1        1         0    0  55000    47008    47008 3156054.8 POSIX      0
Finished            : Tue Jul 14 15:28:55 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Tue Jul 14 15:07:51 2020
Command line        : /work/BMLab/Lustre/io500_isc20/io500-app/bin/ior -w -s 55000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux ec01
TestID              : 0
StartTime           : Tue Jul 14 15:07:51 2020
Path                : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard
FS                  : 94.9 TiB   Used FS: 29.9%   Inodes: 884.0 Mi   Used Inodes: 13.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 55000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 40
tasks               : 1280
clients per node    : 32
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 3.01 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 54600 max: 55000 -- min data: 2.4 GiB mean data: 2.4 GiB time: 300.1s
write     10438      232842     300.31      45.91      45.91      0.011040   302.35     0.011695   302.37     0   
Max Write: 10437.58 MiB/sec (10944.60 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       10437.58   10437.58   10437.58       0.00  232824.14  232824.14  232824.14       0.00  302.37415     300.06      10517.97     0   1280  32    1   0     1        1         0    0  55000    47008    47008 3156054.8 POSIX      0
Finished            : Tue Jul 14 15:12:55 2020
mdtest_easy_delete
-- started at 07/14/2020 15:29:43 --

mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s)
Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-r' '-F' '-P' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy' '-n' '95000' '-u' '-L' '-a' 'POSIX' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy-stonewall' '-N' '1'
Path: /ai400/datafiles/2020.07.14-14.55.35-scr
FS: 94.9 TiB   Used FS: 33.1%   Inodes: 926.5 Mi   Used Inodes: 17.1%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 32 for each phase.
1280 tasks, 121600000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     237276.720     237276.674     237276.700          0.010
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.694          0.694          0.694          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        512.482        512.482        512.482          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.442          1.442          1.442          0.000
-- finished at 07/14/2020 15:38:22 --

mdtest_easy_stat
-- started at 07/14/2020 15:26:15 --

mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s)
Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-T' '-F' '-P' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy' '-n' '95000' '-u' '-L' '-a' 'POSIX' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy-stonewall' '-N' '1'
Path: /ai400/datafiles/2020.07.14-14.55.35-scr
FS: 94.9 TiB   Used FS: 33.1%   Inodes: 926.5 Mi   Used Inodes: 17.1%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 32 for each phase.
1280 tasks, 121600000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    1069149.529    1069148.888    1069149.207          0.179
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        113.735        113.735        113.735          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/14/2020 15:28:08 --

mdtest_easy_write
-- started at 07/14/2020 15:02:26 --

mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s)
Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-Y' '-C' '-F' '-P' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy' '-n' '95000' '-u' '-L' '-a' 'POSIX' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /ai400/datafiles/2020.07.14-14.55.35-scr
FS: 94.9 TiB   Used FS: 29.9%   Inodes: 768.0 Mi   Used Inodes: 0.0%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 32 for each phase.
1280 tasks, 121600000 files
[1594706700.740772] [ec31:23151:0]         tcp_cm.c:604  UCX  WARN  tcp_iface 0x100e2e0: connection establishment for socket fd 58 from  to 10.128.11.31:44321 was unsuccessful
[1594706731.932133] [ec31:23151:1]           sock.c:344  UCX  ERROR recv(fd=55) failed: Connection reset by peer
[1594706731.932170] [ec31:23151:1]           sock.c:344  UCX  ERROR recv(fd=56) failed: Connection reset by peer
[1594706731.932180] [ec31:23151:1]           sock.c:344  UCX  ERROR recv(fd=57) failed: Connection reset by peer
[1594706731.932188] [ec31:23151:1]           sock.c:344  UCX  ERROR recv(fd=54) failed: Connection reset by peer
[1594706731.932196] [ec31:23151:1]           sock.c:344  UCX  ERROR recv(fd=58) failed: Connection reset by peer
[1594706820.116146] [ec31:23151:1]           sock.c:344  UCX  ERROR recv(fd=54) failed: Connection reset by peer
[1594706830.770108] [ec31:23151:1]           sock.c:344  UCX  ERROR recv(fd=54) failed: Connection reset by peer
Continue stonewall hit min: 85758 max: 95000 avg: 94184.3 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     378835.691     378835.480     378835.575          0.038
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     402457.892             NA
   Tree creation             :          2.333          2.333          2.333          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        320.984        320.983        320.984          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.549             NA
   Tree creation             :          0.429          0.429          0.429          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/14/2020 15:07:48 --

mdtest_hard_delete
-- started at 07/14/2020 15:40:39 --

mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s)
Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard' '-n' '200000' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /ai400/datafiles/2020.07.14-14.55.35-scr
FS: 94.9 TiB   Used FS: 33.1%   Inodes: 810.5 Mi   Used Inodes: 5.2%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 32 for each phase.
1280 tasks, 256000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      99585.845      99585.815      99585.828          0.007
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.218          0.218          0.218          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        447.408        447.408        447.408          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          4.588          4.588          4.588          0.000
-- finished at 07/14/2020 15:48:11 --

mdtest_hard_read
-- started at 07/14/2020 15:38:28 --

mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s)
Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard' '-n' '200000' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /ai400/datafiles/2020.07.14-14.55.35-scr
FS: 94.9 TiB   Used FS: 33.1%   Inodes: 810.5 Mi   Used Inodes: 5.2%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 32 for each phase.
1280 tasks, 256000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     350849.430     350828.488     350849.251          0.587
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        127.001        126.993        126.993          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/14/2020 15:40:35 --

mdtest_hard_stat
-- started at 07/14/2020 15:28:59 --

mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s)
Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard' '-n' '200000' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /ai400/datafiles/2020.07.14-14.55.35-scr
FS: 94.9 TiB   Used FS: 33.1%   Inodes: 926.5 Mi   Used Inodes: 17.1%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 32 for each phase.
1280 tasks, 256000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    1108623.408    1108622.020    1108622.769          0.336
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         40.190         40.190         40.190          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/14/2020 15:29:39 --

mdtest_hard_write
-- started at 07/14/2020 15:12:59 --

mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s)
Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-Y' '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard' '-n' '200000' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /ai400/datafiles/2020.07.14-14.55.35-scr
FS: 94.9 TiB   Used FS: 33.1%   Inodes: 884.0 Mi   Used Inodes: 13.1%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 32 for each phase.
1280 tasks, 256000000 files
Continue stonewall hit min: 30291 max: 34809 avg: 32711.1 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     131451.559     131451.518     131451.542          0.009
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     139575.822             NA
   Tree creation             :          9.931          9.931          9.931          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        338.950        338.950        338.950          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.982             NA
   Tree creation             :          0.101          0.101          0.101          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/14/2020 15:18:38 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               73.568 GiB/s : time 394.35 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              378.836 kiops : time 320.98 seconds
[RESULT] BW   phase 2            ior_hard_write               10.193 GiB/s : time 302.35 seconds
[RESULT] IOPS phase 2         mdtest_hard_write              131.452 kiops : time 338.95 seconds
[RESULT] IOPS phase 3                      find             1361.850 kiops : time 122.01 seconds
[RESULT] BW   phase 3             ior_easy_read               88.894 GiB/s : time 326.42 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             1069.150 kiops : time 113.74 seconds
[RESULT] BW   phase 4             ior_hard_read               72.235 GiB/s : time  42.65 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat             1108.620 kiops : time  40.19 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              237.277 kiops : time 512.48 seconds
[RESULT] IOPS phase 7          mdtest_hard_read              350.849 kiops : time 127.00 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               99.586 kiops : time 455.94 seconds
[SCORE] Bandwidth 46.8444 GiB/s : IOPS 400.837 kiops : TOTAL 137.029