MOGON II

Institution Johannes Gutenberg University Mainz
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version
Client Kernel Version 3.10.0-1127.8.2.el7.x86_64

DATA SERVER

Storage Type Intel SSD DC S4600 Series
Volatile Memory
Storage Interface SATA
Network Omnipath
Software Version 0.9dev
OS Version

INFORMATION

Client Nodes 10
Client Total Procs 240
Metadata Nodes 224
Metadata Storage Devices 1
Data Nodes 224
Data Storage Devices 1

METADATA

Easy Write 2,019.87 kIOP/s
Easy Stat 1,970.75 kIOP/s
Easy Delete 2,082.08 kIOP/s
Hard Write 426.96 kIOP/s
Hard Read 419.00 kIOP/s
Hard Stat 2,030.35 kIOP/s
Hard Delete 377.66 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.
#
# The only parts of the script that may need to be modified are:
#  - setup_paths() to configure the binary locations and MPI parameters
#  - setup_directories() to create/tune the IOR/mdtest output directories
#
# Please visit https://vi4io.org/io500-info-creator/ to help generate the
# "system-information.txt" file, by pasting the output of the info-creator.
# This file contains details of your system hardware for your submission.

# Set the paths to the binaries and how to launch MPI jobs.
# If you ran ./prepare.sh successfully, then binaries are in ./bin/
function setup_paths {
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mpirun="mpiexec"
  io500_mpiargs="-np 240 --oversubscribe --map-by node --hostfile /home/vef/hostfile -x LIBGKFS_LOG_OUTPUT=/dev/shm/gkfs_client.log -x LIBGKFS_LOG=info -x LD_PRELOAD=/lustre/miifs01/project/m2_zdvresearch/vef/sshfs/gekkofs/build/src/client/libgkfs_intercept.so -x LIBGKFS_HOSTS_FILE=/lustre/miifs01/project/m2_zdvresearch/vef/gkfs_hostfile -x FI_PSM2_DISCONNECT=1 -x PSM2_MULTI_EP=1"
  #export PATH=/lustre/miifs01/project/m2_zdvresearch/vef/io500_bins:${PATH}
export FI_PSM2_DISCONNECT=1
}

# Set directories where benchmark files are created and where the results go.
# If you want to set up stripe tuning on your output directories or anything
# similar, then this is the right place to do it.
function setup_directories {
  local workdir
  local resultdir
  local ts

  mkdir -p $io500_workdir $io500_resultdir

  # Example commands to create output directories for Lustre.  Creating
  # top-level directories is allowed, but not the whole directory tree.
  #if (( $(lfs df $io500_workdir | grep -c MDT) > 1 )); then
  #  lfs setdirstripe -D -c -1 $io500_workdir
  #fi
  #lfs setstripe -c 1 $io500_workdir
  #mkdir $io500_workdir/ior-easy $io500_workdir/ior-hard
  #mkdir $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard
  #local osts=$(lfs df $io500_workdir | grep -c OST)
  # Try overstriping for ior-hard to improve scaling, or use wide striping
  #lfs setstripe -C $((osts * 4)) $io500_workdir/ior-hard ||
  #  lfs setstripe -c -1 $io500_workdir/ior-hard
  # Try to use DoM if available, otherwise use default for small files
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-easy || true #DoM?
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-hard || true #DoM?

  LIBGKFS_HOSTS_FILE=/lustre/miifs01/project/m2_zdvresearch/vef/gkfs_hostfile LD_PRELOAD=/lustre/miifs01/project/m2_zdvresearch/vef/sshfs/gekkofs/build/src/client/libgkfs_intercept.so mkdir -p $io500_workdir-{scr,app}/{ior_easy,ior_hard,mdtest_easy,mdtest_hard} $io500_resultdir-{scr,app}

  mkdir -p $io500_workdir-{scr,app} $io500_resultdir-{scr,app}
}

# *****  YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE  *****
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

function run_benchmarks {
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_resultdir)
  local fname=$(basename ${io500_resultdir})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_resultdir
  tar czf $tarball -C $sourcedir $fname
  echo "Created result tarball $tarball"
}

function main {
  # These commands extract the 'datadir' and 'resultdir' from .ini file
  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # working directory where the test files will be created
  export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts

  setup_directories
  setup_paths
  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a 'system-information.txt' description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_resultdir
  fi

  create_tarball
}

main
ior-easy-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Oct 31 03:04:27 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /dev/shm/vef_mountdir/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.31-02.38.19/ior-easy.stonewall -t 64m -b 9920000m -F -r -R -a POSIX
Machine             : Linux x0004.mogon
TestID              : 0
StartTime           : Sat Oct 31 03:04:27 2020
Path                : /dev/shm/vef_mountdir/ior-easy/ior_file_easy.00000000
FS                  : 93.2 TiB   Used FS: 55.2%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /dev/shm/vef_mountdir/ior-easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 240
clients per node    : 24
repetitions         : 1
xfersize            : 64 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 2270.51 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 2496449740800000
WARNING: Stat() of aggregate file size      = 14833743298560
WARNING: Using actual aggregate bytes moved = 14833743298560
read      27542      430.35     0.003314    10158080000 65536      0.009583   513.63     0.000814   513.63     0   
ior-easy-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Oct 31 02:38:23 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /dev/shm/vef_mountdir/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.31-02.38.19/ior-easy.stonewall -t 64m -b 9920000m -F -w -D 300 -O stoneWallingWearOut=1 -a POSIX
Machine             : Linux x0004.mogon
TestID              : 0
StartTime           : Sat Oct 31 02:38:23 2020
Path                : /dev/shm/vef_mountdir/ior-easy/ior_file_easy.00000000
FS                  : 93.2 TiB   Used FS: 34.2%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /dev/shm/vef_mountdir/ior-easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 240
clients per node    : 24
repetitions         : 1
xfersize            : 64 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 2270.51 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 804 max: 921 -- min data: 50.2 GiB mean data: 53.7 GiB time: 300.4s
WARNING: Expected aggregate file size       = 2496449740800000
WARNING: Stat() of aggregate file size      = 14833743298560
WARNING: Using actual aggregate bytes moved = 14833743298560
WARNING: Maybe caused by deadlineForStonewalling
write     43333      677.29     0.002072    10158080000 65536      0.010866   326.36     0.087819   326.46     0   
ior-hard-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Oct 31 03:19:37 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /dev/shm/vef_mountdir/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.31-02.38.19/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a POSIX
Machine             : Linux x0004.mogon
TestID              : 0
StartTime           : Sat Oct 31 03:19:37 2020
Path                : /dev/shm/vef_mountdir/ior-hard/file
FS                  : 93.2 TiB   Used FS: 55.2%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /dev/shm/vef_mountdir/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 240
clients per node    : 24
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 102.61 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 529811 max: 529811 -- min data: 23.2 GiB mean data: 23.2 GiB time: 399.4s
WARNING: Expected aggregate file size       = 112819200000000
WARNING: Stat() of aggregate file size      = 5977285317120
WARNING: Using actual aggregate bytes moved = 5977285317120
read      14270      318318     399.46      45.91      45.91      0.009960   399.46     0.000844   399.47     0   
ior-hard-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Oct 31 02:50:19 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /dev/shm/vef_mountdir/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.31-02.38.19/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a POSIX
Machine             : Linux x0004.mogon
TestID              : 0
StartTime           : Sat Oct 31 02:50:19 2020
Path                : /dev/shm/vef_mountdir/ior-hard/file
FS                  : 93.2 TiB   Used FS: 48.7%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /dev/shm/vef_mountdir/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 240
clients per node    : 24
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 102.61 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 492377 max: 529811 -- min data: 21.6 GiB mean data: 22.5 GiB time: 300.1s
WARNING: Expected aggregate file size       = 112819200000000
WARNING: Stat() of aggregate file size      = 5977285317120
WARNING: Using actual aggregate bytes moved = 5977285317120
WARNING: Maybe caused by deadlineForStonewalling
write     17962      400705     308.78      45.91      45.91      0.013815   317.33     0.009077   317.35     0   
mdtest-easy-delete
-- started at 10/31/2020 03:27:24 --

mdtest-3.4.0+dev was launched with 240 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '5000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/dev/shm/vef_mountdir/mdtest-easy' '-x' './results/2020.10.31-02.38.19/mdtest-easy.stonewall' '-r' '-a' 'POSIX'
Path: /dev/shm/vef_mountdir
FS: 46.5 GiB   Used FS: 0.3%   Inodes: 11.6 Mi   Used Inodes: 0.0%

Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
240 tasks, 1200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    2082084.716    2082081.307    2082084.371          0.251
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.036          0.036          0.036          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        374.681        374.680        374.680          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         28.136         28.136         28.136          0.000
-- finished at 10/31/2020 03:34:07 --

mdtest-easy-stat
-- started at 10/31/2020 03:13:01 --

mdtest-3.4.0+dev was launched with 240 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '5000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/dev/shm/vef_mountdir/mdtest-easy' '-x' './results/2020.10.31-02.38.19/mdtest-easy.stonewall' '-T' '-a' 'POSIX'
Path: /dev/shm/vef_mountdir
FS: 46.5 GiB   Used FS: 0.3%   Inodes: 11.6 Mi   Used Inodes: 0.0%

Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
240 tasks, 1200000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    1970751.626    1970751.080    1970751.386          0.096
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        395.847        395.847        395.847          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/31/2020 03:19:37 --

mdtest-easy-write
-- started at 10/31/2020 02:43:49 --

mdtest-3.4.0+dev was launched with 240 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '5000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/dev/shm/vef_mountdir/mdtest-easy' '-x' './results/2020.10.31-02.38.19/mdtest-easy.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /dev/shm/vef_mountdir
FS: 46.5 GiB   Used FS: 0.3%   Inodes: 11.6 Mi   Used Inodes: 0.0%

Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
240 tasks, 1200000000 files
Continue stonewall hit min: 2709353 max: 3250482 avg: 3007554.9 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    2019868.422    2019865.546    2019867.428          0.692
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    2405971.085             NA
   Tree creation             :         41.456         41.456         41.456          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        386.222        386.221        386.221          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.009             NA
   Tree creation             :          0.024          0.024          0.024          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/31/2020 02:50:19 --

mdtest-hard-delete
-- started at 10/31/2020 03:39:33 --

mdtest-3.4.0+dev was launched with 240 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/dev/shm/vef_mountdir/mdtest-hard' '-x' './results/2020.10.31-02.38.19/mdtest-hard.stonewall' '-r' '-a' 'POSIX'
Path: /dev/shm/vef_mountdir
FS: 46.5 GiB   Used FS: 0.3%   Inodes: 11.6 Mi   Used Inodes: 0.0%

Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
240 tasks, 240000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     377660.255     377660.094     377660.174          0.035
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.430          3.430          3.430          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        361.976        361.976        361.976          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.292          0.292          0.292          0.000
-- finished at 10/31/2020 03:45:36 --

mdtest-hard-read
-- started at 10/31/2020 03:34:07 --

mdtest-3.4.0+dev was launched with 240 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/dev/shm/vef_mountdir/mdtest-hard' '-x' './results/2020.10.31-02.38.19/mdtest-hard.stonewall' '-E' '-X' '-a' 'POSIX'
Path: /dev/shm/vef_mountdir
FS: 46.5 GiB   Used FS: 0.3%   Inodes: 11.6 Mi   Used Inodes: 0.0%

Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
240 tasks, 240000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     418998.457     418998.269     418998.381          0.046
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        326.263        326.263        326.263          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/31/2020 03:39:33 --

mdtest-hard-stat
-- started at 10/31/2020 03:26:16 --

mdtest-3.4.0+dev was launched with 240 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/dev/shm/vef_mountdir/mdtest-hard' '-x' './results/2020.10.31-02.38.19/mdtest-hard.stonewall' '-T' '-a' 'POSIX'
Path: /dev/shm/vef_mountdir
FS: 46.5 GiB   Used FS: 0.3%   Inodes: 11.6 Mi   Used Inodes: 0.0%

Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
240 tasks, 240000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    2030349.456    2030347.506    2030348.730          0.461
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         67.330         67.330         67.330          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/31/2020 03:27:23 --

mdtest-hard-write
-- started at 10/31/2020 02:55:36 --

mdtest-3.4.0+dev was launched with 240 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/dev/shm/vef_mountdir/mdtest-hard' '-x' './results/2020.10.31-02.38.19/mdtest-hard.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /dev/shm/vef_mountdir
FS: 46.5 GiB   Used FS: 0.3%   Inodes: 11.6 Mi   Used Inodes: 0.0%

Nodemap: 100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
240 tasks, 240000000 files
Continue stonewall hit min: 545236 max: 569599 avg: 557410.5 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     426962.101     426961.799     426961.971          0.066
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     445910.675             NA
   Tree creation             :       2110.413       2110.413       2110.413          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        320.178        320.178        320.178          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.012             NA
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/31/2020 03:00:59 --

result
version         = io500-sc20
config-hash     = 875215F5
result-dir      = ./results/2020.10.31-02.38.19
; START 2020-10-31 02:38:23


[ior-easy-write]
t_start         = 2020-10-31 02:38:23
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /dev/shm/vef_mountdir/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.31-02.38.19/ior-easy.stonewall -t 64m -b 9920000m -F -w -D 300 -O stoneWallingWearOut=1 -a POSIX
throughput-stonewall = 9476.51
score           = 42.317550
t_delta         = 326.6049
t_end           = 2020-10-31 02:43:49

[mdtest-easy-write]
t_start         = 2020-10-31 02:43:49
exe             = ./mdtest -n 5000000 -u -L -F -P -N 1 -d /dev/shm/vef_mountdir/mdtest-easy -x ./results/2020.10.31-02.38.19/mdtest-easy.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 2405.971085
score           = 2019.867908
t_delta         = 389.3933
t_end           = 2020-10-31 02:50:19

[timestamp]
t_start         = 2020-10-31 02:50:19
t_delta         = 0.0007
t_end           = 2020-10-31 02:50:19

[ior-hard-write]
t_start         = 2020-10-31 02:50:19
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /dev/shm/vef_mountdir/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.31-02.38.19/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a POSIX
throughput-stonewall = 4086.92
score           = 17.541483
t_delta         = 317.3685
t_end           = 2020-10-31 02:55:36

[mdtest-hard-write]
t_start         = 2020-10-31 02:55:36
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /dev/shm/vef_mountdir/mdtest-hard -x ./results/2020.10.31-02.38.19/mdtest-hard.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 445.910675
score           = 426.961931
t_delta         = 323.5172
t_end           = 2020-10-31 03:00:59

[find]
t_start         = 2020-10-31 03:00:59
exe             = ./pfind /dev/shm/vef_mountdir -newer ./results/2020.10.31-02.38.19/timestampfile -size 3901c -name *01* -C -q 10000
nproc           = 40
found           = 6334500
total-files     = 916819681
score           = 4419.430753
t_delta         = 207.5011
t_end           = 2020-10-31 03:04:27

[ior-easy-read]
t_start         = 2020-10-31 03:04:27
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /dev/shm/vef_mountdir/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.31-02.38.19/ior-easy.stonewall -t 64m -b 9920000m -F -r -R -a POSIX
score           = 26.896545
t_delta         = 513.7326
t_end           = 2020-10-31 03:13:01

[mdtest-easy-stat]
t_start         = 2020-10-31 03:13:01
exe             = ./mdtest -n 5000000 -u -L -F -P -N 1 -d /dev/shm/vef_mountdir/mdtest-easy -x ./results/2020.10.31-02.38.19/mdtest-easy.stonewall -T -a POSIX
score           = 1970.751491
t_delta         = 395.8666
t_end           = 2020-10-31 03:19:37

[ior-hard-read]
t_start         = 2020-10-31 03:19:37
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /dev/shm/vef_mountdir/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.31-02.38.19/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a POSIX
score           = 13.935514
t_delta         = 399.6216
t_end           = 2020-10-31 03:26:16

[mdtest-hard-stat]
t_start         = 2020-10-31 03:26:16
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /dev/shm/vef_mountdir/mdtest-hard -x ./results/2020.10.31-02.38.19/mdtest-hard.stonewall -T -a POSIX
score           = 2030.348575
t_delta         = 67.3377
t_end           = 2020-10-31 03:27:23

[mdtest-easy-delete]
t_start         = 2020-10-31 03:27:23
exe             = ./mdtest -n 5000000 -u -L -F -P -N 1 -d /dev/shm/vef_mountdir/mdtest-easy -x ./results/2020.10.31-02.38.19/mdtest-easy.stonewall -r -a POSIX
score           = 2082.084418
t_delta         = 403.6912
t_end           = 2020-10-31 03:34:07

[mdtest-hard-read]
t_start         = 2020-10-31 03:34:07
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /dev/shm/vef_mountdir/mdtest-hard -x ./results/2020.10.31-02.38.19/mdtest-hard.stonewall -E -X -a POSIX
score           = 418.998388
t_delta         = 326.2702
t_end           = 2020-10-31 03:39:33

[mdtest-hard-delete]
t_start         = 2020-10-31 03:39:33
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /dev/shm/vef_mountdir/mdtest-hard -x ./results/2020.10.31-02.38.19/mdtest-hard.stonewall -r -a POSIX
score           = 377.660185
t_delta         = 362.5656
t_end           = 2020-10-31 03:45:36

[SCORE]
MD              = 1223.591923
BW              = 22.966858
SCORE           = 167.636695 
hash            = 95A0F404
; END 2020-10-31 03:46:58
result_summary
IO500 version io500-sc20
[RESULT]       ior-easy-write       42.317550 GiB/s : time 326.605 seconds
[RESULT]    mdtest-easy-write     2019.867908 kIOPS : time 389.393 seconds
[RESULT]       ior-hard-write       17.541483 GiB/s : time 317.369 seconds
[RESULT]    mdtest-hard-write      426.961931 kIOPS : time 323.517 seconds
[RESULT]                 find     4419.430753 kIOPS : time 207.501 seconds
[RESULT]        ior-easy-read       26.896545 GiB/s : time 513.733 seconds
[RESULT]     mdtest-easy-stat     1970.751491 kIOPS : time 395.867 seconds
[RESULT]        ior-hard-read       13.935514 GiB/s : time 399.622 seconds
[RESULT]     mdtest-hard-stat     2030.348575 kIOPS : time 67.338 seconds
[RESULT]   mdtest-easy-delete     2082.084418 kIOPS : time 403.691 seconds
[RESULT]     mdtest-hard-read      418.998388 kIOPS : time 326.270 seconds
[RESULT]   mdtest-hard-delete      377.660185 kIOPS : time 362.566 seconds
[SCORE] Bandwidth 22.966858 GB/s : IOPS 1223.591923 kiops : TOTAL 167.636695