NextGENIO

Institution EPCC
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version
Client Kernel Version 3.10.0-862.14.4.el7.x86_64

DATA SERVER

Storage Type 3DXPNVMeSSD
Volatile Memory 2
Storage Interface NVMe
Network Omnipath
Software Version 0.9dev
OS Version

INFORMATION

Client Nodes 10
Client Total Procs 3,800
Metadata Nodes 31
Metadata Storage Devices 2
Data Nodes 31
Data Storage Devices 2

METADATA

Easy Write 1,709.39 kIOP/s
Easy Stat 1,926.37 kIOP/s
Easy Delete 1,844.14 kIOP/s
Hard Write 288.53 kIOP/s
Hard Read 583.34 kIOP/s
Hard Stat 1,721.46 kIOP/s
Hard Delete 691.37 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.
#
# The only parts of the script that may need to be modified are:
#  - setup_paths() to configure the binary locations and MPI parameters
#  - setup_directories() to create/tune the IOR/mdtest output directories
#
# Please visit https://vi4io.org/io500-info-creator/ to help generate the
# "system-information.txt" file, by pasting the output of the info-creator.
# This file contains details of your system hardware for your submission.

# Set the paths to the binaries and how to launch MPI jobs.
# If you ran ./prepare.sh successfully, then binaries are in ./bin/
function setup_paths {
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mpirun="srun"
  io500_mpiargs="-N 10 -n ${NPROCS_MPI} --export=ALL,LIBGKFS_LOG_OUTPUT_TRUNC=0,PSM2_DEVICES=self,hfi,shm,PSM2_MULTIRAIL=1,PSM2_MULTI_EP=0,LD_PRELOAD=${GKFS_PRLD}"
#  io500_mpirun="mpiexec"
#  io500_mpiargs="-np 2"
}

# Set directories where benchmark files are created and where the results go.
# If you want to set up stripe tuning on your output directories or anything
# similar, then this is the right place to do it.
function setup_directories {
  local workdir
  local resultdir
  local ts
  FI_PSM2_DISCONNECT=1 PSM2_MULTIRAIL=1 PSM2_MULTI_EP=0 LD_PRELOAD=$GKFS_PRLD mkdir -p $io500_workdir
  FI_PSM2_DISCONNECT=1 PSM2_MULTIRAIL=1 PSM2_MULTI_EP=0 LD_PRELOAD=$GKFS_PRLD mkdir -p $io500_workdir/ior-easy $io500_workdir/ior-hard
  FI_PSM2_DISCONNECT=1 PSM2_MULTIRAIL=1 PSM2_MULTI_EP=0 LD_PRELOAD=$GKFS_PRLD mkdir -p $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard

  mkdir -p $io500_workdir $io500_resultdir
  # Example commands to create output directories for Lustre.  Creating
  # top-level directories is allowed, but not the whole directory tree.
  #if (( $(lfs df $io500_workdir | grep -c MDT) > 1 )); then
  #  lfs setdirstripe -D -c -1 $io500_workdir
  #fi
  #lfs setstripe -c 1 $io500_workdir
  #mkdir $io500_workdir/ior-easy $io500_workdir/ior-hard
  #mkdir $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard
  #local osts=$(lfs df $io500_workdir | grep -c OST)
  # Try overstriping for ior-hard to improve scaling, or use wide striping
  #lfs setstripe -C $((osts * 4)) $io500_workdir/ior-hard ||
  #  lfs setstripe -c -1 $io500_workdir/ior-hard
  # Try to use DoM if available, otherwise use default for small files
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-easy || true #DoM?
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-hard || true #DoM?
}

# *****  YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE  *****
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

function run_benchmarks {
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_resultdir)
  local fname=$(basename ${io500_resultdir})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_resultdir
  tar czf $tarball -C $sourcedir $fname
  echo "Created result tarball $tarball"
}

function main {
  # These commands extract the 'datadir' and 'resultdir' from .ini file
  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # working directory where the test files will be created
  export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts

  setup_directories
  setup_paths
  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a 'system-information.txt' description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_resultdir
  fi

  create_tarball
}

main
ior-easy-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Oct 30 16:24:18 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/r/ior-easy/ior_file_easy -O stoneWallingStatusFile=results/ior-easy.stonewall -t 512kb -b 9920000m -F -r -R -a POSIX
Machine             : Linux nextgenio-cn01
TestID              : 0
StartTime           : Fri Oct 30 16:24:18 2020
Path                : /tmp/r/ior-easy/ior_file_easy.00000000
FS                  : 56.3 TiB   Used FS: 61.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/r/ior-easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 380
clients per node    : 38
repetitions         : 1
xfersize            : 524288 bytes
blocksize           : 9.46 TiB
aggregate filesize  : 3594.97 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 3952712089600000
WARNING: Stat() of aggregate file size      = 25005087784960
WARNING: Using actual aggregate bytes moved = 25005087784960
read      78248      156601     0.000015    10158080000 512.00     0.202380   304.55     0.008066   304.76     0   
ior-easy-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Oct 30 16:00:05 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/r/ior-easy/ior_file_easy -O stoneWallingStatusFile=results/ior-easy.stonewall -t 512kb -b 9920000m -F -w -D 300 -O stoneWallingWearOut=1 -a POSIX
Machine             : Linux nextgenio-cn01
TestID              : 0
StartTime           : Fri Oct 30 16:00:05 2020
Path                : /tmp/r/ior-easy/ior_file_easy.00000000
FS                  : 56.3 TiB   Used FS: 5.1%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/r/ior-easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 380
clients per node    : 38
repetitions         : 1
xfersize            : 524288 bytes
blocksize           : 9.46 TiB
aggregate filesize  : 3594.97 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 117662 max: 125509 -- min data: 57.5 GiB mean data: 58.9 GiB time: 300.1s
WARNING: Expected aggregate file size       = 3952712089600000
WARNING: Stat() of aggregate file size      = 25005087784960
WARNING: Using actual aggregate bytes moved = 25005087784960
WARNING: Maybe caused by deadlineForStonewalling
write     75428      150867     0.000016    10158080000 512.00     0.010941   316.13     0.012135   316.15     0   
ior-hard-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Oct 30 16:34:23 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/r/ior-hard/file -O stoneWallingStatusFile=results/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a POSIX
Machine             : Linux nextgenio-cn01
TestID              : 0
StartTime           : Fri Oct 30 16:34:23 2020
Path                : /tmp/r/ior-hard/file
FS                  : 56.3 TiB   Used FS: 61.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/r/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 380
clients per node    : 38
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 162.46 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 528872 max: 528872 -- min data: 23.2 GiB mean data: 23.2 GiB time: 302.0s
WARNING: Expected aggregate file size       = 178630400000000
WARNING: Stat() of aggregate file size      = 9447261690880
WARNING: Using actual aggregate bytes moved = 9447261690880
read      29800      665174     302.09      45.91      45.91      0.209843   302.13     0.010661   302.34     0   
ior-hard-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Oct 30 16:11:40 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/r/ior-hard/file -O stoneWallingStatusFile=results/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a POSIX
Machine             : Linux nextgenio-cn01
TestID              : 0
StartTime           : Fri Oct 30 16:11:40 2020
Path                : /tmp/r/ior-hard/file
FS                  : 56.3 TiB   Used FS: 45.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/r/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 380
clients per node    : 38
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 162.46 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 482381 max: 528872 -- min data: 21.1 GiB mean data: 21.7 GiB time: 300.1s
WARNING: Expected aggregate file size       = 178630400000000
WARNING: Stat() of aggregate file size      = 9447261690880
WARNING: Using actual aggregate bytes moved = 9447261690880
WARNING: Maybe caused by deadlineForStonewalling
write     27486      613874     317.26      45.91      45.91      0.206236   327.38     0.209802   327.79     0   
mdtest-easy-delete
-- started at 10/30/2020 16:40:02 --

mdtest-3.4.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '2200000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/r/mdtest-easy' '-x' 'results/mdtest-easy.stonewall' '-r' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 10.0%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 11111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 38 for each phase.
380 tasks, 836000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    1844147.873    1844131.861    1844140.584          3.329
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.424          0.424          0.424          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        313.853        313.851        313.852          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.358          2.358          2.358          0.000
-- finished at 10/30/2020 16:45:18 --

mdtest-easy-stat
-- started at 10/30/2020 16:29:22 --

mdtest-3.4.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '2200000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/r/mdtest-easy' '-x' 'results/mdtest-easy.stonewall' '-T' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 10.0%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 11111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 38 for each phase.
380 tasks, 836000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    1926377.676    1926361.143    1926370.215          3.480
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        300.456        300.453        300.455          0.001
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/30/2020 16:34:23 --

mdtest-easy-write
-- started at 10/30/2020 16:05:22 --

mdtest-3.4.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '2200000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/r/mdtest-easy' '-x' 'results/mdtest-easy.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 10.0%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 11111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 38 for each phase.
380 tasks, 836000000 files
Continue stonewall hit min: 1338616 max: 1523123 avg: 1448162.2 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    1709396.481    1709381.780    1709389.802          3.082
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    1834270.339             NA
   Tree creation             :         64.537         64.537         64.537          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        338.594        338.591        338.593          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.011             NA
   Tree creation             :          0.015          0.015          0.015          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/30/2020 16:11:40 --

mdtest-hard-delete
-- started at 10/30/2020 16:47:06 --

mdtest-3.4.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '165000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/r/mdtest-hard' '-x' 'results/mdtest-hard.stonewall' '-r' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 10.0%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 11111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 38 for each phase.
380 tasks, 62700000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     691372.095     691366.151     691369.383          1.249
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.527          2.527          2.527          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :         90.690         90.689         90.690          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.396          0.396          0.396          0.000
-- finished at 10/30/2020 16:48:37 --

mdtest-hard-read
-- started at 10/30/2020 16:45:18 --

mdtest-3.4.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '165000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/r/mdtest-hard' '-x' 'results/mdtest-hard.stonewall' '-E' '-X' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 10.0%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 11111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 38 for each phase.
380 tasks, 62700000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     583345.155     583340.142     583342.891          1.053
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        107.484        107.484        107.484          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/30/2020 16:47:06 --

mdtest-hard-stat
-- started at 10/30/2020 16:39:25 --

mdtest-3.4.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '165000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/r/mdtest-hard' '-x' 'results/mdtest-hard.stonewall' '-T' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 10.0%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 11111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 38 for each phase.
380 tasks, 62700000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    1721464.210    1721449.164    1721457.394          3.120
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         36.423         36.422         36.423          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/30/2020 16:40:02 --

mdtest-hard-write
-- started at 10/30/2020 16:17:08 --

mdtest-3.4.0+dev was launched with 380 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '165000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/r/mdtest-hard' '-x' 'results/mdtest-hard.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 10.0%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 11111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 38 for each phase.
380 tasks, 62700000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     288532.131     288529.660     288531.011          0.520
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     331253.163             NA
   Tree creation             :       1983.148       1983.148       1983.148          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        217.309        217.307        217.308          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        189.281             NA
   Tree creation             :          0.001          0.001          0.001          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/30/2020 16:22:08 --

result
version         = io500-sc20_v3
config-hash     = A0C0BC36
result-dir      = results
; START 2020-10-30 16:00:05


[ior-easy-write]
t_start         = 2020-10-30 16:00:05
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/r/ior-easy/ior_file_easy -O stoneWallingStatusFile=results/ior-easy.stonewall -t 512kb -b 9920000m -F -w -D 300 -O stoneWallingWearOut=1 -a POSIX
throughput-stonewall = 26883.15
score           = 73.660290
t_delta         = 317.2223
t_end           = 2020-10-30 16:05:22

[mdtest-easy-write]
t_start         = 2020-10-30 16:05:22
exe             = ./mdtest -n 2200000 -u -L -F -P -N 1 -d /tmp/r/mdtest-easy -x results/mdtest-easy.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 1834.270339
score           = 1709.389499
t_delta         = 377.6713
t_end           = 2020-10-30 16:11:40

[timestamp]
t_start         = 2020-10-30 16:11:40
t_delta         = 0.0007
t_end           = 2020-10-30 16:11:40

[ior-hard-write]
t_start         = 2020-10-30 16:11:40
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/r/ior-hard/file -O stoneWallingStatusFile=results/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a POSIX
throughput-stonewall = 9573.20
score           = 26.841359
t_delta         = 328.0219
t_end           = 2020-10-30 16:17:08

[mdtest-hard-write]
t_start         = 2020-10-30 16:17:08
exe             = ./mdtest -n 165000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/r/mdtest-hard -x results/mdtest-hard.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 331.253163
score           = 288.530970
t_delta         = 300.1474
t_end           = 2020-10-30 16:22:08

[find]
t_start         = 2020-10-30 16:22:08
exe             = ./pfind /tmp/r -newer results/timestampfile -size 3901c -name *01* -C -N -H 1 -q 100000
nproc           = 80
found           = 2701581
total-files     = 641487121
score           = 4941.520633
t_delta         = 129.8230
t_end           = 2020-10-30 16:24:18

[ior-easy-read]
t_start         = 2020-10-30 16:24:18
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/r/ior-easy/ior_file_easy -O stoneWallingStatusFile=results/ior-easy.stonewall -t 512kb -b 9920000m -F -r -R -a POSIX
score           = 76.414370
t_delta         = 304.7856
t_end           = 2020-10-30 16:29:22

[mdtest-easy-stat]
t_start         = 2020-10-30 16:29:22
exe             = ./mdtest -n 2200000 -u -L -F -P -N 1 -d /tmp/r/mdtest-easy -x results/mdtest-easy.stonewall -T -a POSIX
score           = 1926.370017
t_delta         = 300.4656
t_end           = 2020-10-30 16:34:23

[ior-hard-read]
t_start         = 2020-10-30 16:34:23
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/r/ior-hard/file -O stoneWallingStatusFile=results/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a POSIX
score           = 29.101179
t_delta         = 302.5582
t_end           = 2020-10-30 16:39:25

[mdtest-hard-stat]
t_start         = 2020-10-30 16:39:25
exe             = ./mdtest -n 165000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/r/mdtest-hard -x results/mdtest-hard.stonewall -T -a POSIX
score           = 1721.456790
t_delta         = 36.4334
t_end           = 2020-10-30 16:40:02

[mdtest-easy-delete]
t_start         = 2020-10-30 16:40:02
exe             = ./mdtest -n 2200000 -u -L -F -P -N 1 -d /tmp/r/mdtest-easy -x results/mdtest-easy.stonewall -r -a POSIX
score           = 1844.140322
t_delta         = 316.4252
t_end           = 2020-10-30 16:45:18

[mdtest-hard-read]
t_start         = 2020-10-30 16:45:18
exe             = ./mdtest -n 165000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/r/mdtest-hard -x results/mdtest-hard.stonewall -E -X -a POSIX
score           = 583.342793
t_delta         = 107.4961
t_end           = 2020-10-30 16:47:06

[mdtest-hard-delete]
t_start         = 2020-10-30 16:47:06
exe             = ./mdtest -n 165000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/r/mdtest-hard -x results/mdtest-hard.stonewall -r -a POSIX
score           = 691.369215
t_delta         = 91.4932
t_end           = 2020-10-30 16:48:37

[SCORE]
MD              = 1251.323842
BW              = 45.791080
SCORE           = 239.373077 
hash            = 5D23E7D6
; END 2020-10-30 16:49:35
result_summary
IO500 version io500-sc20_v3
[RESULT]       ior-easy-write       73.660290 GiB/s : time 317.222 seconds
[RESULT]    mdtest-easy-write     1709.389499 kIOPS : time 377.671 seconds
[RESULT]       ior-hard-write       26.841359 GiB/s : time 328.022 seconds
[RESULT]    mdtest-hard-write      288.530970 kIOPS : time 300.147 seconds
[RESULT]                 find     4941.520633 kIOPS : time 129.823 seconds
[RESULT]        ior-easy-read       76.414370 GiB/s : time 304.786 seconds
[RESULT]     mdtest-easy-stat     1926.370017 kIOPS : time 300.466 seconds
[RESULT]        ior-hard-read       29.101179 GiB/s : time 302.558 seconds
[RESULT]     mdtest-hard-stat     1721.456790 kIOPS : time 36.433 seconds
[RESULT]   mdtest-easy-delete     1844.140322 kIOPS : time 316.425 seconds
[RESULT]     mdtest-hard-read      583.342793 kIOPS : time 107.496 seconds
[RESULT]   mdtest-hard-delete      691.369215 kIOPS : time 91.493 seconds
[SCORE] Bandwidth 45.791080 GiB/s : IOPS 1251.323842 kiops : TOTAL 239.373077