NextGENIO

Institution EPCC
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version
Client Kernel Version 3.10.0-862.14.4.el7.x86_64

DATA SERVER

Storage Type 3DXPNVMeSSD
Volatile Memory 2
Storage Interface NVMe
Network Omnipath
Software Version 0.9dev
OS Version

INFORMATION

Client Nodes 10
Client Total Procs 360
Metadata Nodes 31
Metadata Storage Devices 2
Data Nodes 31
Data Storage Devices 2

METADATA

Easy Write 1,962.98 kIOP/s
Easy Stat 2,055.94 kIOP/s
Easy Delete 2,020.56 kIOP/s
Hard Write 205.67 kIOP/s
Hard Read 594.75 kIOP/s
Hard Stat 1,956.59 kIOP/s
Hard Delete 707.95 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.
#
# The only parts of the script that may need to be modified are:
#  - setup_paths() to configure the binary locations and MPI parameters
#  - setup_directories() to create/tune the IOR/mdtest output directories
#
# Please visit https://vi4io.org/io500-info-creator/ to help generate the
# "system-information.txt" file, by pasting the output of the info-creator.
# This file contains details of your system hardware for your submission.

# Set the paths to the binaries and how to launch MPI jobs.
# If you ran ./prepare.sh successfully, then binaries are in ./bin/
function setup_paths {
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mpirun="srun"
  io500_mpiargs="-N 10 -n ${NPROCS_MPI} --export=ALL,PSM2_DEVICES=self,hfi,shm,PSM2_MULTIRAIL=1,PSM2_MULTI_EP=0,LD_PRELOAD=${GKFS_PRLD}"
#  io500_mpirun="mpiexec"
#  io500_mpiargs="-np 2"
}

# Set directories where benchmark files are created and where the results go.
# If you want to set up stripe tuning on your output directories or anything
# similar, then this is the right place to do it.
function setup_directories {
  local workdir
  local resultdir
  local ts
  FI_PSM2_DISCONNECT=1 PSM2_MULTIRAIL=1 PSM2_MULTI_EP=0 LD_PRELOAD=$GKFS_PRLD mkdir -p $io500_workdir
  FI_PSM2_DISCONNECT=1 PSM2_MULTIRAIL=1 PSM2_MULTI_EP=0 LD_PRELOAD=$GKFS_PRLD mkdir -p $io500_workdir/ior-easy $io500_workdir/ior-hard
  FI_PSM2_DISCONNECT=1 PSM2_MULTIRAIL=1 PSM2_MULTI_EP=0 LD_PRELOAD=$GKFS_PRLD mkdir -p $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard

  mkdir -p $io500_workdir $io500_resultdir

  # Example commands to create output directories for Lustre.  Creating
  # top-level directories is allowed, but not the whole directory tree.
  #if (( $(lfs df $io500_workdir | grep -c MDT) > 1 )); then
  #  lfs setdirstripe -D -c -1 $io500_workdir
  #fi
  #lfs setstripe -c 1 $io500_workdir
  #mkdir $io500_workdir/ior-easy $io500_workdir/ior-hard
  #mkdir $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard
  #local osts=$(lfs df $io500_workdir | grep -c OST)
  # Try overstriping for ior-hard to improve scaling, or use wide striping
  #lfs setstripe -C $((osts * 4)) $io500_workdir/ior-hard ||
  #  lfs setstripe -c -1 $io500_workdir/ior-hard
  # Try to use DoM if available, otherwise use default for small files
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-easy || true #DoM?
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-hard || true #DoM?
}

# *****  YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE  *****
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

function run_benchmarks {
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_resultdir)
  local fname=$(basename ${io500_resultdir})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_resultdir
  tar czf $tarball -C $sourcedir $fname
  echo "Created result tarball $tarball"
}

function main {
  # These commands extract the 'datadir' and 'resultdir' from .ini file
  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # working directory where the test files will be created
  export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts

  setup_directories
  setup_paths
  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a 'system-information.txt' description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_resultdir
  fi

  create_tarball
}

main
ior-easy-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Oct 26 09:01:05 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/r/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results_run_second/2020.10.26-08.35.23/ior-easy.stonewall -t 512kb -b 9920000m -F -r -R -a POSIX
Machine             : Linux nextgenio-cn01
TestID              : 0
StartTime           : Mon Oct 26 09:01:05 2020
Path                : /tmp/r/ior-easy/ior_file_easy.00000000
FS                  : 56.3 TiB   Used FS: 49.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/r/ior-easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 360
clients per node    : 36
repetitions         : 1
xfersize            : 524288 bytes
blocksize           : 9.46 TiB
aggregate filesize  : 3405.76 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 3744674611200000
WARNING: Stat() of aggregate file size      = 25842973409280
WARNING: Using actual aggregate bytes moved = 25842973409280
read      76551      153108     0.000016    10158080000 512.00     0.014775   321.94     0.007768   321.95     0   
ior-easy-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Oct 26 08:35:32 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/r/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results_run_second/2020.10.26-08.35.23/ior-easy.stonewall -t 512kb -b 9920000m -F -w -D 300 -O stoneWallingWearOut=1 -a POSIX
Machine             : Linux nextgenio-cn01
TestID              : 0
StartTime           : Mon Oct 26 08:35:32 2020
Path                : /tmp/r/ior-easy/ior_file_easy.00000000
FS                  : 56.3 TiB   Used FS: 5.1%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/r/ior-easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 360
clients per node    : 36
repetitions         : 1
xfersize            : 524288 bytes
blocksize           : 9.46 TiB
aggregate filesize  : 3405.76 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 110770 max: 136921 -- min data: 54.1 GiB mean data: 59.7 GiB time: 300.1s
WARNING: Expected aggregate file size       = 3744674611200000
WARNING: Stat() of aggregate file size      = 25842973409280
WARNING: Using actual aggregate bytes moved = 25842973409280
WARNING: Maybe caused by deadlineForStonewalling
write     70143      140292     0.000017    10158080000 512.00     0.013987   351.35     0.002848   351.37     0   
ior-hard-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Oct 26 09:11:38 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/r/ior-hard/file -O stoneWallingStatusFile=./results_run_second/2020.10.26-08.35.23/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a POSIX
Machine             : Linux nextgenio-cn01
TestID              : 0
StartTime           : Mon Oct 26 09:11:38 2020
Path                : /tmp/r/ior-hard/file
FS                  : 56.3 TiB   Used FS: 49.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/r/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 360
clients per node    : 36
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 153.91 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 74777 max: 74777 -- min data: 3.3 GiB mean data: 3.3 GiB time: 42.9s
WARNING: Expected aggregate file size       = 169228800000000
WARNING: Stat() of aggregate file size      = 1265442197760
WARNING: Using actual aggregate bytes moved = 1265442197760
read      27984      627267     42.91       45.91      45.91      0.216062   42.92      0.008112   43.13      0   
ior-hard-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Oct 26 08:47:26 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/r/ior-hard/file -O stoneWallingStatusFile=./results_run_second/2020.10.26-08.35.23/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a POSIX
Machine             : Linux nextgenio-cn01
TestID              : 0
StartTime           : Mon Oct 26 08:47:26 2020
Path                : /tmp/r/ior-hard/file
FS                  : 56.3 TiB   Used FS: 46.9%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /tmp/r/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 360
clients per node    : 36
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 153.91 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 74544 max: 74777 -- min data: 3.3 GiB mean data: 3.3 GiB time: 300.0s
WARNING: Expected aggregate file size       = 169228800000000
WARNING: Stat() of aggregate file size      = 1265442197760
WARNING: Using actual aggregate bytes moved = 1265442197760
WARNING: Maybe caused by deadlineForStonewalling
write     4012       89552      300.43      45.91      45.91      0.206624   300.61     0.005198   300.81     0   
mdtest-easy-delete
-- started at 10/26/2020 09:12:52 --

mdtest-3.4.0+dev was launched with 360 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '2200000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/r/mdtest-easy' '-x' './results_run_second/2020.10.26-08.35.23/mdtest-easy.stonewall' '-r' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 8.1%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 36 for each phase.
360 tasks, 792000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    2020570.145    2020553.748    2020561.873          3.421
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.069          0.069          0.069          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        316.987        316.985        316.986          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         14.412         14.412         14.412          0.000
-- finished at 10/26/2020 09:18:24 --

mdtest-easy-stat
-- started at 10/26/2020 09:06:27 --

mdtest-3.4.0+dev was launched with 360 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '2200000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/r/mdtest-easy' '-x' './results_run_second/2020.10.26-08.35.23/mdtest-easy.stonewall' '-T' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 8.1%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 36 for each phase.
360 tasks, 792000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    2055947.579    2055930.720    2055939.064          3.490
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        311.533        311.530        311.531          0.001
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/26/2020 09:11:38 --

mdtest-easy-write
-- started at 10/26/2020 08:41:23 --

mdtest-3.4.0+dev was launched with 360 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '2200000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/r/mdtest-easy' '-x' './results_run_second/2020.10.26-08.35.23/mdtest-easy.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 8.1%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 36 for each phase.
360 tasks, 792000000 files
Continue stonewall hit min: 1645168 max: 1779138 avg: 1718876.9 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    1962986.853    1962971.040    1962978.938          3.315
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    2062594.760             NA
   Tree creation             :         60.273         60.273         60.273          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        326.286        326.283        326.285          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.008             NA
   Tree creation             :          0.017          0.017          0.017          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/26/2020 08:47:26 --

mdtest-hard-delete
-- started at 10/26/2020 09:20:04 --

mdtest-3.4.0+dev was launched with 360 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '165000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/r/mdtest-hard' '-x' './results_run_second/2020.10.26-08.35.23/mdtest-hard.stonewall' '-r' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 8.1%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 36 for each phase.
360 tasks, 59400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     707954.101     707948.339     707951.200          1.195
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.690          2.690          2.690          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :         83.904         83.904         83.904          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.372          0.372          0.372          0.000
-- finished at 10/26/2020 09:21:29 --

mdtest-hard-read
-- started at 10/26/2020 09:18:24 --

mdtest-3.4.0+dev was launched with 360 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '165000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/r/mdtest-hard' '-x' './results_run_second/2020.10.26-08.35.23/mdtest-hard.stonewall' '-E' '-X' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 8.1%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 36 for each phase.
360 tasks, 59400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     594757.125     594752.301     594754.698          1.006
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :         99.874         99.873         99.873          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/26/2020 09:20:04 --

mdtest-hard-stat
-- started at 10/26/2020 09:12:22 --

mdtest-3.4.0+dev was launched with 360 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '165000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/r/mdtest-hard' '-x' './results_run_second/2020.10.26-08.35.23/mdtest-hard.stonewall' '-T' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 8.1%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 36 for each phase.
360 tasks, 59400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    1956603.231    1956587.316    1956594.893          3.301
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         30.359         30.359         30.359          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/26/2020 09:12:52 --

mdtest-hard-write
-- started at 10/26/2020 08:52:27 --

mdtest-3.4.0+dev was launched with 360 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '165000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/r/mdtest-hard' '-x' './results_run_second/2020.10.26-08.35.23/mdtest-hard.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /tmp/r
FS: 93.0 GiB   Used FS: 8.1%   Inodes: 23.3 Mi   Used Inodes: 0.4%

Nodemap: 111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 36 for each phase.
360 tasks, 59400000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     205674.086     205672.454     205673.255          0.348
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     225179.384             NA
   Tree creation             :       2324.355       2324.355       2324.355          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        288.809        288.806        288.808          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        263.790             NA
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/26/2020 08:58:30 --

result
version         = io500-sc20-6-gf1c40e1e4342
config-hash     = D3F0D8C7
result-dir      = ./results_run_second/2020.10.26-08.35.23
; START 2020-10-26 08:35:32


[ior-easy-write]
t_start         = 2020-10-26 08:35:32
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/r/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results_run_second/2020.10.26-08.35.23/ior-easy.stonewall -t 512kb -b 9920000m -F -w -D 300 -O stoneWallingWearOut=1 -a POSIX
throughput-stonewall = 22004.17
score           = 68.498947
t_delta         = 351.4580
t_end           = 2020-10-26 08:41:23

[mdtest-easy-write]
t_start         = 2020-10-26 08:41:23
exe             = ./mdtest -n 2200000 -u -L -F -P -N 1 -d /tmp/r/mdtest-easy -x ./results_run_second/2020.10.26-08.35.23/mdtest-easy.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 2062.594760
score           = 1962.978043
t_delta         = 362.7080
t_end           = 2020-10-26 08:47:26

[timestamp]
t_start         = 2020-10-26 08:47:26
t_delta         = 0.0009
t_end           = 2020-10-26 08:47:26

[ior-hard-write]
t_start         = 2020-10-26 08:47:26
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/r/ior-hard/file -O stoneWallingStatusFile=./results_run_second/2020.10.26-08.35.23/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a POSIX
throughput-stonewall = 1407.98
score           = 3.917847
t_delta         = 301.0305
t_end           = 2020-10-26 08:52:27

[mdtest-hard-write]
t_start         = 2020-10-26 08:52:27
exe             = ./mdtest -n 165000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/r/mdtest-hard -x ./results_run_second/2020.10.26-08.35.23/mdtest-hard.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 225.179384
score           = 205.673154
t_delta         = 362.7638
t_end           = 2020-10-26 08:58:30

[find]
t_start         = 2020-10-26 08:58:30
exe             = ./pfind /tmp/r -newer ./results_run_second/2020.10.26-08.35.23/timestampfile -size 3901c -name *01* -C -N -H 1 -q 100000
nproc           = 120
found           = 2584521
total-files     = 699890041
score           = 4527.324049
t_delta         = 154.7618
t_end           = 2020-10-26 09:01:05

[ior-easy-read]
t_start         = 2020-10-26 09:01:05
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/r/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results_run_second/2020.10.26-08.35.23/ior-easy.stonewall -t 512kb -b 9920000m -F -r -R -a POSIX
score           = 74.756642
t_delta         = 322.0011
t_end           = 2020-10-26 09:06:27

[mdtest-easy-stat]
t_start         = 2020-10-26 09:06:27
exe             = ./mdtest -n 2200000 -u -L -F -P -N 1 -d /tmp/r/mdtest-easy -x ./results_run_second/2020.10.26-08.35.23/mdtest-easy.stonewall -T -a POSIX
score           = 2055.938123
t_delta         = 311.5461
t_end           = 2020-10-26 09:11:38

[ior-hard-read]
t_start         = 2020-10-26 09:11:38
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/r/ior-hard/file -O stoneWallingStatusFile=./results_run_second/2020.10.26-08.35.23/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a POSIX
score           = 27.328138
t_delta         = 43.3354
t_end           = 2020-10-26 09:12:22

[mdtest-hard-stat]
t_start         = 2020-10-26 09:12:22
exe             = ./mdtest -n 165000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/r/mdtest-hard -x ./results_run_second/2020.10.26-08.35.23/mdtest-hard.stonewall -T -a POSIX
score           = 1956.594009
t_delta         = 30.3689
t_end           = 2020-10-26 09:12:52

[mdtest-easy-delete]
t_start         = 2020-10-26 09:12:52
exe             = ./mdtest -n 2200000 -u -L -F -P -N 1 -d /tmp/r/mdtest-easy -x ./results_run_second/2020.10.26-08.35.23/mdtest-easy.stonewall -r -a POSIX
score           = 2020.560896
t_delta         = 332.4563
t_end           = 2020-10-26 09:18:24

[mdtest-hard-read]
t_start         = 2020-10-26 09:18:24
exe             = ./mdtest -n 165000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/r/mdtest-hard -x ./results_run_second/2020.10.26-08.35.23/mdtest-hard.stonewall -E -X -a POSIX
score           = 594.754407
t_delta         = 99.8846
t_end           = 2020-10-26 09:20:04

[mdtest-hard-delete]
t_start         = 2020-10-26 09:20:04
exe             = ./mdtest -n 165000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/r/mdtest-hard -x ./results_run_second/2020.10.26-08.35.23/mdtest-hard.stonewall -r -a POSIX
score           = 707.950840
t_delta         = 84.6672
t_end           = 2020-10-26 09:21:29

[SCORE]
MD              = 1257.570274
BW              = 27.211206
SCORE           = 184.986496 
hash            = EE5F1983
; END 2020-10-26 09:23:03
result_summary
IO500 version io500-sc20-6-gf1c40e1e4342
[RESULT]       ior-easy-write       68.498947 GiB/s : time 351.458 seconds
[RESULT]    mdtest-easy-write     1962.978043 kIOPS : time 362.708 seconds
[RESULT]       ior-hard-write        3.917847 GiB/s : time 301.031 seconds
[RESULT]    mdtest-hard-write      205.673154 kIOPS : time 362.764 seconds
[RESULT]                 find     4527.324049 kIOPS : time 154.762 seconds
[RESULT]        ior-easy-read       74.756642 GiB/s : time 322.001 seconds
[RESULT]     mdtest-easy-stat     2055.938123 kIOPS : time 311.546 seconds
[RESULT]        ior-hard-read       27.328138 GiB/s : time 43.335 seconds
[RESULT]     mdtest-hard-stat     1956.594009 kIOPS : time 30.369 seconds
[RESULT]   mdtest-easy-delete     2020.560896 kIOPS : time 332.456 seconds
[RESULT]     mdtest-hard-read      594.754407 kIOPS : time 99.885 seconds
[RESULT]   mdtest-hard-delete      707.950840 kIOPS : time 84.667 seconds
[SCORE] Bandwidth 27.211206 GiB/s : IOPS 1257.570274 kiops : TOTAL 184.986496