JUWELS

Institution Forschungszentrum Juelich (FZJ)
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.8
Client Kernel Version 3.10.0-1127

DATA SERVER

Storage Type SSD-NVMe
Volatile Memory 96GB
Storage Interface NVMe
Network IB-HDR
Software Version IME 1.4.1
OS Version Centos 7.7

INFORMATION

Client Nodes 10
Client Total Procs 400

METADATA

Easy Write 9.14 kIOP/s
Easy Stat 109.31 kIOP/s
Easy Delete 8.63 kIOP/s
Hard Write 1.21 kIOP/s
Hard Read 1.41 kIOP/s
Hard Stat 1.56 kIOP/s
Hard Delete 0.66 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.
#
# The only parts of the script that may need to be modified are:
#  - setup_paths() to configure the binary locations and MPI parameters
#  - setup_directories() to create/tune the IOR/mdtest output directories
#
# Please visit https://vi4io.org/io500-info-creator/ to help generate the
# "system-information.txt" file, by pasting the output of the info-creator.
# This file contains details of your system hardware for your submission.

# Set the paths to the binaries and how to launch MPI jobs.
# If you ran ./prepare.sh successfully, then binaries are in ./bin/
function setup_paths {
  CNODES="jwc09n048,jwc09n072,jwc09n096,jwc09n099,jwc09n120,jwc09n144,jwc09n147,jwc09n174,jwc09n177,jwc09n021"
  BINDING="-genv MV2_CPU_MAPPING 0:1:2:3:4:5:6:7:8:9:10:11:12:13:14:15:16:17:18:19:20:21:22:23:24:25:26:27:28:29:30:31:32:33:34:35:36:37:38:39"

  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mpirun="/opt/ddn/mvapich/bin/mpirun"
  #io500_mpiargs=" -np 400 -ppn 40 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_HANDSHAKE_ENABLED 1 -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_ENABLE_SGL 1 -genv LD_PRELOAD /p/project/ddn-ime/SC20/posix_2_ime/libposix2ime.so -genv IM_CLIENT_BFS_PATH /p/scratch -genv IM_CLIENT_NO_MKNOD_CREATE 1 -genv IM_CLIENT_NO_LARGE_DIR_BUFFER 1"
  io500_mpiargs=" -np 400 -ppn 40 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_HANDSHAKE_ENABLED 1 -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_ENABLE_SGL 1 "
}

# Set directories where benchmark files are created and where the results go.
# If you want to set up stripe tuning on your output directories or anything
# similar, then this is the right place to do it.
function setup_directories {
  local workdir
  local resultdir
  local bfspath

  mkdir -p $io500_workdir $io500_resultdir

  eval $(grep "^BFSDIR" /etc/ddn/ime/ime-fuse.conf)
  eval $(grep "^MNTDIR" /etc/ddn/ime/ime-fuse.conf)
  bfspath=$(echo "$BFSDIR${io500_workdir#"$MNTDIR"}")
}

# *****  YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE  *****
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

function run_benchmarks {
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_resultdir)
  local fname=$(basename ${io500_resultdir})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_resultdir
  tar czf $tarball -C $sourcedir $fname
  echo "Created result tarball $tarball"
}

function main {
  # These commands extract the 'datadir' and 'resultdir' from .ini file
  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # working directory where the test files will be created
  export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts

  setup_directories
  setup_paths
  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a 'system-information.txt' description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_resultdir
  fi

  create_tarball
}

main
ior-easy-read
WARNING: fsync() not supported in selected backend.  Using value of 0.
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Oct 31 02:47:49 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-easy/ior_file_easy -O stoneWallingStatusFile=/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/ior-easy.stonewall -t 512K -b 9920000m -F -r -R -a IME
Machine             : Linux jwc09n048.adm09.juwels.fzj.de
TestID              : 0
StartTime           : Sat Oct 31 02:47:49 2020
Path                : /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-easy/ior_file_easy.00000000
FS                  : 13830.5 TiB   Used FS: 61.1%   Inodes: 5815.9 Mi   Used Inodes: 3.9%

Options: 
api                 : IME
apiVersion          : 1.4.1 (141020)
test filename       : /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 524288 bytes
blocksize           : 9.46 TiB
aggregate filesize  : 3784.18 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 4160749568000000
WARNING: Stat() of aggregate file size      = 71960205721600
WARNING: Using actual aggregate bytes moved = 71960205721600
read      152650     305473     0.000023    10158080000 512.00     0.260231   449.31     0.011816   449.57     0   
ior-easy-write
WARNING: fsync() not supported in selected backend.  Using value of 0.
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Oct 31 01:04:10 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-easy/ior_file_easy -O stoneWallingStatusFile=/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/ior-easy.stonewall -t 512K -b 9920000m -F -w -D 300 -O stoneWallingWearOut=1 -a IME
Machine             : Linux jwc09n048.adm09.juwels.fzj.de
TestID              : 0
StartTime           : Sat Oct 31 01:04:15 2020
WARNING: Backend returned error during statfs.

Options: 
api                 : IME
apiVersion          : 1.4.1 (141020)
test filename       : /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 524288 bytes
blocksize           : 9.46 TiB
aggregate filesize  : 3784.18 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 335252 max: 343133 -- min data: 163.7 GiB mean data: 165.5 GiB time: 300.0s
WARNING: Expected aggregate file size       = 4160749568000000
WARNING: Stat() of aggregate file size      = 71960205721600
WARNING: Using actual aggregate bytes moved = 71960205721600
WARNING: Maybe caused by deadlineForStonewalling
write     224481     449709     0.000015    10158080000 512.00     0.377186   305.20     0.131224   305.71     0   
ior-hard-read
WARNING: fsync() not supported in selected backend.  Using value of 0.
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Oct 31 03:00:17 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-hard/file -O stoneWallingStatusFile=/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a IME
Machine             : Linux jwc09n048.adm09.juwels.fzj.de
TestID              : 0
StartTime           : Sat Oct 31 03:00:17 2020
Path                : /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-hard/file
FS                  : 13830.5 TiB   Used FS: 61.0%   Inodes: 5815.9 Mi   Used Inodes: 3.9%

Options: 
api                 : IME
apiVersion          : 1.4.1 (141020)
test filename       : /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 171.01 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 3995179 max: 3995179 -- min data: 174.9 GiB mean data: 174.9 GiB time: 500.7s
WARNING: Expected aggregate file size       = 188032000000000
WARNING: Stat() of aggregate file size      = 75122149772800
WARNING: Using actual aggregate bytes moved = 75122149772800
read      143046     3191165    500.76      45.91      45.91      0.050069   500.78     0.032837   500.83     0   
ior-hard-write
WARNING: fsync() not supported in selected backend.  Using value of 0.
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Oct 31 02:11:39 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-hard/file -O stoneWallingStatusFile=/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a IME
Machine             : Linux jwc09n048.adm09.juwels.fzj.de
TestID              : 0
StartTime           : Sat Oct 31 02:11:39 2020
WARNING: Backend returned error during statfs.

Options: 
api                 : IME
apiVersion          : 1.4.1 (141020)
test filename       : /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 171.01 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 3625161 max: 3995179 -- min data: 158.7 GiB mean data: 167.6 GiB time: 300.0s
WARNING: Expected aggregate file size       = 188032000000000
WARNING: Stat() of aggregate file size      = 75122149772800
WARNING: Using actual aggregate bytes moved = 75122149772800
WARNING: Maybe caused by deadlineForStonewalling
write     225714     5041537    312.83      45.91      45.91      0.224378   316.98     0.198403   317.40     0   
mdtest-easy-delete
-- started at 10/31/2020 03:13:21 --

mdtest-3.4.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-easy' '-x' '/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-easy.stonewall' '-r' '-a' 'POSIX'
Path: /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03
FS: 946.1 TiB   Used FS: 63.3%   Inodes: 5815.9 Mi   Used Inodes: 3.9%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 40 for each phase.
400 tasks, 400000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       8633.246       8633.235       8633.237          0.002
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.099          0.099          0.099          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       3761.417       3761.412       3761.417          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         10.090         10.090         10.090          0.000
-- finished at 10/31/2020 04:16:21 --

mdtest-easy-stat
-- started at 10/31/2020 02:55:19 --

mdtest-3.4.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-easy' '-x' '/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-easy.stonewall' '-T' '-a' 'IME'
Path: /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03
FS: 13830.5 TiB   Used FS: 61.0%   Inodes: 5815.9 Mi   Used Inodes: 3.9%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 40 for each phase.
400 tasks, 400000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     109316.974     109311.919     109312.257          0.672
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        297.069        297.055        297.068          0.002
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/31/2020 03:00:16 --

mdtest-easy-write
-- started at 10/31/2020 01:09:22 --

mdtest-3.4.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-easy' '-x' '/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-easy.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03
FS: 946.1 TiB   Used FS: 56.1%   Inodes: 5815.9 Mi   Used Inodes: 3.4%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 40 for each phase.
400 tasks, 400000000 files
Continue stonewall hit min: 6934 max: 81183 avg: 29693.4 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       9143.487       9143.483       9143.485          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      39220.570             NA
   Tree creation             :          1.263          1.263          1.263          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       3551.513       3551.512       3551.512          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        302.835             NA
   Tree creation             :          0.792          0.792          0.792          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/31/2020 02:11:39 --

mdtest-hard-delete
-- started at 10/31/2020 04:21:35 --

mdtest-3.4.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-hard' '-x' '/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-hard.stonewall' '-r' '-a' 'POSIX'
Path: /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03
FS: 946.1 TiB   Used FS: 63.3%   Inodes: 5815.9 Mi   Used Inodes: 3.5%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 40 for each phase.
400 tasks, 400000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        661.209        661.209        661.209          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.472          1.472          1.472          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        668.473        668.472        668.473          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.679          0.679          0.679          0.000
-- finished at 10/31/2020 04:32:45 --

mdtest-hard-read
-- started at 10/31/2020 04:16:22 --

mdtest-3.4.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-hard' '-x' '/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-hard.stonewall' '-E' '-X' '-a' 'POSIX'
Path: /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03
FS: 946.1 TiB   Used FS: 63.3%   Inodes: 5815.9 Mi   Used Inodes: 3.5%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 40 for each phase.
400 tasks, 400000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :       1410.906       1410.906       1410.906          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        313.274        313.274        313.274          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/31/2020 04:21:35 --

mdtest-hard-stat
-- started at 10/31/2020 03:08:38 --

mdtest-3.4.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-hard' '-x' '/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-hard.stonewall' '-T' '-a' 'IME'
Path: /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03
FS: 13830.5 TiB   Used FS: 61.0%   Inodes: 5815.9 Mi   Used Inodes: 3.9%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 40 for each phase.
400 tasks, 400000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :       1560.903       1560.767       1560.786          0.032
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        283.194        283.169        283.191          0.006
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/31/2020 03:13:21 --

mdtest-hard-write
-- started at 10/31/2020 02:16:57 --

mdtest-3.4.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-hard' '-x' '/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-hard.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03
FS: 946.1 TiB   Used FS: 63.3%   Inodes: 5815.9 Mi   Used Inodes: 3.9%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 40 for each phase.
400 tasks, 400000000 files
Continue stonewall hit min: 1045 max: 1105 avg: 1081.3 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       1213.452       1213.449       1213.450          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA       1440.914             NA
   Tree creation             :         22.809         22.809         22.809          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        364.251        364.250        364.251          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.182             NA
   Tree creation             :          0.044          0.044          0.044          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/31/2020 02:26:05 --

result
version         = 
config-hash     = C44128B6
result-dir      = /p/project/ddn-ime/SC20/results/2020.10.31-01.04.03
; START 2020-10-31 01:04:10


[ior-easy-write]
t_start         = 2020-10-31 01:04:10
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-easy/ior_file_easy -O stoneWallingStatusFile=/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/ior-easy.stonewall -t 512K -b 9920000m -F -w -D 300 -O stoneWallingWearOut=1 -a IME
throughput-stonewall = 86600.42
score           = 219.220071
t_delta         = 311.6453
t_end           = 2020-10-31 01:09:22

[mdtest-easy-write]
t_start         = 2020-10-31 01:09:22
exe             = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-easy -x /p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-easy.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 39.220570
score           = 9.143485
t_delta         = 3737.5082
t_end           = 2020-10-31 02:11:39

[timestamp]
t_start         = 2020-10-31 02:11:39
t_delta         = 0.0155
t_end           = 2020-10-31 02:11:39

[ior-hard-write]
t_start         = 2020-10-31 02:11:39
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-hard/file -O stoneWallingStatusFile=/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a IME
throughput-stonewall = 84508.06
score           = 220.424072
t_delta         = 317.5926
t_end           = 2020-10-31 02:16:57

[mdtest-hard-write]
t_start         = 2020-10-31 02:16:57
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-hard -x /p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-hard.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 1.440914
score           = 1.213449
t_delta         = 547.9459
t_end           = 2020-10-31 02:26:05

[find]
t_start         = 2020-10-31 02:26:05
exe             = ./pfind /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03 -newer /p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/timestampfile -size 3901c -name *01* -C -q 50000
found           = 11652
total-files     = 32915601
score           = 25.240802
t_delta         = 1304.0805
t_end           = 2020-10-31 02:47:49

[ior-easy-read]
t_start         = 2020-10-31 02:47:49
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-easy/ior_file_easy -O stoneWallingStatusFile=/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/ior-easy.stonewall -t 512K -b 9920000m -F -r -R -a IME
score           = 149.072155
t_delta         = 450.1238
t_end           = 2020-10-31 02:55:19

[mdtest-easy-stat]
t_start         = 2020-10-31 02:55:19
exe             = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-easy -x /p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-easy.stonewall -T -a IME
score           = 109.311940
t_delta         = 297.2756
t_end           = 2020-10-31 03:00:16

[ior-hard-read]
t_start         = 2020-10-31 03:00:16
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/ior-hard/file -O stoneWallingStatusFile=/p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a IME
score           = 139.692970
t_delta         = 501.2321
t_end           = 2020-10-31 03:08:38

[mdtest-hard-stat]
t_start         = 2020-10-31 03:08:38
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-hard -x /p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-hard.stonewall -T -a IME
score           = 1.560767
t_delta         = 283.4060
t_end           = 2020-10-31 03:13:21

[mdtest-easy-delete]
t_start         = 2020-10-31 03:13:21
exe             = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-easy -x /p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-easy.stonewall -r -a POSIX
score           = 8.633245
t_delta         = 3780.1051
t_end           = 2020-10-31 04:16:21

[mdtest-hard-read]
t_start         = 2020-10-31 04:16:21
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-hard -x /p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-hard.stonewall -E -X -a POSIX
score           = 1.410906
t_delta         = 313.9719
t_end           = 2020-10-31 04:21:35

[mdtest-hard-delete]
t_start         = 2020-10-31 04:21:35
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /p/ime-scratch/fs/vet1/io500/2020.10.31-01.04.03/mdtest-hard -x /p/project/ddn-ime/SC20/results/2020.10.31-01.04.03/mdtest-hard.stonewall -r -a POSIX
score           = 0.661209
t_delta         = 669.3375
t_end           = 2020-10-31 04:32:45

[SCORE]
MD              = 4.990644
BW              = 178.105612
SCORE           = 29.813784 
hash            = 76B35D43
; END 2020-10-31 04:32:46
result_summary
IO500 version 
[RESULT]       ior-easy-write      219.220071 GiB/s : time 311.645 seconds
[RESULT]    mdtest-easy-write        9.143485 kIOPS : time 3737.508 seconds
[RESULT]       ior-hard-write      220.424072 GiB/s : time 317.593 seconds
[RESULT]    mdtest-hard-write        1.213449 kIOPS : time 547.946 seconds
[RESULT]                 find       25.240802 kIOPS : time 1304.080 seconds
[RESULT]        ior-easy-read      149.072155 GiB/s : time 450.124 seconds
[RESULT]     mdtest-easy-stat      109.311940 kIOPS : time 297.276 seconds
[RESULT]        ior-hard-read      139.692970 GiB/s : time 501.232 seconds
[RESULT]     mdtest-hard-stat        1.560767 kIOPS : time 283.406 seconds
[RESULT]   mdtest-easy-delete        8.633245 kIOPS : time 3780.105 seconds
[RESULT]     mdtest-hard-read        1.410906 kIOPS : time 313.972 seconds
[RESULT]   mdtest-hard-delete        0.661209 kIOPS : time 669.337 seconds
[SCORE] Bandwidth 178.105612 GiB/s : IOPS 4.990644 kiops : TOTAL 29.813784