Frontera

Institution TACC
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.7
Client Kernel Version 3.1

DATA SERVER

Storage Type NVMe-SSD
Volatile Memory 128GB
Storage Interface Infiniband
Network IB-HDR
Software Version 1.4.1
OS Version CentOS 7.7

INFORMATION

Client Nodes 10
Client Total Procs 280
Metadata Nodes 2
Metadata Storage Devices 468
Data Nodes 32
Data Storage Devices 22

METADATA

Easy Write 34.65 kIOP/s
Easy Stat 259.21 kIOP/s
Easy Delete 24.09 kIOP/s
Hard Write 6.54 kIOP/s
Hard Read 174.36 kIOP/s
Hard Stat 144.32 kIOP/s
Hard Delete 20.57 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.
#
# The only parts of the script that may need to be modified are:
#  - setup_paths() to configure the binary locations and MPI parameters
#  - setup_directories() to create/tune the IOR/mdtest output directories
#
# Please visit https://vi4io.org/io500-info-creator/ to help generate the
# "system-information.txt" file, by pasting the output of the info-creator.
# This file contains details of your system hardware for your submission.

# Set the paths to the binaries and how to launch MPI jobs.
# If you ran ./prepare.sh successfully, then binaries are in ./bin/
function setup_paths {
  ##CNODES="flash-32,flash-33,flash-34,flash-35,flash-36,flash-37,flash-38,flash-39,flash-40,flash-41"
  CNODES="flash-42,flash-43,flash-44,flash-45,flash-47,flash-48,flash-50,flash-51,flash-52,flash-53"
  #CNODES="flash-32,flash-33,flash-37,flash-38,flash-44,flash-45,flash-50,flash-51,flash-56,flash-57"
  #CNODES="flash-32,flash-37,flash-40,flash-41,flash-42,flash-45,flash-47,flash-48,flash-63"
#BINDING="-genv MV2_CPU_MAPPING 0,28:14,42:1,29:15,43:2,30:16,44:3,31:17,45:4,32:18,46:5,33:19,47:6,34:20,48:7,35:21,49:8,36:22,50:9,37:23,51:10,38:24,52:11,39:25,53:12,40:26,54:13,41:27,55"
#BINDING="-genv MV2_CPU_MAPPING 0,28:1,29:2,30:3,31:4,32:5,33:6,34:7,35:8,36:9,37:10,38:11,39:12,40:13,41:14,42:15,43:16,44:17,45:18,46:19,47:20,48:21,49:22,50:23,51:24,52:25,53:26,54:27,55"
#BINDING="-genv MV2_CPU_MAPPING 0:28:1:29:2:30:3:31:4:32:5:33:6:34:7:35:8:36:9:37:10:38:11:39:12:40:13:41:14:42:15:43:16:44:17:45:18:46:19:47:20:48:21:49:22:50:23:51:24:52:25:53:26:54:27:55"
BINDING="-genv MV2_CPU_MAPPING 0:1:14:15:2:3:16:17:4:5:18:19:6:7:20:21:8:9:22:23:10:11:24:25:12:13:26:27"
#BINDING="-genv MV2_CPU_MAPPING 0,28:1,29:2,30:3,31:4,32:5,33:6,34:7,35:8,36:9,37:10,38:11,39:12,40:13,41"
#BINDING="-genv MV2_CPU_MAPPING 0,28,14,42,1,29,15,43,2,30,16,44,3,31,17,45,4,32,18,46,5,33,19,47,6,34,20,48,7,35,21,49,8,36,22,50,9,37,23,51,10,38,24,52,11,39,25,53,12,40,26,54,13,41,27,55"
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mpirun="/opt/ddn/mvapich/bin/mpirun"
  #io500_mpiargs="-np 110 -ppn 11 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_NUM_HCAS 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_BFS_PATH /scratch2/flash -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so"
  #io500_mpiargs="-np 280 -ppn 28 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_NUM_HCAS 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so"
  #io500_mpiargs="-np 120 -ppn 12 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so"
  #io500_mpiargs="-np 280 -ppn 28 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_BFS_PATH /scratch2/flash -genv IM_CLIENT_ENABLE_SGL 1 -genv IM_CLIENT_NO_MKNOD_CREATE 1 -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so"
#w/o mmd
#io500_mpiargs="-np 280 -ppn 28 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 "
#mmd
#io500_mpiargs="-ppn 14 -np 140 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 --hosts $CNODES  -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 -env IM_CLIENT_NET_IF ib0-verbs  $PWD/io500 $io500_ini --timestamp $timestamp : -np 140 -env IM_CLIENT_NET_IF ib1-verbs "
#taskset
io500_mpiargs="-np 140 -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so -genv IM_CLIENT_BFS_PATH /scratch2/flash/ -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 --hosts $CNODES  -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 -env IM_CLIENT_NET_IF ib0-verbs   taskset -c 0-13 $PWD/io500 $io500_ini --timestamp $timestamp : -np 140 -env IM_CLIENT_NET_IF ib1-verbs taskset -c 14-27 "

#io500_mpiargs="-np 28 -ppn 28 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_NUM_HCAS 1 -genv MV2_USE_RDMA_CM 0 $BINDING -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so"
}

# Set directories where benchmark files are created and where the results go.
# If you want to set up stripe tuning on your output directories or anything
# similar, then this is the right place to do it.
function setup_directories {
  local workdir
  local resultdir
  local ts

  mkdir -p $io500_workdir $io500_resultdir

  eval $(grep "^BFSDIR" /etc/ddn/ime/ime-fuse.conf)
  eval $(grep "^MNTDIR" /etc/ddn/ime/ime-fuse.conf)
  bfspath=$(echo "$BFSDIR${io500_workdir#"$MNTDIR"}")

#  lfs setstripe -L mdt -E 1M $bfspath
#  lfs setdirstripe -D -c -1 $bfspath


  # Example commands to create output directories for Lustre.  Creating
  # top-level directories is allowed, but not the whole directory tree.
  #if (( $(lfs df $io500_workdir | grep -c MDT) > 1 )); then
  #  lfs setdirstripe -D -c -1 $io500_workdir
  #fi
  #lfs setstripe -c 1 $io500_workdir
  #mkdir $io500_workdir/ior-easy $io500_workdir/ior-hard
  #mkdir $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard
  #local osts=$(lfs df $io500_workdir | grep -c OST)
  # Try overstriping for ior-hard to improve scaling, or use wide striping
  #lfs setstripe -C $((osts * 4)) $io500_workdir/ior-hard ||
  #  lfs setstripe -c -1 $io500_workdir/ior-hard
  # Try to use DoM if available, otherwise use default for small files
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-easy || true #DoM?
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-hard || true #DoM?
}

# *****  YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE  *****
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

function run_benchmarks {
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_resultdir)
  local fname=$(basename ${io500_resultdir})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_resultdir
  tar czf $tarball -C $sourcedir $fname
  echo "Created result tarball $tarball"
}

function main {
  # These commands extract the 'datadir' and 'resultdir' from .ini file
  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # working directory where the test files will be created
  export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts

  setup_directories
  setup_paths
  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a 'system-information.txt' description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_resultdir
  fi

  create_tarball
}

main
ior-easy-read
WARNING: fsync() not supported in selected backend.  Using value of 0.
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Oct 26 08:27:03 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-easy.stonewall -t 1M -b 9920000m -r -R -a IME
Machine             : Linux flash-42
TestID              : 0
StartTime           : Mon Oct 26 08:27:03 2020
Path                : /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy
FS                  : 10835.6 TiB   Used FS: 12.7%   Inodes: 1144.4 Mi   Used Inodes: 28.9%

Options: 
api                 : IME
apiVersion          : 1.4.1 (141015)
test filename       : /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 280
clients per node    : 28
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 2648.93 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 2912524697600000
WARNING: Stat() of aggregate file size      = 2902345410150400
WARNING: Using actual aggregate bytes moved = 62324211712000
read      191757     191782     0.000031    10158080000 1024.00    0.038910   309.92     0.005796   309.96     0   
ior-easy-write
WARNING: fsync() not supported in selected backend.  Using value of 0.
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Oct 26 08:05:42 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-easy.stonewall -t 1M -b 9920000m -w -D 300 -O stoneWallingWearOut=1 -a IME
Machine             : Linux flash-42
TestID              : 0
StartTime           : Mon Oct 26 08:05:45 2020
WARNING: Backend returned error during statfs.

Options: 
api                 : IME
apiVersion          : 1.4.1 (141015)
test filename       : /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 280
clients per node    : 28
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 2648.93 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 206542 max: 212275 -- min data: 201.7 GiB mean data: 204.2 GiB time: 300.0s
WARNING: Expected aggregate file size       = 2912524697600000
WARNING: Stat() of aggregate file size      = 2902345410150400
WARNING: Using actual aggregate bytes moved = 62324211712000
WARNING: Maybe caused by deadlineForStonewalling
write     194209     194275     0.000031    10158080000 1024.00    0.035703   305.94     0.068537   306.05     0   
ior-hard-read
WARNING: fsync() not supported in selected backend.  Using value of 0.
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Oct 26 08:32:55 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a IME
Machine             : Linux flash-42
TestID              : 0
StartTime           : Mon Oct 26 08:32:55 2020
Path                : /flash/datafiles/2020.10.26-08.05.37/ior-hard/file
FS                  : 10835.6 TiB   Used FS: 12.7%   Inodes: 1144.4 Mi   Used Inodes: 28.9%

Options: 
api                 : IME
apiVersion          : 1.4.1 (141015)
test filename       : /flash/datafiles/2020.10.26-08.05.37/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 280
clients per node    : 28
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 119.71 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 4851523 max: 4851523 -- min data: 212.4 GiB mean data: 212.4 GiB time: 415.9s
WARNING: Expected aggregate file size       = 131622400000000
WARNING: Stat() of aggregate file size      = 63856910091520
WARNING: Using actual aggregate bytes moved = 63856910091520
read      146329     3264376    416.12      45.91      45.91      0.003747   416.14     0.040449   416.18     0   
ior-hard-write
WARNING: fsync() not supported in selected backend.  Using value of 0.
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Oct 26 08:16:01 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a IME
Machine             : Linux flash-42
TestID              : 0
StartTime           : Mon Oct 26 08:16:01 2020
WARNING: Backend returned error during statfs.

Options: 
api                 : IME
apiVersion          : 1.4.1 (141015)
test filename       : /flash/datafiles/2020.10.26-08.05.37/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 280
clients per node    : 28
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 119.71 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 4501129 max: 4851523 -- min data: 197.1 GiB mean data: 204.9 GiB time: 300.0s
WARNING: Expected aggregate file size       = 131622400000000
WARNING: Stat() of aggregate file size      = 63856910091520
WARNING: Using actual aggregate bytes moved = 63856910091520
WARNING: Maybe caused by deadlineForStonewalling
write     194609     4342120    310.53      45.91      45.91      0.034108   312.85     0.047459   312.93     0   
mdtest-easy-delete
-- started at 10/26/2020 08:40:06 --

mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-easy' '-x' './results/2020.10.26-08.05.37/mdtest-easy.stonewall' '-r' '-a' 'POSIX'
Path: /flash/datafiles/2020.10.26-08.05.37
FS: 1222.6 TiB   Used FS: 10.7%   Inodes: 1144.4 Mi   Used Inodes: 28.9%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
280 tasks, 280000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      24087.462      24087.398      24087.421          0.016
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.000          1.000          1.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        442.550        442.549        442.550          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.000          1.000          1.000          0.000
-- finished at 10/26/2020 08:47:30 --

mdtest-easy-stat
-- started at 10/26/2020 08:32:13 --

mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-easy' '-x' './results/2020.10.26-08.05.37/mdtest-easy.stonewall' '-T' '-a' 'IME'
Path: /flash/datafiles/2020.10.26-08.05.37
FS: 10835.6 TiB   Used FS: 12.7%   Inodes: 1144.4 Mi   Used Inodes: 28.9%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
280 tasks, 280000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     259212.035     259211.333     259211.663          0.160
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         41.124         41.124         41.124          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/26/2020 08:32:55 --

mdtest-easy-write
-- started at 10/26/2020 08:10:53 --

mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-easy' '-x' './results/2020.10.26-08.05.37/mdtest-easy.stonewall' '-C' '-Y' '-W' '300' '-a' 'IME'
Path: /flash/datafiles/2020.10.26-08.05.37
FS: 10835.6 TiB   Used FS: 12.7%   Inodes: 1144.4 Mi   Used Inodes: 27.9%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
280 tasks, 280000000 files
Continue stonewall hit min: 36683 max: 38071 avg: 37265.1 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      34650.332      34650.239      34650.273          0.024
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      34760.625             NA
   Tree creation             :        185.695        185.695        185.695          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        307.642        307.641        307.642          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.174             NA
   Tree creation             :          0.005          0.005          0.005          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/26/2020 08:16:01 --

mdtest-hard-delete
-- started at 10/26/2020 08:47:42 --

mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-hard' '-x' './results/2020.10.26-08.05.37/mdtest-hard.stonewall' '-r' '-a' 'IME'
Path: /flash/datafiles/2020.10.26-08.05.37
FS: 10835.6 TiB   Used FS: 12.7%   Inodes: 1144.4 Mi   Used Inodes: 28.1%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
280 tasks, 280000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      20565.437      20565.082      20565.123          0.029
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         17.441         17.441         17.441          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        104.212        104.210        104.211          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.057          0.057          0.057          0.000
-- finished at 10/26/2020 08:49:26 --

mdtest-hard-read
-- started at 10/26/2020 08:47:30 --

mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-hard' '-x' './results/2020.10.26-08.05.37/mdtest-hard.stonewall' '-E' '-X' '-a' 'IME'
Path: /flash/datafiles/2020.10.26-08.05.37
FS: 10835.6 TiB   Used FS: 12.7%   Inodes: 1144.4 Mi   Used Inodes: 28.1%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
280 tasks, 280000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     174357.878     174344.497     174348.117          5.548
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :         12.292         12.292         12.292          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/26/2020 08:47:42 --

mdtest-hard-stat
-- started at 10/26/2020 08:39:51 --

mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-hard' '-x' './results/2020.10.26-08.05.37/mdtest-hard.stonewall' '-T' '-a' 'IME'
Path: /flash/datafiles/2020.10.26-08.05.37
FS: 10835.6 TiB   Used FS: 12.7%   Inodes: 1144.4 Mi   Used Inodes: 28.9%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
280 tasks, 280000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     144316.009     144315.395     144315.651          0.117
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         14.850         14.850         14.850          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/26/2020 08:40:06 --

mdtest-hard-write
-- started at 10/26/2020 08:21:14 --

mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-hard' '-x' './results/2020.10.26-08.05.37/mdtest-hard.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /flash/datafiles/2020.10.26-08.05.37
FS: 1222.6 TiB   Used FS: 10.7%   Inodes: 1144.4 Mi   Used Inodes: 28.8%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
280 tasks, 280000000 files
Continue stonewall hit min: 7364 max: 7654 avg: 7561.3 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       6542.175       6542.157       6542.164          0.004
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA       7055.864             NA
   Tree creation             :        645.575        645.575        645.575          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        327.586        327.585        327.586          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.059             NA
   Tree creation             :          0.002          0.002          0.002          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/26/2020 08:26:44 --

result
version         = io500-sc20-6-gf1c40e1e4342
config-hash     = 1C74EB35
result-dir      = ./results/2020.10.26-08.05.37
; START 2020-10-26 08:05:41


[ior-easy-write]
t_start         = 2020-10-26 08:05:42
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-easy.stonewall -t 1M -b 9920000m -w -D 300 -O stoneWallingWearOut=1 -a IME
throughput-stonewall = 52319.40
score           = 189.657067
t_delta         = 310.7931
t_end           = 2020-10-26 08:10:53

[mdtest-easy-write]
t_start         = 2020-10-26 08:10:53
exe             = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /flash/datafiles/2020.10.26-08.05.37/mdtest-easy -x ./results/2020.10.26-08.05.37/mdtest-easy.stonewall -C -Y -W 300 -a IME
rate-stonewall  = 34.760625
score           = 34.650264
t_delta         = 307.8700
t_end           = 2020-10-26 08:16:01

[timestamp]
t_start         = 2020-10-26 08:16:01
t_delta         = 0.0032
t_end           = 2020-10-26 08:16:01

[ior-hard-write]
t_start         = 2020-10-26 08:16:01
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a IME
throughput-stonewall = 51337.45
score           = 190.047767
t_delta         = 313.0129
t_end           = 2020-10-26 08:21:14

[mdtest-hard-write]
t_start         = 2020-10-26 08:21:14
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /flash/datafiles/2020.10.26-08.05.37/mdtest-hard -x ./results/2020.10.26-08.05.37/mdtest-hard.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 7.055864
score           = 6.542162
t_delta         = 330.1941
t_end           = 2020-10-26 08:26:44

[find]
t_start         = 2020-10-26 08:26:44
exe             = ./pfind /flash/datafiles/2020.10.26-08.05.37 -newer ./results/2020.10.26-08.05.37/timestampfile -size 3901c -name *01* -C -q 50000
nproc           = 140
found           = 55896
total-files     = 12803002
score           = 665.439184
t_delta         = 19.4988
t_end           = 2020-10-26 08:27:03

[ior-easy-read]
t_start         = 2020-10-26 08:27:03
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-easy.stonewall -t 1M -b 9920000m -r -R -a IME
score           = 187.262238
t_delta         = 310.1218
t_end           = 2020-10-26 08:32:13

[mdtest-easy-stat]
t_start         = 2020-10-26 08:32:13
exe             = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /flash/datafiles/2020.10.26-08.05.37/mdtest-easy -x ./results/2020.10.26-08.05.37/mdtest-easy.stonewall -T -a IME
score           = 259.211637
t_delta         = 41.1936
t_end           = 2020-10-26 08:32:55

[ior-hard-read]
t_start         = 2020-10-26 08:32:55
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a IME
score           = 142.899363
t_delta         = 416.3154
t_end           = 2020-10-26 08:39:51

[mdtest-hard-stat]
t_start         = 2020-10-26 08:39:51
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /flash/datafiles/2020.10.26-08.05.37/mdtest-hard -x ./results/2020.10.26-08.05.37/mdtest-hard.stonewall -T -a IME
score           = 144.315536
t_delta         = 14.9572
t_end           = 2020-10-26 08:40:06

[mdtest-easy-delete]
t_start         = 2020-10-26 08:40:06
exe             = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /flash/datafiles/2020.10.26-08.05.37/mdtest-easy -x ./results/2020.10.26-08.05.37/mdtest-easy.stonewall -r -a POSIX
score           = 24.087414
t_delta         = 443.9772
t_end           = 2020-10-26 08:47:30

[mdtest-hard-read]
t_start         = 2020-10-26 08:47:30
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /flash/datafiles/2020.10.26-08.05.37/mdtest-hard -x ./results/2020.10.26-08.05.37/mdtest-hard.stonewall -E -X -a IME
score           = 174.357658
t_delta         = 12.3219
t_end           = 2020-10-26 08:47:42

[mdtest-hard-delete]
t_start         = 2020-10-26 08:47:42
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /flash/datafiles/2020.10.26-08.05.37/mdtest-hard -x ./results/2020.10.26-08.05.37/mdtest-hard.stonewall -r -a IME
score           = 20.565134
t_delta         = 104.3508
t_end           = 2020-10-26 08:49:27

[SCORE]
MD              = 68.546190
BW              = 176.229284
SCORE           = 109.908352 
hash            = 894133ED
; END 2020-10-26 08:49:27
result_summary
IO500 version io500-sc20-6-gf1c40e1e4342
[RESULT]       ior-easy-write      189.657067 GiB/s : time 310.793 seconds
[RESULT]    mdtest-easy-write       34.650264 kIOPS : time 307.870 seconds
[RESULT]       ior-hard-write      190.047767 GiB/s : time 313.013 seconds
[RESULT]    mdtest-hard-write        6.542162 kIOPS : time 330.194 seconds
[RESULT]                 find      665.439184 kIOPS : time 19.499 seconds
[RESULT]        ior-easy-read      187.262238 GiB/s : time 310.122 seconds
[RESULT]     mdtest-easy-stat      259.211637 kIOPS : time 41.194 seconds
[RESULT]        ior-hard-read      142.899363 GiB/s : time 416.315 seconds
[RESULT]     mdtest-hard-stat      144.315536 kIOPS : time 14.957 seconds
[RESULT]   mdtest-easy-delete       24.087414 kIOPS : time 443.977 seconds
[RESULT]     mdtest-hard-read      174.357658 kIOPS : time 12.322 seconds
[RESULT]   mdtest-hard-delete       20.565134 kIOPS : time 104.351 seconds
[SCORE] Bandwidth 176.229284 GiB/s : IOPS 68.546190 kiops : TOTAL 109.908352