Endeavour

Institution Intel
Client Procs Per Node
Client Operating System Centos
Client Operating System Version 8.2
Client Kernel Version 4.18.0

DATA SERVER

Storage Type SSD + PMEM
Volatile Memory 195GB
Storage Interface SPDK. PMDK
Network Infiniband
Software Version 1.1.1
OS Version 7.8

INFORMATION

Client Nodes 10
Client Total Procs 640
Metadata Nodes 0
Metadata Storage Devices 0
Data Nodes 33
Data Storage Devices 8

METADATA

Easy Write 5,702.33 kIOP/s
Easy Stat 5,600.55 kIOP/s
Easy Delete 3,463.99 kIOP/s
Hard Write 1,253.01 kIOP/s
Hard Read 1,660.42 kIOP/s
Hard Stat 2,706.87 kIOP/s
Hard Delete 2,727.03 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.
#
# The only parts of the script that may need to be modified are:
#  - setup_paths() to configure the binary locations and MPI parameters
#  - setup_directories() to create/tune the IOR/mdtest output directories
#
# Please visit https://vi4io.org/io500-info-creator/ to help generate the
# "system-information.txt" file, by pasting the output of the info-creator.
# This file contains details of your system hardware for your submission.

# Set the paths to the binaries and how to launch MPI jobs.
# If you ran ./prepare.sh successfully, then binaries are in ./bin/
function setup_paths {
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mpirun="mpirun"
  io500_mpiargs="-np 640 --hostfile /home/schan15/work/gits/CLIENT/io500/config/daos_client_hostlist"
}

# Set directories where benchmark files are created and where the results go.
# If you want to set up stripe tuning on your output directories or anything
# similar, then this is the right place to do it.
function setup_directories {
  local workdir
  local resultdir
  local ts

  mkdir -p $io500_workdir $io500_resultdir

  # Example commands to create output directories for Lustre.  Creating
  # top-level directories is allowed, but not the whole directory tree.
  #if (( $(lfs df $io500_workdir | grep -c MDT) > 1 )); then
  #  lfs setdirstripe -D -c -1 $io500_workdir
  #fi
  #lfs setstripe -c 1 $io500_workdir
  #mkdir $io500_workdir/ior-easy $io500_workdir/ior-hard
  #mkdir $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard
  #local osts=$(lfs df $io500_workdir | grep -c OST)
  # Try overstriping for ior-hard to improve scaling, or use wide striping
  #lfs setstripe -C $((osts * 4)) $io500_workdir/ior-hard ||
  #  lfs setstripe -c -1 $io500_workdir/ior-hard
  # Try to use DoM if available, otherwise use default for small files
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-easy || true #DoM?
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-hard || true #DoM?
}

# *****  YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE  *****
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

function run_benchmarks {
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_resultdir)
  local fname=$(basename ${io500_resultdir})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_resultdir
  tar czf $tarball -C $sourcedir $fname
  echo "Created result tarball $tarball"
}

function main {
  # These commands extract the 'datadir' and 'resultdir' from .ini file
  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # working directory where the test files will be created
  export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts

  setup_directories
  setup_paths
  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a 'system-information.txt' description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_resultdir
  fi

  create_tarball
}

main
ior-easy-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Oct 30 10:04:47 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-easy.stonewall -t 256k -b 100000m -r -R -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.prefix=/tmp/dfuse
Machine             : Linux eia161
TestID              : 0
StartTime           : Fri Oct 30 10:04:48 2020
Path                : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy
FS                  : -1710806887.6 GiB   Used FS: 100.0%   Inodes: 134206810.4 Mi   Used Inodes: 83.9%

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 640
clients per node    : 64
repetitions         : 1
xfersize            : 262144 bytes
blocksize           : 97.66 GiB
aggregate filesize  : 61.04 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 67108864000000
WARNING: Stat() of aggregate file size      = 67057587322880
WARNING: Using actual aggregate bytes moved = 34291790643200
read      44380      179022     0.001827    102400000  256.00     6.18       730.71     0.000803   736.89     0   
ior-easy-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Oct 30 09:33:00 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-easy.stonewall -t 256k -b 100000m -w -D 300 -O stoneWallingWearOut=1 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.prefix=/tmp/dfuse
Machine             : Linux eia161
TestID              : 0
StartTime           : Fri Oct 30 09:33:02 2020
Path                : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy
FS                  : -1710806887.6 GiB   Used FS: 100.0%   Inodes: 134206810.4 Mi   Used Inodes: 83.9%

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy
access              : single-shared-file
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 640
clients per node    : 64
repetitions         : 1
xfersize            : 262144 bytes
blocksize           : 97.66 GiB
aggregate filesize  : 61.04 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 75520 max: 204395 -- min data: 18.4 GiB mean data: 24.8 GiB time: 300.0s
WARNING: Expected aggregate file size       = 67108864000000
WARNING: Stat() of aggregate file size      = 67057587322880
WARNING: Using actual aggregate bytes moved = 34291790643200
WARNING: Maybe caused by deadlineForStonewalling
write     54654      221676     0.000752    102400000  256.00     8.26       590.11     0.000597   598.37     0   
ior-hard-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Oct 30 10:22:27 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse
Machine             : Linux eia161
TestID              : 0
StartTime           : Fri Oct 30 10:22:28 2020
Path                : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file
FS                  : -1710806887.6 GiB   Used FS: 100.0%   Inodes: 134206810.4 Mi   Used Inodes: 83.9%

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 640
clients per node    : 64
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 273.62 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 492370 max: 492370 -- min data: 21.6 GiB mean data: 21.6 GiB time: 354.8s
WARNING: Expected aggregate file size       = 300851200000000
WARNING: Stat() of aggregate file size      = 14813010534400
WARNING: Using actual aggregate bytes moved = 14813010534400
read      39119      888108     354.81      45.91      45.91      6.30       354.82     0.001111   361.12     0   
ior-hard-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Oct 30 09:48:24 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse
Machine             : Linux eia161
TestID              : 0
StartTime           : Fri Oct 30 09:48:25 2020
Path                : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file
FS                  : -1710806887.6 GiB   Used FS: 100.0%   Inodes: 134206810.4 Mi   Used Inodes: 83.9%

Options: 
api                 : DFS
apiVersion          : DAOS
test filename       : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 640
clients per node    : 64
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 273.62 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 425969 max: 492370 -- min data: 18.6 GiB mean data: 19.8 GiB time: 300.0s
WARNING: Expected aggregate file size       = 300851200000000
WARNING: Stat() of aggregate file size      = 14813010534400
WARNING: Using actual aggregate bytes moved = 14813010534400
WARNING: Maybe caused by deadlineForStonewalling
write     41820      956760     307.70      45.91      45.91      8.44       329.36     0.000857   337.80     0   
mdtest-easy-delete
-- started at 10/30/2020 10:31:12 --

mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall' '-r' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB   Used FS: -nan%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    3463993.020    3463968.599    3463990.977          1.484
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.117          0.117          0.117          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        503.016        503.013        503.013          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          8.530          8.530          8.530          0.000
-- finished at 10/30/2020 10:39:43 --

mdtest-easy-stat
-- started at 10/30/2020 10:17:12 --

mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall' '-T' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB   Used FS: -nan%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    5600632.447    5600546.025    5600548.968          3.572
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        311.118        311.114        311.118          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/30/2020 10:22:23 --

mdtest-easy-write
-- started at 10/30/2020 09:43:08 --

mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall' '-C' '-Y' '-W' '300' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB   Used FS: -nan%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files
Continue stonewall hit min: 2658635 max: 2722551 avg: 2696152.2 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    5702340.422    5702331.857    5702335.773          2.119
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    5751786.562             NA
   Tree creation             :          0.167          0.167          0.167          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        305.565        305.564        305.565          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.000             NA
   Tree creation             :          5.976          5.976          5.976          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/30/2020 09:48:20 --

mdtest-hard-delete
-- started at 10/30/2020 10:43:58 --

mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall' '-r' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB   Used FS: -nan%   Inodes: 4427640616128.0 Mi   Used Inodes: 100.0%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    2727036.292    2727023.269    2727028.932          2.802
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.001          0.001          0.001          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        147.332        147.331        147.332          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :        775.443        775.443        775.443          0.000
-- finished at 10/30/2020 10:59:21 --

mdtest-hard-read
-- started at 10/30/2020 10:39:50 --

mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall' '-E' '-X' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB   Used FS: -nan%   Inodes: 0.0 Mi   Used Inodes: -inf%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :    1660422.283    1660393.861    1660420.800          1.636
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        241.977        241.973        241.973          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/30/2020 10:43:52 --

mdtest-hard-stat
-- started at 10/30/2020 10:28:37 --

mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall' '-T' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB   Used FS: -nan%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    2706871.411    2706861.567    2706866.759          2.270
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        148.429        148.429        148.429          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/30/2020 10:31:05 --

mdtest-hard-write
-- started at 10/30/2020 09:54:10 --

mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall' '-C' '-Y' '-W' '300' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB   Used FS: -nan%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank   0 Line  2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files
Continue stonewall hit min: 576207 max: 627777 avg: 589105.8 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    1253006.906    1253005.141    1253005.929          0.449
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    1256729.203             NA
   Tree creation             :         38.679         38.679         38.679          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        320.651        320.650        320.651          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.007             NA
   Tree creation             :          0.026          0.026          0.026          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/30/2020 09:59:31 --

result
version         = io500-sc20_v3-1-g4c17ac501883-12
config-hash     = 3CE19019
result-dir      = /tmp/dfuse/results/2020.10.30-09.32.49
; START 2020-10-30 09:33:00


[ior-easy-write]
t_start         = 2020-10-30 09:33:00
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-easy.stonewall -t 256k -b 100000m -w -D 300 -O stoneWallingWearOut=1 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.prefix=/tmp/dfuse
throughput-stonewall = 17001.62
score           = 53.372671
t_delta         = 606.0921
t_end           = 2020-10-30 09:43:06

[mdtest-easy-write]
t_start         = 2020-10-30 09:43:06
exe             = ./mdtest -n 10000000 -u -L -F -P -N 1 -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall -C -Y -W 300 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
rate-stonewall  = 5751.786562
score           = 5702.333575
t_delta         = 317.7481
t_end           = 2020-10-30 09:48:24

[timestamp]
t_start         = 2020-10-30 09:48:24
t_delta         = 0.0242
t_end           = 2020-10-30 09:48:24

[ior-hard-write]
t_start         = 2020-10-30 09:48:24
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse
throughput-stonewall = 23994.88
score           = 40.839710
t_delta         = 344.2002
t_end           = 2020-10-30 09:54:08

[mdtest-hard-write]
t_start         = 2020-10-30 09:54:08
exe             = ./mdtest -n 10000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall -C -Y -W 300 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
rate-stonewall  = 1256.729203
score           = 1253.005913
t_delta         = 327.0532
t_end           = 2020-10-30 09:59:35

[find]
t_start         = 2020-10-30 09:59:35
exe             = ./pfind /tmp/dfuse/datafiles/2020.10.30-09.32.49 -newer /tmp/dfuse/results/2020.10.30-09.32.49/timestampfile -size 3901c -name *01* -C -q 10000
nproc           = 640
found           = 12763278
total-files     = 843723591
score           = 2714.238117
t_delta         = 310.9214
t_end           = 2020-10-30 10:04:46

[ior-easy-read]
t_start         = 2020-10-30 10:04:46
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-easy.stonewall -t 256k -b 100000m -r -R -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.prefix=/tmp/dfuse
score           = 43.340062
t_delta         = 743.4720
t_end           = 2020-10-30 10:17:10

[mdtest-easy-stat]
t_start         = 2020-10-30 10:17:10
exe             = ./mdtest -n 10000000 -u -L -F -P -N 1 -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall -T -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
score           = 5600.548081
t_delta         = 317.3790
t_end           = 2020-10-30 10:22:27

[ior-hard-read]
t_start         = 2020-10-30 10:22:27
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse
score           = 38.202499
t_delta         = 367.5066
t_end           = 2020-10-30 10:28:35

[mdtest-hard-stat]
t_start         = 2020-10-30 10:28:35
exe             = ./mdtest -n 10000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall -T -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
score           = 2706.865398
t_delta         = 154.8284
t_end           = 2020-10-30 10:31:10

[mdtest-easy-delete]
t_start         = 2020-10-30 10:31:10
exe             = ./mdtest -n 10000000 -u -L -F -P -N 1 -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall -r -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
score           = 3463.991234
t_delta         = 518.0797
t_end           = 2020-10-30 10:39:48

[mdtest-hard-read]
t_start         = 2020-10-30 10:39:48
exe             = ./mdtest -n 10000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall -E -X -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
score           = 1660.421548
t_delta         = 248.4449
t_end           = 2020-10-30 10:43:56

[mdtest-hard-delete]
t_start         = 2020-10-30 10:43:56
exe             = ./mdtest -n 10000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall -r -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
score           = 2727.029792
t_delta         = 929.2505
t_end           = 2020-10-30 10:59:26

[SCORE]
MD              = 2870.644995
BW              = 43.585870
SCORE           = 353.722434 
hash            = 701ABADD
; END 2020-10-30 10:59:27
result_summary
IO500 version io500-sc20_v3-1-g4c17ac501883-12
[RESULT]       ior-easy-write       53.372671 GiB/s : time 606.092 seconds
[RESULT]    mdtest-easy-write     5702.333575 kIOPS : time 317.748 seconds
[RESULT]       ior-hard-write       40.839710 GiB/s : time 344.200 seconds
[RESULT]    mdtest-hard-write     1253.005913 kIOPS : time 327.053 seconds
[RESULT]                 find     2714.238117 kIOPS : time 310.921 seconds
[RESULT]        ior-easy-read       43.340062 GiB/s : time 743.472 seconds
[RESULT]     mdtest-easy-stat     5600.548081 kIOPS : time 317.379 seconds
[RESULT]        ior-hard-read       38.202499 GiB/s : time 367.507 seconds
[RESULT]     mdtest-hard-stat     2706.865398 kIOPS : time 154.828 seconds
[RESULT]   mdtest-easy-delete     3463.991234 kIOPS : time 518.080 seconds
[RESULT]     mdtest-hard-read     1660.421548 kIOPS : time 248.445 seconds
[RESULT]   mdtest-hard-delete     2727.029792 kIOPS : time 929.251 seconds
[SCORE] Bandwidth 43.585870 GiB/s : IOPS 2870.644995 kiops : TOTAL 353.722434