Pengcheng Cloudbrain-II on Atlas 900

Institution Pengcheng Laboratory
Client Procs Per Node
Client Operating System KylinOS
Client Operating System Version v10sp1
Client Kernel Version 4.19.148

DATA SERVER

Storage Type NVMe-SSD
Volatile Memory 2TB
Storage Interface NVMe
Network Ethernet 100GbE
Software Version 0.95
OS Version KylinOS v10sp1

INFORMATION

Client Nodes 10
Client Total Procs 1,440

METADATA

Easy Write 10,096.61 kIOP/s
Easy Stat 24,879.73 kIOP/s
Easy Delete 8,894.57 kIOP/s
Hard Write 1,845.73 kIOP/s
Hard Read 3,594.17 kIOP/s
Hard Stat 18,555.25 kIOP/s
Hard Delete 2,891.54 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.
#
# The only parts of the script that may need to be modified are:
#  - setup_paths() to configure the binary locations and MPI parameters
#  - setup_directories() to create/tune the IOR/mdtest output directories
#
# Please visit https://vi4io.org/io500-info-creator/ to help generate the
# "system-information.txt" file, by pasting the output of the info-creator.
# This file contains details of your system hardware for your submission.

# Set the paths to the binaries and how to launch MPI jobs.
# If you ran ./prepare.sh successfully, then binaries are in ./bin/
function setup_paths {
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mpirun="mpirun"
  io500_mpiargs="--allow-run-as-root \
    --mca btl ^vader,tcp,openib,uct -mca pml ucx -mca routed direct \
    --hostfile hosts-client.txt \
    -N 144 \
    -x MADFS_CHUNK_SIZE=470080 \
    -x UCX_TLS=ud \
    -x UCX_NET_DEVICES=hns_0:1,hns_1:1 \
    -x LD_PRELOAD=$PWD/libmadfs_intercept.so"
}

# Set directories where benchmark files are created and where the results go.
# If you want to set up stripe tuning on your output directories or anything
# similar, then this is the right place to do it.
function setup_directories {
  local workdir
  local resultdir
  local ts

  mkdir -p $io500_workdir $io500_resultdir

  # Example commands to create output directories for Lustre.  Creating
  # top-level directories is allowed, but not the whole directory tree.
  #if (( $(lfs df $io500_workdir | grep -c MDT) > 1 )); then
  #  lfs setdirstripe -D -c -1 $io500_workdir
  #fi
  #lfs setstripe -c 1 $io500_workdir
  #mkdir $io500_workdir/ior-easy $io500_workdir/ior-hard
  #mkdir $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard
  #local osts=$(lfs df $io500_workdir | grep -c OST)
  # Try overstriping for ior-hard to improve scaling, or use wide striping
  #lfs setstripe -C $((osts * 4)) $io500_workdir/ior-hard ||
  #  lfs setstripe -c -1 $io500_workdir/ior-hard
  # Try to use DoM if available, otherwise use default for small files
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-easy || true #DoM?
  #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-hard || true #DoM?
}

# *****  YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE  *****
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

function run_benchmarks {
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_resultdir)
  local fname=$(basename ${io500_resultdir})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_resultdir
  tar czf $tarball -C $sourcedir $fname
  echo "Created result tarball $tarball"
}

function main {
  # These commands extract the 'datadir' and 'resultdir' from .ini file
  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # working directory where the test files will be created
  export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
    ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts

  setup_directories
  setup_paths
  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a 'system-information.txt' description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_resultdir
  fi

  create_tarball
}

main
ior-easy-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Wed Oct 28 01:42:43 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o ./out//2020.10.28-01.15.30/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results//2020.10.28-01.15.30/ior-easy.stonewall -t 470080 -b 47008000m -F -r -R -a POSIX
Machine             : Linux a01
TestID              : 0
StartTime           : Wed Oct 28 01:42:43 2020
Path                : ./out//2020.10.28-01.15.30/ior-easy/ior_file_easy.00000000
FS                  : 859.3 TiB   Used FS: 19.3%   Inodes: 268423659.1 Mi   Used Inodes: 100.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.10.28-01.15.30/ior-easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 1440
clients per node    : 144
repetitions         : 1
xfersize            : 470080 bytes
blocksize           : 44.83 TiB
aggregate filesize  : 64555.66 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 70979703275520000
WARNING: Stat() of aggregate file size      = 72586293811200
WARNING: Using actual aggregate bytes moved = 72586293811200
read      214705     478931     0.000003    48136192000 459.06     0.003881   322.41     0.005458   322.41     0   
ior-easy-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Wed Oct 28 01:16:54 2020
Command line        : ./ior -C -Q 1 -g -G 271 -k -e -o ./out//2020.10.28-01.15.30/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results//2020.10.28-01.15.30/ior-easy.stonewall -t 470080 -b 47008000m -F -w -D 300 -O stoneWallingWearOut=1 -a POSIX
Machine             : Linux a01
TestID              : 0
StartTime           : Wed Oct 28 01:16:54 2020
Path                : ./out//2020.10.28-01.15.30/ior-easy/ior_file_easy.00000000
FS                  : 859.3 TiB   Used FS: 5.1%   Inodes: 0.0 Mi   Used Inodes: inf%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.10.28-01.15.30/ior-easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 1440
clients per node    : 144
repetitions         : 1
xfersize            : 470080 bytes
blocksize           : 44.83 TiB
aggregate filesize  : 64555.66 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 105679 max: 107231 -- min data: 46.3 GiB mean data: 46.6 GiB time: 300.0s
WARNING: Expected aggregate file size       = 70979703275520000
WARNING: Stat() of aggregate file size      = 72586293811200
WARNING: Using actual aggregate bytes moved = 72586293811200
WARNING: Maybe caused by deadlineForStonewalling
write     223588     498746     0.000003    48136192000 459.06     0.000264   309.60     0.002361   309.60     0   
ior-hard-read
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Wed Oct 28 01:50:15 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o ./out//2020.10.28-01.15.30/ior-hard/file -O stoneWallingStatusFile=./results//2020.10.28-01.15.30/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a POSIX
Machine             : Linux a01
TestID              : 0
StartTime           : Wed Oct 28 01:50:15 2020
Path                : ./out//2020.10.28-01.15.30/ior-hard/file
FS                  : 859.3 TiB   Used FS: 19.3%   Inodes: 268423659.1 Mi   Used Inodes: 100.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.10.28-01.15.30/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 1440
clients per node    : 144
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 615.65 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 834059 max: 834059 -- min data: 36.5 GiB mean data: 36.5 GiB time: 492.8s
WARNING: Expected aggregate file size       = 676915200000000
WARNING: Stat() of aggregate file size      = 56458721479680
WARNING: Using actual aggregate bytes moved = 56458721479680
read      109239     2436785    492.87      45.91      45.91      0.014781   492.88     0.008165   492.89     0   
ior-hard-write
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Wed Oct 28 01:27:25 2020
Command line        : ./ior -C -Q 1 -g -G 27 -k -e -o ./out//2020.10.28-01.15.30/ior-hard/file -O stoneWallingStatusFile=./results//2020.10.28-01.15.30/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a POSIX
Machine             : Linux a01
TestID              : 0
StartTime           : Wed Oct 28 01:27:25 2020
Path                : ./out//2020.10.28-01.15.30/ior-hard/file
FS                  : 859.3 TiB   Used FS: 12.8%   Inodes: 268423659.1 Mi   Used Inodes: 100.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : ./out//2020.10.28-01.15.30/ior-hard/file
access              : single-shared-file
type                : independent
segments            : 10000000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 1440
clients per node    : 144
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 615.65 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 786451 max: 834059 -- min data: 34.4 GiB mean data: 35.0 GiB time: 300.0s
WARNING: Expected aggregate file size       = 676915200000000
WARNING: Stat() of aggregate file size      = 56458721479680
WARNING: Using actual aggregate bytes moved = 56458721479680
WARNING: Maybe caused by deadlineForStonewalling
write     168716     3763729    317.72      45.91      45.91      0.010665   319.11     0.018459   319.14     0   
mdtest-easy-delete
-- started at 10/28/2020 01:58:59 --

mdtest-3.4.0+dev was launched with 1440 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' './out//2020.10.28-01.15.30/mdtest-easy' '-x' './results//2020.10.28-01.15.30/mdtest-easy.stonewall' '-r' '-a' 'POSIX'
Path: /mnt/nfs/io500-wrj/io500-10node/out/2020.10.28-01.15.30
FS: 859.3 TiB   Used FS: 19.3%   Inodes: 268423659.1 Mi   Used Inodes: 100.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 144 for each phase.
1440 tasks, 1440000000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    8894624.347    8894571.034    8894601.626         16.636
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :       1618.254       1618.254       1618.254          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        362.339        362.337        362.338          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.001          0.001          0.001          0.000
-- finished at 10/28/2020 02:05:01 --

mdtest-easy-stat
-- started at 10/28/2020 01:48:05 --

mdtest-3.4.0+dev was launched with 1440 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' './out//2020.10.28-01.15.30/mdtest-easy' '-x' './results//2020.10.28-01.15.30/mdtest-easy.stonewall' '-T' '-a' 'POSIX'
Path: /mnt/nfs/io500-wrj/io500-10node/out/2020.10.28-01.15.30
FS: 859.3 TiB   Used FS: 19.3%   Inodes: 268423659.1 Mi   Used Inodes: 100.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 144 for each phase.
1440 tasks, 1440000000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :   24879894.172   24879725.035   24879813.242         46.858
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        129.537        129.536        129.537          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/28/2020 01:50:15 --

mdtest-easy-write
-- started at 10/28/2020 01:22:04 --

mdtest-3.4.0+dev was launched with 1440 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' './out//2020.10.28-01.15.30/mdtest-easy' '-x' './results//2020.10.28-01.15.30/mdtest-easy.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /mnt/nfs/io500-wrj/io500-10node/out/2020.10.28-01.15.30
FS: 859.3 TiB   Used FS: 12.8%   Inodes: 268423659.1 Mi   Used Inodes: 100.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 144 for each phase.
1440 tasks, 1440000000000 files
Continue stonewall hit min: 2090210 max: 2238093 avg: 2167904.2 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :   10096669.220   10096610.743   10096644.342         19.011
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA   10405859.033             NA
   Tree creation             :       4294.979       4294.979       4294.979          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        319.202        319.200        319.200          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.002             NA
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/28/2020 01:27:25 --

mdtest-hard-delete
-- started at 10/28/2020 02:07:45 --

mdtest-3.4.0+dev was launched with 1440 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' './out//2020.10.28-01.15.30/mdtest-hard' '-x' './results//2020.10.28-01.15.30/mdtest-hard.stonewall' '-r' '-a' 'POSIX'
Path: /mnt/nfs/io500-wrj/io500-10node/out/2020.10.28-01.15.30
FS: 859.3 TiB   Used FS: 19.3%   Inodes: 268423659.1 Mi   Used Inodes: 100.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 144 for each phase.
1440 tasks, 1440000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :    2891556.686    2891538.153    2891548.683          5.480
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :       7674.597       7674.597       7674.597          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        203.011        203.009        203.010          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/28/2020 02:11:08 --

mdtest-hard-read
-- started at 10/28/2020 02:05:01 --

mdtest-3.4.0+dev was launched with 1440 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' './out//2020.10.28-01.15.30/mdtest-hard' '-x' './results//2020.10.28-01.15.30/mdtest-hard.stonewall' '-E' '-X' '-a' 'POSIX'
Path: /mnt/nfs/io500-wrj/io500-10node/out/2020.10.28-01.15.30
FS: 859.3 TiB   Used FS: 19.3%   Inodes: 268423659.1 Mi   Used Inodes: 100.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 144 for each phase.
1440 tasks, 1440000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :    3594188.300    3594164.284    3594177.440          6.765
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        163.324        163.323        163.323          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/28/2020 02:07:45 --

mdtest-hard-stat
-- started at 10/28/2020 01:58:28 --

mdtest-3.4.0+dev was launched with 1440 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' './out//2020.10.28-01.15.30/mdtest-hard' '-x' './results//2020.10.28-01.15.30/mdtest-hard.stonewall' '-T' '-a' 'POSIX'
Path: /mnt/nfs/io500-wrj/io500-10node/out/2020.10.28-01.15.30
FS: 859.3 TiB   Used FS: 19.3%   Inodes: 268423659.1 Mi   Used Inodes: 100.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 144 for each phase.
1440 tasks, 1440000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :   18555404.765   18555221.358   18555311.707         40.455
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         31.636         31.636         31.636          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/28/2020 01:58:59 --

mdtest-hard-write
-- started at 10/28/2020 01:32:44 --

mdtest-3.4.0+dev was launched with 1440 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' './out//2020.10.28-01.15.30/mdtest-hard' '-x' './results//2020.10.28-01.15.30/mdtest-hard.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX'
Path: /mnt/nfs/io500-wrj/io500-10node/out/2020.10.28-01.15.30
FS: 859.3 TiB   Used FS: 18.8%   Inodes: 268423659.1 Mi   Used Inodes: 100.0%

Nodemap: 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2216 Shifting ranks by 144 for each phase.
1440 tasks, 1440000000 files
Continue stonewall hit min: 383348 max: 407648 avg: 394758.5 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    1845739.654    1845728.867    1845735.130          3.459
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    1893257.327             NA
   Tree creation             :       6528.694       6528.694       6528.694          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        318.039        318.037        318.038          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.251             NA
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 10/28/2020 01:38:04 --

result
version         = io500-sc20
config-hash     = 1ABD79E5
result-dir      = ./results//2020.10.28-01.15.30
; START 2020-10-28 01:16:54


[ior-easy-write]
t_start         = 2020-10-28 01:16:54
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o ./out//2020.10.28-01.15.30/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results//2020.10.28-01.15.30/ior-easy.stonewall -t 470080 -b 47008000m -F -w -D 300 -O stoneWallingWearOut=1 -a POSIX
throughput-stonewall = 312081.00
score           = 218.347788
t_delta         = 309.6121
t_end           = 2020-10-28 01:22:04

[mdtest-easy-write]
t_start         = 2020-10-28 01:22:04
exe             = ./mdtest -n 1000000000 -u -L -F -P -N 1 -d ./out//2020.10.28-01.15.30/mdtest-easy -x ./results//2020.10.28-01.15.30/mdtest-easy.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 10405.859033
score           = 10096.610886
t_delta         = 321.1164
t_end           = 2020-10-28 01:27:25

[timestamp]
t_start         = 2020-10-28 01:27:25
t_delta         = 0.0007
t_end           = 2020-10-28 01:27:25

[ior-hard-write]
t_start         = 2020-10-28 01:27:25
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o ./out//2020.10.28-01.15.30/ior-hard/file -O stoneWallingStatusFile=./results//2020.10.28-01.15.30/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a POSIX
throughput-stonewall = 227563.14
score           = 164.761712
t_delta         = 319.1493
t_end           = 2020-10-28 01:32:44

[mdtest-hard-write]
t_start         = 2020-10-28 01:32:44
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d ./out//2020.10.28-01.15.30/mdtest-hard -x ./results//2020.10.28-01.15.30/mdtest-hard.stonewall -C -Y -W 300 -a POSIX
rate-stonewall  = 1893.257327
score           = 1845.728978
t_delta         = 319.9371
t_end           = 2020-10-28 01:38:04

[find]
t_start         = 2020-10-28 01:38:04
exe             = ./pfind ./out//2020.10.28-01.15.30 -newer ./results//2020.10.28-01.15.30/timestampfile -size 3901c -name *01* -C -q 10000
found           = 32476368
total-files     = 3809868481
score           = 13675.860737
t_delta         = 278.5875
t_end           = 2020-10-28 01:42:43

[ior-easy-read]
t_start         = 2020-10-28 01:42:43
exe             = ./ior -C -Q 1 -g -G 271 -k -e -o ./out//2020.10.28-01.15.30/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results//2020.10.28-01.15.30/ior-easy.stonewall -t 470080 -b 47008000m -F -r -R -a POSIX
score           = 209.672740
t_delta         = 322.4175
t_end           = 2020-10-28 01:48:05

[mdtest-easy-stat]
t_start         = 2020-10-28 01:48:05
exe             = ./mdtest -n 1000000000 -u -L -F -P -N 1 -d ./out//2020.10.28-01.15.30/mdtest-easy -x ./results//2020.10.28-01.15.30/mdtest-easy.stonewall -T -a POSIX
score           = 24879.733021
t_delta         = 129.5448
t_end           = 2020-10-28 01:50:15

[ior-hard-read]
t_start         = 2020-10-28 01:50:15
exe             = ./ior -C -Q 1 -g -G 27 -k -e -o ./out//2020.10.28-01.15.30/ior-hard/file -O stoneWallingStatusFile=./results//2020.10.28-01.15.30/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a POSIX
score           = 106.679098
t_delta         = 492.9046
t_end           = 2020-10-28 01:58:27

[mdtest-hard-stat]
t_start         = 2020-10-28 01:58:27
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d ./out//2020.10.28-01.15.30/mdtest-hard -x ./results//2020.10.28-01.15.30/mdtest-hard.stonewall -T -a POSIX
score           = 18555.250866
t_delta         = 31.6433
t_end           = 2020-10-28 01:58:59

[mdtest-easy-delete]
t_start         = 2020-10-28 01:58:59
exe             = ./mdtest -n 1000000000 -u -L -F -P -N 1 -d ./out//2020.10.28-01.15.30/mdtest-easy -x ./results//2020.10.28-01.15.30/mdtest-easy.stonewall -r -a POSIX
score           = 8894.572633
t_delta         = 362.3474
t_end           = 2020-10-28 02:05:01

[mdtest-hard-read]
t_start         = 2020-10-28 02:05:01
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d ./out//2020.10.28-01.15.30/mdtest-hard -x ./results//2020.10.28-01.15.30/mdtest-hard.stonewall -E -X -a POSIX
score           = 3594.165535
t_delta         = 163.3312
t_end           = 2020-10-28 02:07:45

[mdtest-hard-delete]
t_start         = 2020-10-28 02:07:45
exe             = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d ./out//2020.10.28-01.15.30/mdtest-hard -x ./results//2020.10.28-01.15.30/mdtest-hard.stonewall -r -a POSIX
score           = 2891.539567
t_delta         = 203.0184
t_end           = 2020-10-28 02:11:08

[SCORE]
MD              = 7578.057274
BW              = 168.425017
SCORE           = 1129.749719 
hash            = 8509C8C8
; END 2020-10-28 02:11:31
result_summary
IO500 version io500-sc20
[RESULT]       ior-easy-write      218.347788 GiB/s : time 309.612 seconds
[RESULT]    mdtest-easy-write    10096.610886 kIOPS : time 321.116 seconds
[RESULT]       ior-hard-write      164.761712 GiB/s : time 319.149 seconds
[RESULT]    mdtest-hard-write     1845.728978 kIOPS : time 319.937 seconds
[RESULT]                 find    13675.860737 kIOPS : time 278.587 seconds
[RESULT]        ior-easy-read      209.672740 GiB/s : time 322.418 seconds
[RESULT]     mdtest-easy-stat    24879.733021 kIOPS : time 129.545 seconds
[RESULT]        ior-hard-read      106.679098 GiB/s : time 492.905 seconds
[RESULT]     mdtest-hard-stat    18555.250866 kIOPS : time 31.643 seconds
[RESULT]   mdtest-easy-delete     8894.572633 kIOPS : time 362.347 seconds
[RESULT]     mdtest-hard-read     3594.165535 kIOPS : time 163.331 seconds
[RESULT]   mdtest-hard-delete     2891.539567 kIOPS : time 203.018 seconds
[SCORE] Bandwidth 168.425017 GiB/s : IOPS 7578.057274 kiops : TOTAL 1129.749719