Venus2

Institution National Supercomputer Center in Guangzhou
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.3
Client Kernel Version 3.10.0-514.el7.x86_64

DATA SERVER

Storage Type NVMe
Volatile Memory 64G
Storage Interface NVMe
Network Ethernet
Software Version 1.0
OS Version 7.3

INFORMATION

Client Nodes 10
Client Total Procs 260
Metadata Nodes 20
Metadata Storage Devices 1
Data Nodes 38
Data Storage Devices 38

METADATA

Easy Write 316.99 kIOP/s
Easy Stat 1,344.21 kIOP/s
Easy Delete 643.98 kIOP/s
Hard Write 224.26 kIOP/s
Hard Read 351.72 kIOP/s
Hard Stat 1,237.39 kIOP/s
Hard Delete 552.90 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.

function setup_paths {
  # Set the paths to the binaries and how to launch MPI jobs.
  # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="--hostfile hostfile -np 260"
}

function setup_directories {
  local workdir
  local resultdir
  local ts

  # set directories where benchmark files are created and where the results go
  # If you want to set up stripe tuning on your output directories or anything
  # similar, then this is the right place to do it.  This creates the output
  # directories for both the app run and the script run.

  timestamp=$(date +%Y.%m.%d-%H.%M.%S)           # create a uniquifier
  [ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # directory where the data will be stored
  workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
  io500_workdir=$workdir-scr
  [ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
	ts="$timestamp" || ts="io500"
  # the directory where the output results will be kept
  resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
  io500_result_dir=$resultdir-scr

  mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
  mkdir -p $workdir-scr/ior_hard $workdir-app/ior-hard
  mkdir -p $workdir-scr/ior_easy $workdir-app/ior-easy
  mkdir -p $workdir-scr/mdt_hard $workdir-app/mdtest-hard
  mkdir -p $workdir-scr/mdt_easy $workdir-app/mdtest-easy
  llfs-ctl --cfgFile=/etc/llfs/llfs-client.conf --setpattern --numtargets=152 --chunksize=1024k $workdir-scr/ior_easy >> /etc/null
  llfs-ctl --cfgFile=/etc/llfs/llfs-client.conf --setpattern --numtargets=152 --chunksize=256k $workdir-scr/ior_hard >> /etc/null
  llfs-ctl --cfgFile=/etc/llfs/llfs-client.conf --setpattern --numtargets=152 --chunksize=1024k $workdir-app/ior-easy >> /etc/null
  llfs-ctl --cfgFile=/etc/llfs/llfs-client.conf --setpattern --numtargets=152 --chunksize=256k $workdir-app/ior-hard >> /etc/null
}

# you should not edit anything below this line
set -eo pipefail  # better error handling

io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
  echo "error: ini file must be specified.  usage: $0 "
  exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
  echo "error: ini file '$io500_ini' not found or empty"
  exit 2
fi

function get_ini_section_param() {
  local section="$1"
  local param="$2"
  local inside=false

  while read LINE; do
    LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
    $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
    [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
    ! $inside && continue
    #echo $LINE | awk -F = "/^$param/ { print \$2 }"
    if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
      # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
      echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
      return
    fi
  done < $io500_ini
  echo ""
}

function get_ini_param() {
  local section="$1"
  local param="$2"
  local default="$3"

  # try and get the most-specific param first, then more generic params
  val=$(get_ini_section_param $section $param)
  [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
  [ -n "$val" ] || val="$(get_ini_section_param global $param)"

  echo "${val:-$default}" |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}

function get_ini_run_param() {
  local section="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param $section noRun)

  # logic is reversed from "noRun=TRUE" to "run=False"
  [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}

function get_ini_global_param() {
  local param="$1"
  local default="$2"
  local val

  val=$(get_ini_section_param global $param |
  	sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')

  echo "${val:-$default}"
}

# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
#io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.  Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional

  run_benchmarks

  if [[ ! -s "system-information.txt" ]]; then
    echo "Warning: please create a system-information.txt description by"
    echo "copying the information from https://vi4io.org/io500-info-creator/"
  else
    cp "system-information.txt" $io500_result_dir
  fi

  create_tarball
}

function setup_ior_easy {
  local params

  io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
  val=$(get_ini_param ior-easy API POSIX)
  [ -n "$val" ] && params+=" -a $val"
  val="$(get_ini_param ior-easy transferSize)"
  [ -n "$val" ] && params+=" -t $val"
  val="$(get_ini_param ior-easy hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-easy posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_easy_params="$params"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves

  val=$(get_ini_param mdtest-easy n 1000000)
  [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
  val=$(get_ini_param mdtest-easy API POSIX)
  [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
  val=$(get_ini_param mdtest-easy posix.odirect)
  [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
  echo -n ""
}

function setup_ior_hard {
  local params

  io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
  io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
  val="$(get_ini_param ior-hard hintsFileName)"
  [ -n "$val" ] && params+=" -U $val"
  val="$(get_ini_param ior-hard posix.odirect)"
  [ "$val" = "True" ] && params+=" --posix.odirect"
  val="$(get_ini_param ior-easy verbosity)"
  if [ -n "$val" ]; then
    for i in $(seq $val); do
      params+=" -v"
    done
  fi
  io500_ior_hard_api_specific_options="$params"
  echo -n ""
}

function setup_mdt_hard {
  val=$(get_ini_param mdtest-hard n 1000000)
  [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
  io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
  io500_mdtest_hard_api_specific_options=""
  echo -n ""
}

function setup_find {
  io500_find_mpi="False"
  val="$(get_ini_param find external-script)"
  [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
    io500_find_cmd="$val"
  # uses stonewalling, run pfind
  io500_find_cmd_args="$(get_ini_param find external-extra-args)"
  echo -n ""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  local app_first=$((RANDOM % 100))
  local app_rc=0

  # run the app and C version in random order to try and avoid bias
  (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?

  # Important: source the io500_fixed.sh script.  Do not change it. If you
  # discover a need to change it, please email the mailing list to discuss.
  source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
    tee $io500_result_dir/io-500-summary.$timestamp.txt

  (( $app_first >= 50 )) && return $app_rc

  echo "The io500.sh was run"
  echo
  echo "Running the C version of the benchmark now"
  # run the app and C version in random order to try and avoid bias
  $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}

create_tarball() {
  local sourcedir=$(dirname $io500_result_dir)
  local fname=$(basename ${io500_result_dir%-scr})
  local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz

  cp -v $0 $io500_ini $io500_result_dir
  tar czf $tarball -C $sourcedir $fname-{app,scr}
  echo "Created result tarball $tarball"
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
  # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
  # EXAMPLE:
  # io500_info_system_name='xxx'
  # DO NOT ADD IT HERE
  :
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Jul 10 18:22:17 2020
Command line        : /root/io500-app/bin/ior -r -R -a POSIX -t 1m --posix.odirect -b 59392m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /mnt/llfs//2020.07.10-17.14.40-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/llfs//2020.07.10-17.14.40-scr/ior_easy/stonewall
Machine             : Linux cn16609-gn0
TestID              : 0
StartTime           : Fri Jul 10 18:22:17 2020
Path                : /mnt/llfs/2020.07.10-17.14.40-scr/ior_easy
FS                  : 51.1 TiB   Used FS: 54.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/llfs//2020.07.10-17.14.40-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 260
clients per node    : 26
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 58 GiB
aggregate filesize  : 14.73 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 16192026705920.
WARNING: Stat() of aggregate file size      = 0.
WARNING: Using actual aggregate bytes moved = 11716809195520.
read      65971      66006      0.002850    60817408   1024.00    0.063422   169.29     0.029660   169.38     0   
Max Read:  65970.95 MiB/sec (69175.55 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        65970.95   65970.95   65970.95       0.00   65970.95   65970.95   65970.95       0.00  169.37789         NA            NA     0    260  26    1   1     1        1         0    0      1 62277025792  1048576 11174020.0 POSIX      0
Finished            : Fri Jul 10 18:25:06 2020
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Jul 10 18:00:22 2020
Command line        : /root/io500-app/bin/ior -w -a POSIX -t 1m --posix.odirect -b 59392m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /mnt/llfs//2020.07.10-17.14.40-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/llfs//2020.07.10-17.14.40-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux cn16609-gn0
TestID              : 0
StartTime           : Fri Jul 10 18:00:22 2020
Path                : /mnt/llfs/2020.07.10-17.14.40-scr/ior_easy
FS                  : 51.1 TiB   Used FS: 28.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/llfs//2020.07.10-17.14.40-scr/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 260
clients per node    : 26
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 58 GiB
aggregate filesize  : 14.73 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 42489 max: 42977 -- min data: 41.5 GiB mean data: 41.7 GiB time: 300.1s
WARNING: Expected aggregate file size       = 16192026705920.
WARNING: Stat() of aggregate file size      = 0.
WARNING: Using actual aggregate bytes moved = 11716809195520.
WARNING: maybe caused by deadlineForStonewalling
write     36788      36802      0.005075    60817408   1024.00    0.099011   303.63     0.016726   303.74     0   
Max Write: 36787.53 MiB/sec (38574.52 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       36787.53   36787.53   36787.53       0.00   36787.53   36787.53   36787.53       0.00  303.74481     300.14      37008.98     0    260  26    1   1     1        1         0    0      1 62277025792  1048576 11174020.0 POSIX      0
Finished            : Fri Jul 10 18:05:25 2020
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Jul 10 18:26:30 2020
Command line        : /root/io500-app/bin/ior -r -R -s 420000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /mnt/llfs//2020.07.10-17.14.40-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/llfs//2020.07.10-17.14.40-scr/ior_hard/stonewall
Machine             : Linux cn16609-gn0
TestID              : 0
StartTime           : Fri Jul 10 18:26:30 2020
Path                : /mnt/llfs/2020.07.10-17.14.40-scr/ior_hard
FS                  : 51.1 TiB   Used FS: 54.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/llfs//2020.07.10-17.14.40-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 420000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 260
clients per node    : 26
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 4.67 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 5133273600000.
WARNING: Stat() of aggregate file size      = 3048736745600.
WARNING: Using actual aggregate bytes moved = 3048736745600.
read      34182      765429     84.70       45.91      45.91      0.276866   84.73      0.040972   85.06      0   
Max Read:  34181.66 MiB/sec (35842.07 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        34181.66   34181.66   34181.66       0.00  762467.37  762467.37  762467.37       0.00   85.06030         NA            NA     0    260  26    1   0     1        1         0    0 420000    47008    47008 2907502.0 POSIX      0
Finished            : Fri Jul 10 18:27:55 2020
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Jul 10 18:11:07 2020
Command line        : /root/io500-app/bin/ior -w -s 420000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /mnt/llfs//2020.07.10-17.14.40-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/llfs//2020.07.10-17.14.40-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux cn16609-gn0
TestID              : 0
StartTime           : Fri Jul 10 18:11:07 2020
Path                : /mnt/llfs/2020.07.10-17.14.40-scr/ior_hard
FS                  : 51.1 TiB   Used FS: 49.4%   Inodes: 0.0 Mi   Used Inodes: -nan%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /mnt/llfs//2020.07.10-17.14.40-scr/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 420000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 260
clients per node    : 26
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 4.67 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 236502 max: 249445 -- min data: 10.4 GiB mean data: 10.6 GiB time: 300.0s
WARNING: Expected aggregate file size       = 5133273600000.
WARNING: Stat() of aggregate file size      = 0.
WARNING: Using actual aggregate bytes moved = 3048736745600.
WARNING: maybe caused by deadlineForStonewalling
write     9365       209168     308.36      45.91      45.91      0.373330   310.07     0.004178   310.45     0   
Max Write: 9365.50 MiB/sec (9820.44 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        9365.50    9365.50    9365.50       0.00  208909.92  208909.92  208909.92       0.00  310.44816     300.04       9437.35     0    260  26    1   0     1        1         0    0 420000    47008    47008 2907502.0 POSIX      0
Finished            : Fri Jul 10 18:16:17 2020
mdtest_easy_delete
-- started at 07/10/2020 18:28:59 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 10 node(s)
Command line used: /root/io500-app/bin/mdtest '-r' '-F' '-P' '-d' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_easy' '-n' '550000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_easy-stonewall' '-N' '1'
Path: /mnt/llfs/2020.07.10-17.14.40-scr
FS: 51.1 TiB   Used FS: 54.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 26 for each phase.
260 tasks, 143000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     643976.603     643975.852     643976.164          0.142
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.931          2.931          2.931          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        165.621        165.621        165.621          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.341          0.341          0.341          0.000
-- finished at 07/10/2020 18:31:55 --

mdtest_easy_stat
-- started at 07/10/2020 18:25:08 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 10 node(s)
Command line used: /root/io500-app/bin/mdtest '-T' '-F' '-P' '-d' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_easy' '-n' '550000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_easy-stonewall' '-N' '1'
Path: /mnt/llfs/2020.07.10-17.14.40-scr
FS: 51.1 TiB   Used FS: 54.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 26 for each phase.
260 tasks, 143000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    1344214.622    1344205.764    1344212.888          2.253
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         79.345         79.344         79.345          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/10/2020 18:26:28 --

mdtest_easy_write
-- started at 07/10/2020 18:05:28 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 10 node(s)
Command line used: /root/io500-app/bin/mdtest '-C' '-F' '-P' '-d' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_easy' '-n' '550000' '-u' '-L' '-a' 'POSIX' '--posix.odirect' '-x' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /mnt/llfs/2020.07.10-17.14.40-scr
FS: 51.1 TiB   Used FS: 49.4%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 26 for each phase.
260 tasks, 143000000 files
Continue stonewall hit min: 360405 max: 410215 avg: 378477.3 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     316992.764     316992.119     316992.605          0.155
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     328249.627             NA
   Tree creation             :          4.113          4.113          4.113          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        336.462        336.462        336.462          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.784             NA
   Tree creation             :          0.243          0.243          0.243          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/10/2020 18:11:05 --

mdtest_hard_delete
-- started at 07/10/2020 18:35:30 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 10 node(s)
Command line used: /root/io500-app/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_hard' '-n' '400000' '-x' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/llfs/2020.07.10-17.14.40-scr
FS: 51.1 TiB   Used FS: 54.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 26 for each phase.
260 tasks, 104000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     552902.390     552901.790     552902.145          0.126
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.072          0.072          0.072          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        131.947        131.947        131.947          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :         13.860         13.860         13.860          0.000
-- finished at 07/10/2020 18:37:55 --

mdtest_hard_read
-- started at 07/10/2020 18:31:59 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 10 node(s)
Command line used: /root/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_hard' '-n' '400000' '-x' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/llfs/2020.07.10-17.14.40-scr
FS: 51.1 TiB   Used FS: 54.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 26 for each phase.
260 tasks, 104000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     351722.471     351722.189     351722.329          0.057
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        207.419        207.419        207.419          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/10/2020 18:35:26 --

mdtest_hard_stat
-- started at 07/10/2020 18:27:57 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 10 node(s)
Command line used: /root/io500-app/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_hard' '-n' '400000' '-x' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /mnt/llfs/2020.07.10-17.14.40-scr
FS: 51.1 TiB   Used FS: 54.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 26 for each phase.
260 tasks, 104000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    1237390.384    1237387.257    1237388.878          0.613
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         58.958         58.958         58.958          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/10/2020 18:28:56 --

mdtest_hard_write
-- started at 07/10/2020 18:16:20 --

mdtest-3.3.0+dev was launched with 260 total task(s) on 10 node(s)
Command line used: /root/io500-app/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_hard' '-n' '400000' '-x' '/mnt/llfs//2020.07.10-17.14.40-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /mnt/llfs/2020.07.10-17.14.40-scr
FS: 51.1 TiB   Used FS: 54.8%   Inodes: 0.0 Mi   Used Inodes: -nan%

Nodemap: 11111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2166 Shifting ranks by 26 for each phase.
260 tasks, 104000000 files
Continue stonewall hit min: 256554 max: 280592 avg: 270692.4 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     224261.139     224260.982     224261.059          0.033
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     234586.618             NA
   Tree creation             :        814.428        814.428        814.428          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        325.308        325.308        325.308          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.017             NA
   Tree creation             :          0.001          0.001          0.001          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 07/10/2020 18:21:45 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               35.926 GiB/s : time 303.63 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              316.993 kiops : time 336.46 seconds
[RESULT] BW   phase 2            ior_hard_write                9.146 GiB/s : time 310.07 seconds
[RESULT] IOPS phase 2         mdtest_hard_write              224.261 kiops : time 325.31 seconds
[RESULT] IOPS phase 3                      find             6038.830 kiops : time  29.74 seconds
[RESULT] BW   phase 3             ior_easy_read               64.425 GiB/s : time 169.29 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             1344.210 kiops : time  79.35 seconds
[RESULT] BW   phase 4             ior_hard_read               33.381 GiB/s : time  84.73 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat             1237.390 kiops : time  58.96 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              643.977 kiops : time 165.62 seconds
[RESULT] IOPS phase 7          mdtest_hard_read              351.722 kiops : time 207.42 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete              552.902 kiops : time 148.73 seconds
[SCORE] Bandwidth 28.9929 GiB/s : IOPS 739.488 kiops : TOTAL 146.424