- io500
-
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.
#
# The only parts of the script that may need to be modified are:
# - setup_paths() to configure the binary locations and MPI parameters
# - setup_directories() to create/tune the IOR/mdtest output directories
#
# Please visit https://vi4io.org/io500-info-creator/ to help generate the
# "system-information.txt" file, by pasting the output of the info-creator.
# This file contains details of your system hardware for your submission.
# Set the paths to the binaries and how to launch MPI jobs.
# If you ran ./prepare.sh successfully, then binaries are in ./bin/
function setup_paths {
io500_ior_cmd=$PWD/bin/ior
io500_mdtest_cmd=$PWD/bin/mdtest
io500_mpirun="mpirun"
io500_mpiargs="-np 640 --hostfile /home/schan15/work/gits/CLIENT/io500/config/daos_client_hostlist"
}
# Set directories where benchmark files are created and where the results go.
# If you want to set up stripe tuning on your output directories or anything
# similar, then this is the right place to do it.
function setup_directories {
local workdir
local resultdir
local ts
mkdir -p $io500_workdir $io500_resultdir
# Example commands to create output directories for Lustre. Creating
# top-level directories is allowed, but not the whole directory tree.
#if (( $(lfs df $io500_workdir | grep -c MDT) > 1 )); then
# lfs setdirstripe -D -c -1 $io500_workdir
#fi
#lfs setstripe -c 1 $io500_workdir
#mkdir $io500_workdir/ior-easy $io500_workdir/ior-hard
#mkdir $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard
#local osts=$(lfs df $io500_workdir | grep -c OST)
# Try overstriping for ior-hard to improve scaling, or use wide striping
#lfs setstripe -C $((osts * 4)) $io500_workdir/ior-hard ||
# lfs setstripe -c -1 $io500_workdir/ior-hard
# Try to use DoM if available, otherwise use default for small files
#lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-easy || true #DoM?
#lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-hard || true #DoM?
}
# ***** YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE *****
set -eo pipefail # better error handling
io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
echo "error: ini file must be specified. usage: $0 "
exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
echo "error: ini file '$io500_ini' not found or empty"
exit 2
fi
function get_ini_section_param() {
local section="$1"
local param="$2"
local inside=false
while read LINE; do
LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
$inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
[[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
! $inside && continue
#echo $LINE | awk -F = "/^$param/ { print \$2 }"
if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
# echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
return
fi
done < $io500_ini
echo ""
}
function get_ini_global_param() {
local param="$1"
local default="$2"
local val
val=$(get_ini_section_param global $param |
sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')
echo "${val:-$default}"
}
function run_benchmarks {
$io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}
create_tarball() {
local sourcedir=$(dirname $io500_resultdir)
local fname=$(basename ${io500_resultdir})
local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz
cp -v $0 $io500_ini $io500_resultdir
tar czf $tarball -C $sourcedir $fname
echo "Created result tarball $tarball"
}
function main {
# These commands extract the 'datadir' and 'resultdir' from .ini file
timestamp=$(date +%Y.%m.%d-%H.%M.%S) # create a uniquifier
[ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
ts="$timestamp" || ts="io500"
# working directory where the test files will be created
export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
[ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
ts="$timestamp" || ts="io500"
# the directory where the output results will be kept
export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
setup_directories
setup_paths
run_benchmarks
if [[ ! -s "system-information.txt" ]]; then
echo "Warning: please create a 'system-information.txt' description by"
echo "copying the information from https://vi4io.org/io500-info-creator/"
else
cp "system-information.txt" $io500_resultdir
fi
create_tarball
}
main
- ior-easy-read
-
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began : Fri Oct 30 10:04:47 2020
Command line : ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-easy.stonewall -t 256k -b 100000m -r -R -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.prefix=/tmp/dfuse
Machine : Linux eia161
TestID : 0
StartTime : Fri Oct 30 10:04:48 2020
Path : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy
FS : -1710806887.6 GiB Used FS: 100.0% Inodes: 134206810.4 Mi Used Inodes: 83.9%
Options:
api : DFS
apiVersion : DAOS
test filename : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy
access : single-shared-file
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 10
tasks : 640
clients per node : 64
repetitions : 1
xfersize : 262144 bytes
blocksize : 97.66 GiB
aggregate filesize : 61.04 TiB
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
WARNING: Expected aggregate file size = 67108864000000
WARNING: Stat() of aggregate file size = 67057587322880
WARNING: Using actual aggregate bytes moved = 34291790643200
read 44380 179022 0.001827 102400000 256.00 6.18 730.71 0.000803 736.89 0
- ior-easy-write
-
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began : Fri Oct 30 09:33:00 2020
Command line : ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-easy.stonewall -t 256k -b 100000m -w -D 300 -O stoneWallingWearOut=1 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.prefix=/tmp/dfuse
Machine : Linux eia161
TestID : 0
StartTime : Fri Oct 30 09:33:02 2020
Path : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy
FS : -1710806887.6 GiB Used FS: 100.0% Inodes: 134206810.4 Mi Used Inodes: 83.9%
Options:
api : DFS
apiVersion : DAOS
test filename : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy
access : single-shared-file
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 10
tasks : 640
clients per node : 64
repetitions : 1
xfersize : 262144 bytes
blocksize : 97.66 GiB
aggregate filesize : 61.04 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 75520 max: 204395 -- min data: 18.4 GiB mean data: 24.8 GiB time: 300.0s
WARNING: Expected aggregate file size = 67108864000000
WARNING: Stat() of aggregate file size = 67057587322880
WARNING: Using actual aggregate bytes moved = 34291790643200
WARNING: Maybe caused by deadlineForStonewalling
write 54654 221676 0.000752 102400000 256.00 8.26 590.11 0.000597 598.37 0
- ior-hard-read
-
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began : Fri Oct 30 10:22:27 2020
Command line : ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse
Machine : Linux eia161
TestID : 0
StartTime : Fri Oct 30 10:22:28 2020
Path : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file
FS : -1710806887.6 GiB Used FS: 100.0% Inodes: 134206810.4 Mi Used Inodes: 83.9%
Options:
api : DFS
apiVersion : DAOS
test filename : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file
access : single-shared-file
type : independent
segments : 10000000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 10
tasks : 640
clients per node : 64
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 273.62 TiB
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 492370 max: 492370 -- min data: 21.6 GiB mean data: 21.6 GiB time: 354.8s
WARNING: Expected aggregate file size = 300851200000000
WARNING: Stat() of aggregate file size = 14813010534400
WARNING: Using actual aggregate bytes moved = 14813010534400
read 39119 888108 354.81 45.91 45.91 6.30 354.82 0.001111 361.12 0
- ior-hard-write
-
IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O
Began : Fri Oct 30 09:48:24 2020
Command line : ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse
Machine : Linux eia161
TestID : 0
StartTime : Fri Oct 30 09:48:25 2020
Path : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file
FS : -1710806887.6 GiB Used FS: 100.0% Inodes: 134206810.4 Mi Used Inodes: 83.9%
Options:
api : DFS
apiVersion : DAOS
test filename : /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file
access : single-shared-file
type : independent
segments : 10000000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 10
tasks : 640
clients per node : 64
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 273.62 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 425969 max: 492370 -- min data: 18.6 GiB mean data: 19.8 GiB time: 300.0s
WARNING: Expected aggregate file size = 300851200000000
WARNING: Stat() of aggregate file size = 14813010534400
WARNING: Using actual aggregate bytes moved = 14813010534400
WARNING: Maybe caused by deadlineForStonewalling
write 41820 956760 307.70 45.91 45.91 8.44 329.36 0.000857 337.80 0
- mdtest-easy-delete
-
-- started at 10/30/2020 10:31:12 --
mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall' '-r' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB Used FS: -nan% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 3463993.020 3463968.599 3463990.977 1.484
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.117 0.117 0.117 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 503.016 503.013 503.013 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 8.530 8.530 8.530 0.000
-- finished at 10/30/2020 10:39:43 --
- mdtest-easy-stat
-
-- started at 10/30/2020 10:17:12 --
mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall' '-T' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB Used FS: -nan% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 5600632.447 5600546.025 5600548.968 3.572
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 311.118 311.114 311.118 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 10/30/2020 10:22:23 --
- mdtest-easy-write
-
-- started at 10/30/2020 09:43:08 --
mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall' '-C' '-Y' '-W' '300' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB Used FS: -nan% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files
Continue stonewall hit min: 2658635 max: 2722551 avg: 2696152.2
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 5702340.422 5702331.857 5702335.773 2.119
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 5751786.562 NA
Tree creation : 0.167 0.167 0.167 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 305.565 305.564 305.565 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 300.000 NA
Tree creation : 5.976 5.976 5.976 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 10/30/2020 09:48:20 --
- mdtest-hard-delete
-
-- started at 10/30/2020 10:43:58 --
mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall' '-r' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB Used FS: -nan% Inodes: 4427640616128.0 Mi Used Inodes: 100.0%
Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 2727036.292 2727023.269 2727028.932 2.802
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.001 0.001 0.001 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 147.332 147.331 147.332 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 775.443 775.443 775.443 0.000
-- finished at 10/30/2020 10:59:21 --
- mdtest-hard-read
-
-- started at 10/30/2020 10:39:50 --
mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall' '-E' '-X' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB Used FS: -nan% Inodes: 0.0 Mi Used Inodes: -inf%
Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 1660422.283 1660393.861 1660420.800 1.636
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 241.977 241.973 241.973 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 10/30/2020 10:43:52 --
- mdtest-hard-stat
-
-- started at 10/30/2020 10:28:37 --
mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall' '-T' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB Used FS: -nan% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 2706871.411 2706861.567 2706866.759 2.270
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 148.429 148.429 148.429 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 10/30/2020 10:31:05 --
- mdtest-hard-write
-
-- started at 10/30/2020 09:54:10 --
mdtest-3.4.0+dev was launched with 640 total task(s) on 10 node(s)
Command line used: ./mdtest '-n' '10000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard' '-x' '/tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall' '-C' '-Y' '-W' '300' '-a' 'DFS' '--dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a' '--dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a' '--dfs.svcl=1' '--dfs.oclass=S1' '--dfs.prefix=/tmp/dfuse'
Path: /tmp/dfuse/datafiles/2020.10.30-09.32.49
FS: 0.0 GiB Used FS: -nan% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000
V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase.
640 tasks, 6400000000 files
Continue stonewall hit min: 576207 max: 627777 avg: 589105.8
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 1253006.906 1253005.141 1253005.929 0.449
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 1256729.203 NA
Tree creation : 38.679 38.679 38.679 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 320.651 320.650 320.651 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 300.007 NA
Tree creation : 0.026 0.026 0.026 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 10/30/2020 09:59:31 --
- result
-
version = io500-sc20_v3-1-g4c17ac501883-12
config-hash = 3CE19019
result-dir = /tmp/dfuse/results/2020.10.30-09.32.49
; START 2020-10-30 09:33:00
[ior-easy-write]
t_start = 2020-10-30 09:33:00
exe = ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-easy.stonewall -t 256k -b 100000m -w -D 300 -O stoneWallingWearOut=1 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.prefix=/tmp/dfuse
throughput-stonewall = 17001.62
score = 53.372671
t_delta = 606.0921
t_end = 2020-10-30 09:43:06
[mdtest-easy-write]
t_start = 2020-10-30 09:43:06
exe = ./mdtest -n 10000000 -u -L -F -P -N 1 -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall -C -Y -W 300 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
rate-stonewall = 5751.786562
score = 5702.333575
t_delta = 317.7481
t_end = 2020-10-30 09:48:24
[timestamp]
t_start = 2020-10-30 09:48:24
t_delta = 0.0242
t_end = 2020-10-30 09:48:24
[ior-hard-write]
t_start = 2020-10-30 09:48:24
exe = ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse
throughput-stonewall = 23994.88
score = 40.839710
t_delta = 344.2002
t_end = 2020-10-30 09:54:08
[mdtest-hard-write]
t_start = 2020-10-30 09:54:08
exe = ./mdtest -n 10000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall -C -Y -W 300 -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
rate-stonewall = 1256.729203
score = 1253.005913
t_delta = 327.0532
t_end = 2020-10-30 09:59:35
[find]
t_start = 2020-10-30 09:59:35
exe = ./pfind /tmp/dfuse/datafiles/2020.10.30-09.32.49 -newer /tmp/dfuse/results/2020.10.30-09.32.49/timestampfile -size 3901c -name *01* -C -q 10000
nproc = 640
found = 12763278
total-files = 843723591
score = 2714.238117
t_delta = 310.9214
t_end = 2020-10-30 10:04:46
[ior-easy-read]
t_start = 2020-10-30 10:04:46
exe = ./ior -C -Q 1 -g -G 271 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-easy/ior_file_easy -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-easy.stonewall -t 256k -b 100000m -r -R -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.prefix=/tmp/dfuse
score = 43.340062
t_delta = 743.4720
t_end = 2020-10-30 10:17:10
[mdtest-easy-stat]
t_start = 2020-10-30 10:17:10
exe = ./mdtest -n 10000000 -u -L -F -P -N 1 -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall -T -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
score = 5600.548081
t_delta = 317.3790
t_end = 2020-10-30 10:22:27
[ior-hard-read]
t_start = 2020-10-30 10:22:27
exe = ./ior -C -Q 1 -g -G 27 -k -e -o /tmp/dfuse/datafiles/2020.10.30-09.32.49/ior-hard/file -O stoneWallingStatusFile=/tmp/dfuse/results/2020.10.30-09.32.49/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.chunk_size=470080 --dfs.prefix=/tmp/dfuse
score = 38.202499
t_delta = 367.5066
t_end = 2020-10-30 10:28:35
[mdtest-hard-stat]
t_start = 2020-10-30 10:28:35
exe = ./mdtest -n 10000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall -T -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
score = 2706.865398
t_delta = 154.8284
t_end = 2020-10-30 10:31:10
[mdtest-easy-delete]
t_start = 2020-10-30 10:31:10
exe = ./mdtest -n 10000000 -u -L -F -P -N 1 -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-easy -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-easy.stonewall -r -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
score = 3463.991234
t_delta = 518.0797
t_end = 2020-10-30 10:39:48
[mdtest-hard-read]
t_start = 2020-10-30 10:39:48
exe = ./mdtest -n 10000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall -E -X -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
score = 1660.421548
t_delta = 248.4449
t_end = 2020-10-30 10:43:56
[mdtest-hard-delete]
t_start = 2020-10-30 10:43:56
exe = ./mdtest -n 10000000 -t -w 3901 -e 3901 -P -N 1 -F -d /tmp/dfuse/datafiles/2020.10.30-09.32.49/mdtest-hard -x /tmp/dfuse/results/2020.10.30-09.32.49/mdtest-hard.stonewall -r -a DFS --dfs.pool=751d2a8f-3fba-4d3b-9cad-13beafe3ee0a --dfs.cont=77c5931d-c4f4-48fb-b865-8359d6f18b0a --dfs.svcl=1 --dfs.oclass=S1 --dfs.prefix=/tmp/dfuse
score = 2727.029792
t_delta = 929.2505
t_end = 2020-10-30 10:59:26
[SCORE]
MD = 2870.644995
BW = 43.585870
SCORE = 353.722434
hash = 701ABADD
; END 2020-10-30 10:59:27
- result_summary
-
IO500 version io500-sc20_v3-1-g4c17ac501883-12
[RESULT] ior-easy-write 53.372671 GiB/s : time 606.092 seconds
[RESULT] mdtest-easy-write 5702.333575 kIOPS : time 317.748 seconds
[RESULT] ior-hard-write 40.839710 GiB/s : time 344.200 seconds
[RESULT] mdtest-hard-write 1253.005913 kIOPS : time 327.053 seconds
[RESULT] find 2714.238117 kIOPS : time 310.921 seconds
[RESULT] ior-easy-read 43.340062 GiB/s : time 743.472 seconds
[RESULT] mdtest-easy-stat 5600.548081 kIOPS : time 317.379 seconds
[RESULT] ior-hard-read 38.202499 GiB/s : time 367.507 seconds
[RESULT] mdtest-hard-stat 2706.865398 kIOPS : time 154.828 seconds
[RESULT] mdtest-easy-delete 3463.991234 kIOPS : time 518.080 seconds
[RESULT] mdtest-hard-read 1660.421548 kIOPS : time 248.445 seconds
[RESULT] mdtest-hard-delete 2727.029792 kIOPS : time 929.251 seconds
[SCORE] Bandwidth 43.585870 GiB/s : IOPS 2870.644995 kiops : TOTAL 353.722434