- io500
-
#!/bin/bash
#
# INSTRUCTIONS:
# This script takes its parameters from the same .ini file as io500 binary.
function setup_paths {
# Set the paths to the binaries and how to launch MPI jobs.
# If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
io500_ior_cmd=$PWD/bin/ior
io500_mdtest_cmd=$PWD/bin/mdtest
io500_mdreal_cmd=$PWD/bin/md-real-io
io500_mpirun="mpiexec"
io500_mpiargs="-np 96"
}
function setup_directories {
local workdir
local resultdir
local ts
# set directories where benchmark files are created and where the results go
# If you want to set up stripe tuning on your output directories or anything
# similar, then this is the right place to do it. This creates the output
# directories for both the app run and the script run.
timestamp=$(date +%Y.%m.%d-%H.%M.%S) # create a uniquifier
[ $(get_ini_global_param timestamp-datadir True) != "False" ] &&
ts="$timestamp" || ts="io500"
# directory where the data will be stored
workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts
io500_workdir=$workdir-scr
[ $(get_ini_global_param timestamp-resultdir True) != "False" ] &&
ts="$timestamp" || ts="io500"
# the directory where the output results will be kept
resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts
io500_result_dir=$resultdir-scr
mkdir -p $workdir-{scr,app} $resultdir-{scr,app}
}
# you should not edit anything below this line
set -eo pipefail # better error handling
io500_ini="${1:-""}"
if [[ -z "$io500_ini" ]]; then
echo "error: ini file must be specified. usage: $0 "
exit 1
fi
if [[ ! -s "$io500_ini" ]]; then
echo "error: ini file '$io500_ini' not found or empty"
exit 2
fi
function get_ini_section_param() {
local section="$1"
local param="$2"
local inside=false
while read LINE; do
LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE)
$inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break
[[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue
! $inside && continue
#echo $LINE | awk -F = "/^$param/ { print \$2 }"
if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then
# echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging
echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/"
return
fi
done < $io500_ini
echo ""
}
function get_ini_param() {
local section="$1"
local param="$2"
local default="$3"
# try and get the most-specific param first, then more generic params
val=$(get_ini_section_param $section $param)
[ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)"
[ -n "$val" ] || val="$(get_ini_section_param global $param)"
echo "${val:-$default}" |
sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/'
}
function get_ini_run_param() {
local section="$1"
local default="$2"
local val
val=$(get_ini_section_param $section noRun)
# logic is reversed from "noRun=TRUE" to "run=False"
[[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default"
}
function get_ini_global_param() {
local param="$1"
local default="$2"
local val
val=$(get_ini_section_param global $param |
sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/')
echo "${val:-$default}"
}
# does the write phase and enables the subsequent read
io500_run_ior_easy="$(get_ini_run_param ior-easy True)"
# does the creat phase and enables the subsequent stat
io500_run_md_easy="$(get_ini_run_param mdtest-easy True)"
# does the write phase and enables the subsequent read
io500_run_ior_hard="$(get_ini_run_param ior-hard True)"
# does the creat phase and enables the subsequent read
io500_run_md_hard="$(get_ini_run_param mdtest-hard True)"
io500_run_find="$(get_ini_run_param find True)"
io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)"
io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)"
io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)"
io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)"
# turn this off if you want to just run find by itself
io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)"
# turn this off if you want to just run find by itself
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)"
io500_run_mdreal="$(get_ini_run_param mdreal False)"
# attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_clean_cache="$(get_ini_global_param drop-caches False)"
io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)"
io500_cleanup_workdir="$(get_ini_run_param cleanup)"
# Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_stonewall_timer=$(get_ini_param debug stonewall-time 300)
# Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
io500_rules="regular"
# to run this benchmark, find and edit each of these functions. Please also
# also edit 'extra_description' function to help us collect the required data.
function main {
setup_directories
setup_paths
setup_ior_easy # required if you want a complete score
setup_ior_hard # required if you want a complete score
setup_mdt_easy # required if you want a complete score
setup_mdt_hard # required if you want a complete score
setup_find # required if you want a complete score
setup_mdreal # optional
run_benchmarks
if [[ ! -s "system-information.txt" ]]; then
echo "Warning: please create a system-information.txt description by"
echo "copying the information from https://vi4io.org/io500-info-creator/"
else
cp "system-information.txt" $io500_result_dir
fi
create_tarball
}
function setup_ior_easy {
local params
io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m)
val=$(get_ini_param ior-easy API POSIX)
[ -n "$val" ] && params+=" -a $val"
val="$(get_ini_param ior-easy transferSize)"
[ -n "$val" ] && params+=" -t $val"
val="$(get_ini_param ior-easy hintsFileName)"
[ -n "$val" ] && params+=" -U $val"
val="$(get_ini_param ior-easy posix.odirect)"
[ "$val" = "True" ] && params+=" --posix.odirect"
val="$(get_ini_param ior-easy verbosity)"
if [ -n "$val" ]; then
for i in $(seq $val); do
params+=" -v"
done
fi
io500_ior_easy_params="$params"
echo -n ""
}
function setup_mdt_easy {
io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
val=$(get_ini_param mdtest-easy n 1000000)
[ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val"
val=$(get_ini_param mdtest-easy API POSIX)
[ -n "$val" ] && io500_mdtest_easy_params+=" -a $val"
val=$(get_ini_param mdtest-easy posix.odirect)
[ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect"
echo -n ""
}
function setup_ior_hard {
local params
io500_ior_hard_api=$(get_ini_param ior-hard API POSIX)
io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)"
val="$(get_ini_param ior-hard hintsFileName)"
[ -n "$val" ] && params+=" -U $val"
val="$(get_ini_param ior-hard posix.odirect)"
[ "$val" = "True" ] && params+=" --posix.odirect"
val="$(get_ini_param ior-easy verbosity)"
if [ -n "$val" ]; then
for i in $(seq $val); do
params+=" -v"
done
fi
io500_ior_hard_api_specific_options="$params"
echo -n ""
}
function setup_mdt_hard {
val=$(get_ini_param mdtest-hard n 1000000)
[ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val"
io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)"
io500_mdtest_hard_api_specific_options=""
echo -n ""
}
function setup_find {
val="$(get_ini_param find external-script)"
[ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" ||
io500_find_cmd="$val"
# uses stonewalling, run pfind
io500_find_cmd_args="$(get_ini_param find external-extra-args)"
echo -n ""
}
function setup_mdreal {
echo -n ""
}
function run_benchmarks {
local app_first=$((RANDOM % 100))
local app_rc=0
# run the app and C version in random order to try and avoid bias
(( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$?
# Important: source the io500_fixed.sh script. Do not change it. If you
# discover a need to change it, please email the mailing list to discuss.
source build/io500-dev/utilities/io500_fixed.sh 2>&1 |
tee $io500_result_dir/io-500-summary.$timestamp.txt
(( $app_first >= 50 )) && return $app_rc
echo "The io500.sh was run"
echo
echo "Running the C version of the benchmark now"
# run the app and C version in random order to try and avoid bias
$io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp
}
create_tarball() {
local sourcedir=$(dirname $io500_result_dir)
local fname=$(basename ${io500_result_dir%-scr})
local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz
cp -v $0 $io500_ini $io500_result_dir
tar czf $tarball -C $sourcedir $fname-{app,scr}
echo "Created result tarball $tarball"
}
# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about
# your hardware that you want to include publicly!
function extra_description {
# UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator
# EXAMPLE:
# io500_info_system_name='xxx'
# DO NOT ADD IT HERE
:
}
main
- ior_easy_read
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Thu Jul 2 14:33:15 2020
Command line : /ui/ncsa/malone12/io500-app/bin/ior -r -R -a POSIX -t 4m -v -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_easy/stonewall
Machine : Linux mg005
Start time skew across all tasks: 31.62 sec
TestID : 0
StartTime : Thu Jul 2 14:33:15 2020
Path : /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_easy
FS : 3659.2 TiB Used FS: 44.7% Inodes: 266.5 Mi Used Inodes: 25.2%
Participating tasks: 96
Using reorderTasks '-C' (useful to avoid read cache in client)
Options:
api : POSIX
apiVersion :
test filename : /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 8
tasks : 96
clients per node : 12
repetitions : 1
xfersize : 4 MiB
blocksize : 9.46 TiB
aggregate filesize : 908.20 TiB
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
Commencing read performance test: Thu Jul 2 14:33:15 2020
WARNING: Expected aggregate file size = 998579896320000.
WARNING: Stat() of aggregate file size = 6953820487680.
WARNING: Using actual aggregate bytes moved = 6953820487680.
read 10408 2602.04 0.000257 10158080000 4096 0.047890 637.16 0.040442 637.17 0
Max Read: 10408.03 MiB/sec (10913.61 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 10408.03 10408.03 10408.03 0.00 2602.01 2602.01 2602.01 0.00 637.16957 NA NA 0 96 12 1 1 1 1 0 0 1 10401873920000 4194304 6631680.0 POSIX 0
Finished : Thu Jul 2 14:43:53 2020
- ior_easy_write
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Thu Jul 2 13:52:39 2020
Command line : /ui/ncsa/malone12/io500-app/bin/ior -w -a POSIX -t 4m -v -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux mg005
Start time skew across all tasks: 31.61 sec
TestID : 0
StartTime : Thu Jul 2 13:52:39 2020
Path : /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_easy
FS : 3659.2 TiB Used FS: 44.5% Inodes: 266.5 Mi Used Inodes: 14.8%
Participating tasks: 96
Using reorderTasks '-C' (useful to avoid read cache in client)
Options:
api : POSIX
apiVersion :
test filename : /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 8
tasks : 96
clients per node : 12
repetitions : 1
xfersize : 4 MiB
blocksize : 9.46 TiB
aggregate filesize : 908.20 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
Commencing write performance test: Thu Jul 2 13:52:39 2020
41: stonewalling pairs accessed: 17130
42: stonewalling pairs accessed: 17228
5: stonewalling pairs accessed: 4917
1: stonewalling pairs accessed: 5123
70: stonewalling pairs accessed: 7699
0: stonewalling pairs accessed: 5057
79: stonewalling pairs accessed: 13751
32: stonewalling pairs accessed: 7264
77: stonewalling pairs accessed: 14002
78: stonewalling pairs accessed: 13809
76: stonewalling pairs accessed: 13795
28: stonewalling pairs accessed: 7097
46: stonewalling pairs accessed: 17270
20: stonewalling pairs accessed: 15036
39: stonewalling pairs accessed: 17096
54: stonewalling pairs accessed: 6972
69: stonewalling pairs accessed: 7563
80: stonewalling pairs accessed: 13554
83: stonewalling pairs accessed: 13751
35: stonewalling pairs accessed: 6862
25: stonewalling pairs accessed: 7035
61: stonewalling pairs accessed: 7795
38: stonewalling pairs accessed: 17128
23: stonewalling pairs accessed: 14728
62: stonewalling pairs accessed: 7724
68: stonewalling pairs accessed: 7444
2: stonewalling pairs accessed: 4913
64: stonewalling pairs accessed: 7657
16: stonewalling pairs accessed: 15067
85: stonewalling pairs accessed: 6733
95: stonewalling pairs accessed: 6548
10: stonewalling pairs accessed: 5147
36: stonewalling pairs accessed: 17094
17: stonewalling pairs accessed: 14939
15: stonewalling pairs accessed: 15057
24: stonewalling pairs accessed: 7259
51: stonewalling pairs accessed: 6650
60: stonewalling pairs accessed: 7591
3: stonewalling pairs accessed: 4913
31: stonewalling pairs accessed: 7205
34: stonewalling pairs accessed: 7106
47: stonewalling pairs accessed: 17113
9: stonewalling pairs accessed: 4945
27: stonewalling pairs accessed: 7233
11: stonewalling pairs accessed: 4920
93: stonewalling pairs accessed: 6757
8: stonewalling pairs accessed: 5044
65: stonewalling pairs accessed: 7378
82: stonewalling pairs accessed: 13764
67: stonewalling pairs accessed: 7679
72: stonewalling pairs accessed: 14001
6: stonewalling pairs accessed: 5099
66: stonewalling pairs accessed: 7504
29: stonewalling pairs accessed: 7218
37: stonewalling pairs accessed: 16483
4: stonewalling pairs accessed: 4839
74: stonewalling pairs accessed: 13826
45: stonewalling pairs accessed: 16951
21: stonewalling pairs accessed: 15060
14: stonewalling pairs accessed: 14956
71: stonewalling pairs accessed: 7663
18: stonewalling pairs accessed: 14550
19: stonewalling pairs accessed: 14736
12: stonewalling pairs accessed: 14869
53: stonewalling pairs accessed: 6849
7: stonewalling pairs accessed: 5048
81: stonewalling pairs accessed: 13927
44: stonewalling pairs accessed: 16710
30: stonewalling pairs accessed: 7593
22: stonewalling pairs accessed: 14961
63: stonewalling pairs accessed: 7713
13: stonewalling pairs accessed: 14918
40: stonewalling pairs accessed: 17191
88: stonewalling pairs accessed: 6780
43: stonewalling pairs accessed: 16795
73: stonewalling pairs accessed: 13601
26: stonewalling pairs accessed: 7160
75: stonewalling pairs accessed: 13960
92: stonewalling pairs accessed: 6615
89: stonewalling pairs accessed: 6597
52: stonewalling pairs accessed: 6690
55: stonewalling pairs accessed: 6752
56: stonewalling pairs accessed: 6780
50: stonewalling pairs accessed: 6887
49: stonewalling pairs accessed: 6763
57: stonewalling pairs accessed: 6779
59: stonewalling pairs accessed: 6713
33: stonewalling pairs accessed: 7125
58: stonewalling pairs accessed: 6676
48: stonewalling pairs accessed: 6780
87: stonewalling pairs accessed: 6569
84: stonewalling pairs accessed: 6569
86: stonewalling pairs accessed: 6679
94: stonewalling pairs accessed: 6617
91: stonewalling pairs accessed: 6839
90: stonewalling pairs accessed: 6905
stonewalling pairs accessed min: 4839 max: 17270 -- min data: 18.9 GiB mean data: 38.6 GiB time: 300.2s
WARNING: Expected aggregate file size = 998579896320000.
WARNING: Stat() of aggregate file size = 6953820487680.
WARNING: Using actual aggregate bytes moved = 6953820487680.
WARNING: maybe caused by deadlineForStonewalling
write 9512 2378.21 0.000281 10158080000 4096 0.064958 697.13 0.031545 697.18 0
Max Write: 9512.12 MiB/sec (9974.18 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 9512.12 9512.12 9512.12 0.00 2378.03 2378.03 2378.03 0.00 697.18216 300.20 12629.25 0 96 12 1 1 1 1 0 0 1 10401873920000 4194304 6631680.0 POSIX 0
Finished : Thu Jul 2 14:04:16 2020
- ior_hard_read
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Thu Jul 2 14:50:11 2020
Command line : /ui/ncsa/malone12/io500-app/bin/ior -r -R -s 10000000 -a POSIX -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_hard/stonewall
Machine : Linux mg005
Start time skew across all tasks: 31.61 sec
TestID : 0
StartTime : Thu Jul 2 14:50:11 2020
Path : /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_hard
FS : 3659.2 TiB Used FS: 44.7% Inodes: 266.5 Mi Used Inodes: 25.2%
Participating tasks: 96
Using reorderTasks '-C' (useful to avoid read cache in client)
Options:
api : POSIX
apiVersion :
test filename : /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 10000000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 8
tasks : 96
clients per node : 12
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 41.04 TiB
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
Commencing read performance test: Thu Jul 2 14:50:11 2020
WARNING: Expected aggregate file size = 45127680000000.
WARNING: Stat() of aggregate file size = 261352445952.
WARNING: Using actual aggregate bytes moved = 261352445952.
read 1151.36 25683 216.42 45.91 45.91 0.051878 216.47 0.049717 216.48 0
Max Read: 1151.36 MiB/sec (1207.28 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 1151.36 1151.36 1151.36 0.00 25682.53 25682.53 25682.53 0.00 216.47961 NA NA 0 96 12 1 0 1 1 0 0 10000000 47008 47008 249245.1 POSIX 0
Finished : Thu Jul 2 14:53:48 2020
- ior_hard_write
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Thu Jul 2 14:15:18 2020
Command line : /ui/ncsa/malone12/io500-app/bin/ior -w -s 10000000 -a POSIX -v -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux mg005
Start time skew across all tasks: 31.62 sec
TestID : 0
StartTime : Thu Jul 2 14:15:18 2020
Path : /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_hard
FS : 3659.2 TiB Used FS: 44.7% Inodes: 266.5 Mi Used Inodes: 23.7%
Participating tasks: 96
Using reorderTasks '-C' (useful to avoid read cache in client)
Options:
api : POSIX
apiVersion :
test filename : /magnus/io500/datafiles/2020.07.02-12.32.35-scr/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 10000000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 8
tasks : 96
clients per node : 12
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 41.04 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
Commencing write performance test: Thu Jul 2 14:15:18 2020
22: stonewalling pairs accessed: 57913
21: stonewalling pairs accessed: 57913
13: stonewalling pairs accessed: 57913
20: stonewalling pairs accessed: 57913
18: stonewalling pairs accessed: 57913
15: stonewalling pairs accessed: 57913
23: stonewalling pairs accessed: 57913
12: stonewalling pairs accessed: 57913
16: stonewalling pairs accessed: 57914
14: stonewalling pairs accessed: 57914
19: stonewalling pairs accessed: 57914
48: stonewalling pairs accessed: 57903
59: stonewalling pairs accessed: 57903
57: stonewalling pairs accessed: 57903
56: stonewalling pairs accessed: 57903
58: stonewalling pairs accessed: 57903
69: stonewalling pairs accessed: 57894
17: stonewalling pairs accessed: 57914
71: stonewalling pairs accessed: 57894
64: stonewalling pairs accessed: 57894
61: stonewalling pairs accessed: 57894
60: stonewalling pairs accessed: 57894
67: stonewalling pairs accessed: 57894
70: stonewalling pairs accessed: 57894
36: stonewalling pairs accessed: 57887
41: stonewalling pairs accessed: 57887
45: stonewalling pairs accessed: 57887
72: stonewalling pairs accessed: 57870
80: stonewalling pairs accessed: 57870
43: stonewalling pairs accessed: 57887
84: stonewalling pairs accessed: 57883
46: stonewalling pairs accessed: 57887
74: stonewalling pairs accessed: 57870
91: stonewalling pairs accessed: 57883
37: stonewalling pairs accessed: 57887
75: stonewalling pairs accessed: 57870
44: stonewalling pairs accessed: 57887
93: stonewalling pairs accessed: 57883
7: stonewalling pairs accessed: 57807
89: stonewalling pairs accessed: 57883
78: stonewalling pairs accessed: 57870
3: stonewalling pairs accessed: 57814
73: stonewalling pairs accessed: 57870
79: stonewalling pairs accessed: 57870
10: stonewalling pairs accessed: 57807
83: stonewalling pairs accessed: 57870
9: stonewalling pairs accessed: 57804
42: stonewalling pairs accessed: 57887
11: stonewalling pairs accessed: 57799
39: stonewalling pairs accessed: 57887
0: stonewalling pairs accessed: 57746
38: stonewalling pairs accessed: 57887
85: stonewalling pairs accessed: 57883
40: stonewalling pairs accessed: 57887
92: stonewalling pairs accessed: 57883
8: stonewalling pairs accessed: 57812
87: stonewalling pairs accessed: 57883
86: stonewalling pairs accessed: 57883
1: stonewalling pairs accessed: 57746
2: stonewalling pairs accessed: 57746
4: stonewalling pairs accessed: 57746
5: stonewalling pairs accessed: 57746
6: stonewalling pairs accessed: 57746
27: stonewalling pairs accessed: 57748
90: stonewalling pairs accessed: 57883
54: stonewalling pairs accessed: 57903
66: stonewalling pairs accessed: 57894
81: stonewalling pairs accessed: 57870
28: stonewalling pairs accessed: 57748
95: stonewalling pairs accessed: 57883
55: stonewalling pairs accessed: 57903
68: stonewalling pairs accessed: 57894
77: stonewalling pairs accessed: 57870
29: stonewalling pairs accessed: 57748
88: stonewalling pairs accessed: 57883
53: stonewalling pairs accessed: 57903
62: stonewalling pairs accessed: 57894
76: stonewalling pairs accessed: 57870
30: stonewalling pairs accessed: 57748
49: stonewalling pairs accessed: 57903
63: stonewalling pairs accessed: 57894
82: stonewalling pairs accessed: 57870
31: stonewalling pairs accessed: 57748
52: stonewalling pairs accessed: 57903
32: stonewalling pairs accessed: 57748
50: stonewalling pairs accessed: 57903
33: stonewalling pairs accessed: 57748
51: stonewalling pairs accessed: 57903
34: stonewalling pairs accessed: 57748
35: stonewalling pairs accessed: 57748
25: stonewalling pairs accessed: 57749
26: stonewalling pairs accessed: 57748
24: stonewalling pairs accessed: 57749
47: stonewalling pairs accessed: 57887
65: stonewalling pairs accessed: 57894
94: stonewalling pairs accessed: 57883
stonewalling pairs accessed min: 57746 max: 57914 -- min data: 2.5 GiB mean data: 2.5 GiB time: 300.1s
WARNING: Expected aggregate file size = 45127680000000.
WARNING: Stat() of aggregate file size = 261352445952.
WARNING: Using actual aggregate bytes moved = 261352445952.
WARNING: maybe caused by deadlineForStonewalling
write 829.59 18506 300.38 45.91 45.91 0.049085 300.42 0.040391 300.44 0
Max Write: 829.59 MiB/sec (869.89 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 829.59 829.59 829.59 0.00 18505.17 18505.17 18505.17 0.00 300.44271 300.09 829.80 0 96 12 1 0 1 1 0 0 10000000 47008 47008 249245.1 POSIX 0
Finished : Thu Jul 2 14:20:19 2020
- mdtest_easy_delete
-
-- started at 07/02/2020 15:01:18 --
mdtest-3.3.0+dev was launched with 96 total task(s) on 8 node(s)
Command line used: /ui/ncsa/malone12/io500-app/bin/mdtest '-r' '-F' '-P' '-d' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_easy' '-n' '1000000' '-u' '-L' '-a' 'POSIX' '-x' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_easy-stonewall' '-N' '1'
Path: /magnus/io500/datafiles/2020.07.02-12.32.35-scr
FS: 3659.2 TiB Used FS: 44.7% Inodes: 266.5 Mi Used Inodes: 25.2%
Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2166 Shifting ranks by 12 for each phase.
96 tasks, 96000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 55619.807 55619.759 55619.789 0.012
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.579 0.579 0.579 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 487.042 487.042 487.042 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 1.728 1.728 1.728 0.000
-- finished at 07/02/2020 15:09:27 --
- mdtest_easy_stat
-
-- started at 07/02/2020 14:46:01 --
mdtest-3.3.0+dev was launched with 96 total task(s) on 8 node(s)
Command line used: /ui/ncsa/malone12/io500-app/bin/mdtest '-T' '-F' '-P' '-d' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_easy' '-n' '1000000' '-u' '-L' '-a' 'POSIX' '-x' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_easy-stonewall' '-N' '1'
Path: /magnus/io500/datafiles/2020.07.02-12.32.35-scr
FS: 3659.2 TiB Used FS: 44.7% Inodes: 266.5 Mi Used Inodes: 25.2%
Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2166 Shifting ranks by 12 for each phase.
96 tasks, 96000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 222527.255 222504.138 222515.721 11.232
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 121.747 121.734 121.741 0.006
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 07/02/2020 14:48:03 --
- mdtest_easy_write
-
-- started at 07/02/2020 14:06:24 --
mdtest-3.3.0+dev was launched with 96 total task(s) on 8 node(s)
Command line used: /ui/ncsa/malone12/io500-app/bin/mdtest '-Y' '-C' '-F' '-P' '-d' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_easy' '-n' '1000000' '-u' '-L' '-a' 'POSIX' '-x' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /magnus/io500/datafiles/2020.07.02-12.32.35-scr
FS: 3659.2 TiB Used FS: 44.7% Inodes: 266.5 Mi Used Inodes: 14.8%
Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2166 Shifting ranks by 12 for each phase.
96 tasks, 96000000 files
Continue stonewall hit min: 208301 max: 282179 avg: 239461.8
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 66965.583 66965.515 66965.554 0.017
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 76784.907 NA
Tree creation : 1.564 1.564 1.564 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 404.524 404.524 404.524 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 299.386 NA
Tree creation : 0.640 0.640 0.640 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 07/02/2020 14:13:10 --
- mdtest_hard_delete
-
-- started at 07/02/2020 15:17:09 --
mdtest-3.3.0+dev was launched with 96 total task(s) on 8 node(s)
Command line used: /ui/ncsa/malone12/io500-app/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_hard' '-n' '600000' '-x' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /magnus/io500/datafiles/2020.07.02-12.32.35-scr
FS: 3659.2 TiB Used FS: 44.7% Inodes: 266.5 Mi Used Inodes: 15.8%
Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2166 Shifting ranks by 12 for each phase.
96 tasks, 57600000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 9068.418 9068.408 9068.414 0.003
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 35.929 35.929 35.929 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 493.582 493.581 493.581 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.028 0.028 0.028 0.000
-- finished at 07/02/2020 15:25:23 --
- mdtest_hard_read
-
-- started at 07/02/2020 15:11:35 --
mdtest-3.3.0+dev was launched with 96 total task(s) on 8 node(s)
Command line used: /ui/ncsa/malone12/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_hard' '-n' '600000' '-x' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /magnus/io500/datafiles/2020.07.02-12.32.35-scr
FS: 3659.2 TiB Used FS: 44.7% Inodes: 266.5 Mi Used Inodes: 15.8%
Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2166 Shifting ranks by 12 for each phase.
96 tasks, 57600000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 21721.747 21721.686 21721.716 0.014
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 206.061 206.061 206.061 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 07/02/2020 15:15:01 --
- mdtest_hard_stat
-
-- started at 07/02/2020 14:55:57 --
mdtest-3.3.0+dev was launched with 96 total task(s) on 8 node(s)
Command line used: /ui/ncsa/malone12/io500-app/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_hard' '-n' '600000' '-x' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /magnus/io500/datafiles/2020.07.02-12.32.35-scr
FS: 3659.2 TiB Used FS: 44.7% Inodes: 266.5 Mi Used Inodes: 25.2%
Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2166 Shifting ranks by 12 for each phase.
96 tasks, 57600000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 23205.511 23205.461 23205.497 0.010
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 192.886 192.885 192.885 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 07/02/2020 14:59:09 --
- mdtest_hard_write
-
-- started at 07/02/2020 14:22:27 --
mdtest-3.3.0+dev was launched with 96 total task(s) on 8 node(s)
Command line used: /ui/ncsa/malone12/io500-app/bin/mdtest '-Y' '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_hard' '-n' '600000' '-x' '/magnus/io500/datafiles/2020.07.02-12.32.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /magnus/io500/datafiles/2020.07.02-12.32.35-scr
FS: 3659.2 TiB Used FS: 44.7% Inodes: 266.5 Mi Used Inodes: 23.7%
Nodemap: 111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2166 Shifting ranks by 12 for each phase.
96 tasks, 57600000 files
Continue stonewall hit min: 37005 max: 46625 avg: 42529.9
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 12479.930 12479.910 12479.916 0.004
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 13609.242 NA
Tree creation : 3188.237 3188.237 3188.237 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 358.656 358.656 358.656 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 300.008 NA
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 07/02/2020 14:28:26 --
- result_summary
-
[RESULT] BW phase 1 ior_easy_write 9.289 GiB/s : time 697.13 seconds
[RESULT] IOPS phase 1 mdtest_easy_write 66.966 kiops : time 404.52 seconds
[RESULT] BW phase 2 ior_hard_write 0.810 GiB/s : time 300.42 seconds
[RESULT] IOPS phase 2 mdtest_hard_write 12.480 kiops : time 358.66 seconds
[RESULT] IOPS phase 3 find 196.400 kiops : time 160.71 seconds
[RESULT] BW phase 3 ior_easy_read 10.164 GiB/s : time 637.16 seconds
[RESULT] IOPS phase 4 mdtest_easy_stat 222.527 kiops : time 121.75 seconds
[RESULT] BW phase 4 ior_hard_read 1.124 GiB/s : time 216.47 seconds
[RESULT] IOPS phase 5 mdtest_hard_stat 23.206 kiops : time 192.89 seconds
[RESULT] IOPS phase 6 mdtest_easy_delete 55.620 kiops : time 487.04 seconds
[RESULT] IOPS phase 7 mdtest_hard_read 21.722 kiops : time 206.06 seconds
[RESULT] IOPS phase 8 mdtest_hard_delete 9.068 kiops : time 621.85 seconds
[SCORE] Bandwidth 3.04529 GiB/s : IOPS 41.781 kiops : TOTAL 11.2799