- io500
-
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.
set -euo pipefail # better error handling
# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True" # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True" # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False" # this one is optional
io500_cleanup_workdir="False" # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
hostfile="host.io500"
# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
setup_directories
setup_paths
setup_ior_easy # required if you want a complete score
setup_ior_hard # required if you want a complete score
setup_mdt_easy # required if you want a complete score
setup_mdt_hard # required if you want a complete score
setup_find # required if you want a complete score
setup_mdreal # optional
run_benchmarks
}
function setup_directories {
# set directories for where the benchmark files are created and where the results will go.
# If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
timestamp=`date +%Y.%m.%d-%H.%M.%S` # create a uniquifier
io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
io500_result_dir=$PWD/results/$timestamp # the directory where the output results will be kept
mkdir -p $io500_workdir $io500_result_dir
mkdir -p ${io500_workdir}/ior_easy ${io500_workdir}/ior_hard
mkdir -p ${io500_workdir}/mdt_easy ${io500_workdir}/mdt_hard
# for ior_easy.
beegfs-ctl --cfgFile=/etc/beegfs/beegfs-client-cl2.conf --setpattern --numtargets=4 --chunksize=1024k ${io500_workdir}/ior_easy
# stripe across all OSTs for ior_hard, 256k chunksize
#beegfs-ctl --setpattern --numtargets=96 --chunksize=256k ${io500_workdir}/ior_hard
beegfs-ctl --cfgFile=/etc/beegfs/beegfs-client-cl2.conf --setpattern --numtargets=72 --chunksize=256k ${io500_workdir}/ior_hard
# turn off striping and use small chunks for mdtest
beegfs-ctl --cfgFile=/etc/beegfs/beegfs-client-cl2.conf --setpattern --numtargets=1 --chunksize=64k ${io500_workdir}/mdt_easy
beegfs-ctl --cfgFile=/etc/beegfs/beegfs-client-cl2.conf --setpattern --numtargets=1 --chunksize=64k ${io500_workdir}/mdt_hard
}
function setup_paths {
# Set the paths to the binaries. If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
io500_ior_cmd=$PWD/bin/ior
io500_mdtest_cmd=$PWD/bin/mdtest
io500_mdreal_cmd=$PWD/bin/md-real-io
io500_mpirun="mpirun"
io500_mpiargs="--hostfile ${hostfile} "
#io500_mpiargs="-x LD_LIBRARY_PATH -np 184 --hostfile hosts_mpich.io500 "
}
function setup_ior_easy {
#io500_ior_easy_params="-t 2048k -b \${io500_ior_easy_size}m -F"
io500_ior_easy_size=860
io500_ior_easy_params="-t 2048k -b ${io500_ior_easy_size}g -F --posix.odirect "
echo -n ""
}
function setup_mdt_easy {
io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
#io500_mdtest_easy_params="-u -L --posix.odirect" # unique dir per thread, files only at leaves
#io500_mdtest_easy_files_per_proc=430000
}
function setup_ior_hard {
io500_ior_hard_api="POSIX"
io500_ior_hard_api_specific_options=""
io500_ior_hard_writes_per_proc=150000
}
function setup_mdt_hard {
io500_mdtest_hard_api="POSIX"
io500_mdtest_hard_api_specific_options=""
#io500_mdtest_hard_files_per_proc=23500
}
function setup_find {
#
# setup the find command. This is an area where innovation is allowed.
# There are three default options provided. One is a serial find, one is python
# parallel version, one is C parallel version. Current default is to use serial.
# But it is very slow. We recommend to either customize or use the C parallel version.
# For GPFS, we recommend to use the provided mmfind wrapper described below.
# Instructions below.
# If a custom approach is used, please provide enough info so others can reproduce.
# the serial version that should run (SLOWLY) without modification
#io500_find_mpi="False"
#io500_find_cmd=$PWD/bin/sfind.sh
#io500_find_cmd_args=""
# a parallel version in C, the -s adds a stonewall
# for a real run, turn -s (stonewall) off or set it at 300 or more
# to prepare this (assuming you've run ./utilities/prepare.sh already):
# > cd build/pfind
# > ./prepare.sh
# > ./compile.sh
# > cp pfind ../../bin/
# If you use io500_find_mpi="True", then this will run with the same
# number of MPI nodes and ranks as the other phases.
# If you prefer another number, and fewer might be better here,
# Then you can set io500_find_mpi to be "False" and write a wrapper
# script for this which sets up MPI as you would like. Then change
# io500_find_cmd to point to your wrapper script.
io500_find_mpi="True"
io500_find_cmd="$PWD/bin/pfind"
# uses stonewalling, run pfind
#io500_find_cmd_args=""
io500_find_cmd_args="-s 20000 -r $io500_result_dir/pfind_results"
# for GPFS systems, you should probably use the provided mmfind wrapper
# if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
#io500_find_mpi="False"
#io500_find_cmd="$PWD/bin/mmfind.sh"
#io500_find_cmd_args=""
}
function setup_mdreal {
echo -n ""
}
function run_benchmarks {
# Important: source the io500_fixed.sh script. Do not change it. If you discover
# a need to change it, please email the mailing list to discuss
source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}
# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
# TODO: Please add your information using the info-creator!
# EXAMPLE:
io500_info_system_name='scratch2' # e.g. Oakforest-PACS
}
echo "Using these Client Nodes:"
echo "========================="
cat $hostfile
echo "Using these storage nodes:"
echo "=========================="
beegfs-ctl --listnodes --nodetype=storage --mount=/scratch2
echo "Using these metadata nodes:"
echo "=================== ======="
beegfs-ctl --listnodes --nodetype=meta --mount=/scratch2
echo start run
main
cp $hostfile $io500_result_dir
cp $0 $io500_result_dir
- ior_easy_read
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss027', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Sat Nov 9 07:32:08 2019
Command line : /scratch2/leh015/io500-sc19/bin/ior -r -R -t 2048k -b 860g -F --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -o /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_easy/ior_file_easy -O stoneWallingStatusFile=/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_easy/stonewall
Machine : Linux bss022
TestID : 0
StartTime : Sat Nov 9 07:32:08 2019
Path : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_easy
FS : 628.6 TiB Used FS: 5.7% Inodes: 0.0 Mi Used Inodes: -nan%
Options:
api : POSIX
apiVersion :
test filename : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 160
clients per node : 16
repetitions : 1
xfersize : 2 MiB
blocksize : 860 GiB
aggregate filesize : 134.38 TiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
[bss022:09251] 111 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:09251] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
WARNING: Expected aggregate file size = 147746874982400.
WARNING: Stat() of aggregate file size = 38085622497280.
WARNING: Using actual aggregate bytes moved = 38085622497280.
read 95224 901775360 2048.00 0.011019 381.41 0.011710 381.43 0
Max Read: 95223.81 MiB/sec (99849.40 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 95223.81 95223.81 95223.81 0.00 47611.90 47611.90 47611.90 0.00 381.43066 NA NA 0 160 16 1 1 1 1 0 0 1 923417968640 2097152 36321280.0 POSIX 0
Finished : Sat Nov 9 07:38:29 2019
- ior_easy_write
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss022', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Sat Nov 9 07:01:37 2019
Command line : /scratch2/leh015/io500-sc19/bin/ior -w -t 2048k -b 860g -F --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -o /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_easy/ior_file_easy -O stoneWallingStatusFile=/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux bss022
TestID : 0
StartTime : Sat Nov 9 07:01:37 2019
Path : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_easy
FS : 628.6 TiB Used FS: 0.0% Inodes: 0.0 Mi Used Inodes: -nan%
Options:
api : POSIX
apiVersion :
test filename : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 160
clients per node : 16
repetitions : 1
xfersize : 2 MiB
blocksize : 860 GiB
aggregate filesize : 134.38 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
[bss022:04670] 111 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:04670] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
stonewalling pairs accessed min: 102027 max: 113504 -- min data: 199.3 GiB mean data: 214.2 GiB time: 300.0s
WARNING: Expected aggregate file size = 147746874982400.
WARNING: Stat() of aggregate file size = 38085622497280.
WARNING: Using actual aggregate bytes moved = 38085622497280.
WARNING: maybe caused by deadlineForStonewalling
write 114229 901775360 2048.00 0.350003 317.38 0.236214 317.97 0
Max Write: 114228.51 MiB/sec (119777.27 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 114228.51 114228.51 114228.51 0.00 57114.26 57114.26 57114.26 0.00 317.97036 300.00 116962.57 0 160 16 1 1 1 1 0 0 1 923417968640 2097152 36321280.0 POSIX 0
Finished : Sat Nov 9 07:07:16 2019
- ior_hard_read
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss027', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
[bss022:10399] [[18045,0],0] ORTE_ERROR_LOG: Out of resource in file util/show_help.c at line 501
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Sat Nov 9 07:40:23 2019
Command line : /scratch2/leh015/io500-sc19/bin/ior -r -R -s 150000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_hard/IOR_file -O stoneWallingStatusFile=/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_hard/stonewall
Machine : Linux bss022
TestID : 0
StartTime : Sat Nov 9 07:40:23 2019
Path : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_hard
FS : 628.6 TiB Used FS: 5.7% Inodes: 0.0 Mi Used Inodes: -nan%
Options:
api : POSIX
apiVersion :
test filename : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 150000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 160
clients per node : 16
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 1.03 TiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
[bss022:10399] 110 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:10399] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
read 60900 45.91 45.91 0.082922 17.58 0.005967 17.67 0
Max Read: 60899.54 MiB/sec (63857.80 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 60899.54 60899.54 60899.54 0.00 1358445.27 1358445.27 1358445.27 0.00 17.66726 NA NA 0 160 16 1 0 1 1 0 0 150000 47008 47008 1075927.8 POSIX 0
Finished : Sat Nov 9 07:40:40 2019
- ior_hard_write
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss022', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Sat Nov 9 07:17:45 2019
Command line : /scratch2/leh015/io500-sc19/bin/ior -w -s 150000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_hard/IOR_file -O stoneWallingStatusFile=/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux bss022
TestID : 0
StartTime : Sat Nov 9 07:17:45 2019
Path : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_hard
FS : 628.6 TiB Used FS: 5.5% Inodes: 0.0 Mi Used Inodes: -nan%
Options:
api : POSIX
apiVersion :
test filename : /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 150000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 160
clients per node : 16
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 1.03 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
[bss022:06294] 111 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:06294] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
stonewalling pairs accessed min: 7528 max: 150000 -- min data: 0.3 GiB mean data: 3.9 GiB time: 300.0s
write 2158.01 45.91 45.91 0.279005 498.28 0.016365 498.57 0
Max Write: 2158.01 MiB/sec (2262.84 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 2158.01 2158.01 2158.01 0.00 48137.27 48137.27 48137.27 0.00 498.57423 300.00 2122.45 0 160 16 1 0 1 1 0 0 150000 47008 47008 1075927.8 POSIX 0
Finished : Sat Nov 9 07:26:04 2019
- mdtest_easy_delete
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss025', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 11/09/2019 07:41:18 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-r' '-F' '-P' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_easy-stonewall' '-N' '1'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35
FS: 628.6 TiB Used FS: 5.7% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 16 for each phase.
160 tasks, 144000000 files
[bss022:10980] 111 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:10980] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 230531.467 230531.166 230531.395 0.080
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.039 0.039 0.039 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 407.841 407.840 407.840 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 25.347 25.347 25.347 0.000
-- finished at 11/09/2019 07:48:44 --
- mdtest_easy_stat
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss025', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 11/09/2019 07:38:35 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-T' '-F' '-P' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_easy-stonewall' '-N' '1'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35
FS: 628.6 TiB Used FS: 5.7% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 16 for each phase.
160 tasks, 144000000 files
[bss022:09987] 111 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:09987] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 921711.702 921710.871 921711.413 0.201
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 102.006 102.006 102.006 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2019 07:40:17 --
- mdtest_easy_write
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss025', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 11/09/2019 07:07:21 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-C' '-F' '-P' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35
FS: 628.6 TiB Used FS: 5.5% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 16 for each phase.
160 tasks, 144000000 files
[bss022:05335] 111 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:05335] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
Continue stonewall hit min: 272176 max: 587625 avg: 360850.8
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 152158.161 152158.096 152158.133 0.018
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 192441.203 NA
Tree creation : 9.557 9.557 9.557 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 617.910 617.910 617.910 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 300.020 NA
Tree creation : 0.105 0.105 0.105 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2019 07:17:39 --
- mdtest_hard_delete
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss022', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 11/09/2019 07:50:28 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_hard' '-n' '950000' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35
FS: 628.6 TiB Used FS: 5.7% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 16 for each phase.
160 tasks, 152000000 files
[bss022:12224] 111 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:12224] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 18778.971 18778.942 18778.966 0.008
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.256 0.256 0.256 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 186.814 186.813 186.813 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 3.902 3.902 3.902 0.000
-- finished at 11/09/2019 07:53:39 --
- mdtest_hard_read
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss025', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 11/09/2019 07:48:51 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_hard' '-n' '950000' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35
FS: 628.6 TiB Used FS: 5.7% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 16 for each phase.
160 tasks, 152000000 files
[bss022:11814] 111 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:11814] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 38053.692 38053.595 38053.632 0.013
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 92.190 92.190 92.190 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2019 07:50:23 --
- mdtest_hard_stat
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss027', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 11/09/2019 07:40:46 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_hard' '-n' '950000' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35
FS: 628.6 TiB Used FS: 5.7% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 16 for each phase.
160 tasks, 152000000 files
[bss022:10690] 111 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:10690] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 130820.788 130818.292 130820.741 0.196
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 26.817 26.817 26.817 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2019 07:41:13 --
- mdtest_hard_write
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss027', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 11/09/2019 07:26:09 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch2/leh015/io500-sc19/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_hard' '-n' '950000' '-x' '/scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /scratch2/leh015/io500-sc19/datafiles/io500.2019.11.09-07.01.35
FS: 628.6 TiB Used FS: 5.7% Inodes: 0.0 Mi Used Inodes: -nan%
Nodemap: 1111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 16 for each phase.
160 tasks, 152000000 files
[bss022:07210] 111 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss022:07210] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
Continue stonewall hit min: 19052 max: 21926 avg: 20709.5
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 11030.334 11030.316 11030.331 0.005
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 11045.383 NA
Tree creation : 123.795 123.795 123.795 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 318.047 318.047 318.047 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 299.992 NA
Tree creation : 0.008 0.008 0.008 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2019 07:31:27 --
- result_summary
-
[RESULT] BW phase 1 ior_easy_write 111.552 GB/s : time 317.97 seconds
[RESULT] IOPS phase 1 mdtest_easy_write 152.158 kiops : time 617.91 seconds
[RESULT] BW phase 2 ior_hard_write 2.107 GB/s : time 498.57 seconds
[RESULT] IOPS phase 2 mdtest_hard_write 11.030 kiops : time 318.05 seconds
[RESULT] IOPS phase 3 find 2789.710 kiops : time 34.96 seconds
[RESULT] BW phase 3 ior_easy_read 92.992 GB/s : time 381.43 seconds
[RESULT] IOPS phase 4 mdtest_easy_stat 921.712 kiops : time 102.01 seconds
[RESULT] BW phase 4 ior_hard_read 59.473 GB/s : time 17.67 seconds
[RESULT] IOPS phase 5 mdtest_hard_stat 130.821 kiops : time 26.82 seconds
[RESULT] IOPS phase 6 mdtest_easy_delete 230.531 kiops : time 407.84 seconds
[RESULT] IOPS phase 7 mdtest_hard_read 38.054 kiops : time 92.19 seconds
[RESULT] IOPS phase 8 mdtest_hard_delete 18.779 kiops : time 196.27 seconds
[SCORE] Bandwidth 33.7675 GB/s : IOPS 132.149 kiops : TOTAL 66.8008