- io500
-
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.
set -euo pipefail # better error handling
# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True" # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True" # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False" # this one is optional
io500_cleanup_workdir="False" # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_stonewall_timer=0 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...
# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
setup_directories
setup_paths
setup_ior_easy # required if you want a complete score
setup_ior_hard # required if you want a complete score
setup_mdt_easy # required if you want a complete score
setup_mdt_hard # required if you want a complete score
setup_find # required if you want a complete score
setup_mdreal # optional
run_benchmarks
}
function setup_directories {
# set directories for where the benchmark files are created and where the results will go.
# If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
timestamp=`date +%Y.%m.%d-%H.%M.%S` # create a uniquifier
io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
io500_result_dir=$PWD/results/$timestamp # the directory where the output results will be kept
mkdir -p $io500_workdir $io500_result_dir
mkdir -p ${io500_workdir}/ior_easy ${io500_workdir}/ior_hard
mkdir -p ${io500_workdir}/mdt_easy ${io500_workdir}/mdt_hard
# for ior_easy.
beegfs-ctl --setpattern --numtargets=4 --chunksize=1024k ${io500_workdir}/ior_easy
# stripe across all OSTs for ior_hard, 256k chunksize
#beegfs-ctl --setpattern --numtargets=96 --chunksize=256k ${io500_workdir}/ior_hard
beegfs-ctl --setpattern --numtargets=96 --chunksize=256k ${io500_workdir}/ior_hard
# turn off striping and use small chunks for mdtest
beegfs-ctl --setpattern --numtargets=1 --chunksize=64k ${io500_workdir}/mdt_easy
beegfs-ctl --setpattern --numtargets=1 --chunksize=64k ${io500_workdir}/mdt_hard
}
function setup_paths {
# Set the paths to the binaries. If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
io500_ior_cmd=$PWD/bin/ior
io500_mdtest_cmd=$PWD/bin/mdtest
io500_mdreal_cmd=$PWD/bin/md-real-io
io500_mpirun="mpirun"
io500_mpiargs="--hostfile host.10x16 "
#io500_mpiargs="-x LD_LIBRARY_PATH -np 184 --hostfile hosts_mpich.io500 "
}
function setup_ior_easy {
# io500_ior_easy_size is the amount of data written per rank in MiB units,
# but it can be any number as long as it is somehow used to scale the IOR
# runtime as part of io500_ior_easy_params
#io500_ior_easy_size=160
io500_ior_easy_size=220
# 2M writes, 2 GB per proc, file per proc
io500_ior_easy_params="-t 2048k -b ${io500_ior_easy_size}g -F -B "
}
function setup_mdt_easy {
io500_mdtest_easy_params="-z 1 -b 24 -u -L" # unique dir per thread, files only at leaves
#io500_mdtest_easy_files_per_proc=210000
io500_mdtest_easy_files_per_proc=300000
}
function setup_ior_hard {
#io500_ior_hard_writes_per_proc=80000
io500_ior_hard_writes_per_proc=110000
io500_ior_hard_other_options="" #e.g., -E to keep precreated files using lfs setstripe, or -a MPIIO
}
function setup_mdt_hard {
#io500_mdtest_hard_files_per_proc=11500
io500_mdtest_hard_files_per_proc=20000
io500_mdtest_hard_other_options=""
}
function setup_find {
#
# setup the find command. This is an area where innovation is allowed.
# There are three default options provided. One is a serial find, one is python
# parallel version, one is C parallel version. Current default is to use serial.
# But it is very slow. We recommend to either customize or use the C parallel version.
# For GPFS, we recommend to use the provided mmfind wrapper described below.
# Instructions below.
# If a custom approach is used, please provide enough info so others can reproduce.
# the serial version that should run (SLOWLY) without modification
#io500_find_mpi="False"
#io500_find_cmd=$PWD/bin/sfind.sh
#io500_find_cmd_args=""
# a parallel version in C, the -s adds a stonewall
# for a real run, turn -s (stonewall) off or set it at 300 or more
# to prepare this (assuming you've run ./utilities/prepare.sh already):
# > cd build/pfind
# > ./prepare.sh
# > ./compile.sh
# > cp pfind ../../bin/
# If you use io500_find_mpi="True", then this will run with the same
# number of MPI nodes and ranks as the other phases.
# If you prefer another number, and fewer might be better here,
# Then you can set io500_find_mpi to be "False" and write a wrapper
# script for this which sets up MPI as you would like. Then change
# io500_find_cmd to point to your wrapper script.
io500_find_mpi="True"
io500_find_cmd="$PWD/bin/pfind"
# uses stonewalling, run pfind
io500_find_cmd_args="-s 20000 -r $io500_result_dir/pfind_results"
# for GPFS systems, you should probably use the provided mmfind wrapper
# if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
#io500_find_mpi="False"
#io500_find_cmd="$PWD/bin/mmfind.sh"
#io500_find_cmd_args=""
}
function setup_mdreal {
io500_mdreal_params="-P=5000 -I=1000"
}
function run_benchmarks {
# Important: source the io500_fixed.sh script. Do not change it. If you discover
# a need to change it, please email the mailing list to discuss
source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}
# Add key/value pairs defining your system
# Feel free to add extra ones if you'd like
function extra_description {
# top level info
io500_info_system_name='bracewell' # e.g. Oakforest-PACS
io500_info_institute_name='CSIRO' # e.g. JCAHPC
io500_info_storage_age_in_months='10' # not install date but age since last refresh
io500_info_storage_install_date='10/18' # MM/YY
io500_info_filesystem='BeeGFS' # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
io500_info_filesystem_version='7.1.3'
io500_info_filesystem_vendor='DELLi/ThinkParQ'
# client side info
io500_info_num_client_nodes='10'
io500_info_procs_per_node='16'
# server side info
io500_info_num_metadata_server_nodes='2'
io500_info_num_data_server_nodes='16'
io500_info_num_data_storage_devices='384' # if you have 5 data servers, and each has 5 drives, then this number is 25
io500_info_num_metadata_storage_devices='24' # if you have 2 metadata servers, and each has 5 drives, then this number is 10
io500_info_data_storage_type='NVMe' # HDD, SSD, persistent memory, etc, feel free to put specific models
io500_info_metadata_storage_type='NVMe' # HDD, SSD, persistent memory, etc, feel free to put specific models
io500_info_storage_network='IB EDR' # infiniband, omnipath, ethernet, etc
io500_info_storage_interface='NVMe' # SAS, SATA, NVMe, etc
# miscellaneous
io500_info_whatever=''
}
echo contents of MPI host file
cat host.10x16
echo start run
main
- ior_easy_read
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss020', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Tue May 21 23:40:07 2019
Command line : /scratch1/leh015/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 2048k -b 220g -F -B -o /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_easy/ior_file_easy -O stoneWallingStatusFile=/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_easy/stonewall
Machine : Linux bss018
TestID : 0
StartTime : Tue May 21 23:40:07 2019
Path : /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_easy
FS : 1117.6 TiB Used FS: 9.9% Inodes: 0.0 Mi Used Inodes: -nan%
Options:
api : POSIX
apiVersion :
test filename : /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 160
clients per node : 16
repetitions : 1
xfersize : 2 MiB
blocksize : 220 GiB
aggregate filesize : 34.38 TiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
[bss017:128110] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:128110] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
read 96051 230686720 2048.00 0.012581 375.24 0.008360 375.27 0
Max Read: 96051.45 MiB/sec (100717.25 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 96051.45 96051.45 96051.45 0.00 48025.73 48025.73 48025.73 0.00 375.26554 0 160 16 1 1 1 1 0 0 1 236223201280 2097152 36044800.0 POSIX 0
Finished : Tue May 21 23:46:22 2019
- ior_easy_write
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss021', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Tue May 21 23:18:15 2019
Command line : /scratch1/leh015/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 2048k -b 220g -F -B -o /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_easy/ior_file_easy -O stoneWallingStatusFile=/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_easy/stonewall -O stoneWallingWearOut=1 -D 0
Machine : Linux bss018
TestID : 0
StartTime : Tue May 21 23:18:15 2019
Path : /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_easy
FS : 1117.6 TiB Used FS: 6.7% Inodes: 0.0 Mi Used Inodes: -nan%
Options:
api : POSIX
apiVersion :
test filename : /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 160
clients per node : 16
repetitions : 1
xfersize : 2 MiB
blocksize : 220 GiB
aggregate filesize : 34.38 TiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
[bss017:125646] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:125646] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
stonewalling pairs accessed min: 112640 max: 112640 -- min data: 220.0 GiB mean data: 220.0 GiB time: 310.5s
write 116094 230686720 2048.00 0.008820 310.46 0.007064 310.48 0
Max Write: 116093.87 MiB/sec (121733.24 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 116093.87 116093.87 116093.87 0.00 58046.93 58046.93 58046.93 0.00 310.47980 0 160 16 1 1 1 1 0 0 1 236223201280 2097152 36044800.0 POSIX 0
Finished : Tue May 21 23:23:26 2019
- ior_hard_read
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss027', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Tue May 21 23:47:23 2019
Command line : /scratch1/leh015/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 110000 -o /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_hard/IOR_file -O stoneWallingStatusFile=/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_hard/stonewall
Machine : Linux bss018
TestID : 0
StartTime : Tue May 21 23:47:23 2019
Path : /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_hard
FS : 1117.6 TiB Used FS: 9.9% Inodes: 0.0 Mi Used Inodes: -nan%
Options:
api : POSIX
apiVersion :
test filename : /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 110000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 160
clients per node : 16
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 770.52 GiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
[bss017:128579] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:128579] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
read 57920 45.91 45.91 0.134454 13.48 0.012076 13.62 0
Max Read: 57920.29 MiB/sec (60733.83 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 57920.29 57920.29 57920.29 0.00 1291989.21 1291989.21 1291989.21 0.00 13.62240 0 160 16 1 0 1 1 0 0 110000 47008 47008 789013.7 POSIX 0
Finished : Tue May 21 23:47:37 2019
- ior_hard_write
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss018', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Tue May 21 23:28:48 2019
Command line : /scratch1/leh015/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 110000 -o /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_hard/IOR_file -O stoneWallingStatusFile=/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_hard/stonewall -O stoneWallingWearOut=1 -D 0
Machine : Linux bss018
TestID : 0
StartTime : Tue May 21 23:28:48 2019
Path : /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_hard
FS : 1117.6 TiB Used FS: 9.8% Inodes: 0.0 Mi Used Inodes: -nan%
Options:
api : POSIX
apiVersion :
test filename : /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 110000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 160
clients per node : 16
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 770.52 GiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
[bss017:127153] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:127153] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
stonewalling pairs accessed min: 110000 max: 110000 -- min data: 4.8 GiB mean data: 4.8 GiB time: 313.6s
write 2511.63 45.91 45.91 0.138069 313.98 0.028703 314.14 0
Max Write: 2511.63 MiB/sec (2633.63 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 2511.63 2511.63 2511.63 0.00 56025.18 56025.18 56025.18 0.00 314.14443 0 160 16 1 0 1 1 0 0 110000 47008 47008 789013.7 POSIX 0
Finished : Tue May 21 23:34:02 2019
- mdtest_easy_delete
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss019', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 05/21/2019 23:48:06 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch1/leh015/io-500-dev/bin/mdtest "-r" "-F" "-d" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_easy" "-n" "300000" "-z" "1" "-b" "24" "-u" "-L" "-x" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_easy-stonewall"
Path: /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14
FS: 1117.6 TiB Used FS: 9.9% Inodes: 0.0 Mi Used Inodes: -nan%
160 tasks, 48000000 files
WARNING: could not read stonewall status file
[bss017:128755] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:128755] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 247029.906 247029.783 247029.845 0.032
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 3.481 3.481 3.481 0.000
-- finished at 05/21/2019 23:51:28 --
- mdtest_easy_stat
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss025', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 05/21/2019 23:46:25 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch1/leh015/io-500-dev/bin/mdtest "-T" "-F" "-d" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_easy" "-n" "300000" "-z" "1" "-b" "24" "-u" "-L" "-x" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_easy-stonewall"
Path: /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14
FS: 1117.6 TiB Used FS: 9.9% Inodes: 0.0 Mi Used Inodes: -nan%
160 tasks, 48000000 files
WARNING: could not read stonewall status file
[bss017:128546] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:128546] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 873489.740 873488.779 873489.436 0.241
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/21/2019 23:47:20 --
- mdtest_easy_write
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss024', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 05/21/2019 23:23:29 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch1/leh015/io-500-dev/bin/mdtest "-C" "-F" "-d" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_easy" "-n" "300000" "-z" "1" "-b" "24" "-u" "-L" "-x" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_easy-stonewall" "-W" "0"
Path: /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14
FS: 1117.6 TiB Used FS: 9.8% Inodes: 0.0 Mi Used Inodes: -nan%
160 tasks, 48000000 files
[bss017:125932] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:125932] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 151681.343 151681.299 151681.320 0.013
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 323.991 323.991 323.991 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/21/2019 23:28:45 --
- mdtest_hard_delete
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss025', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 05/21/2019 23:53:12 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch1/leh015/io-500-dev/bin/mdtest "-r" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_hard" "-n" "20000" "-x" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_hard-stonewall"
Path: /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14
FS: 1117.6 TiB Used FS: 9.9% Inodes: 0.0 Mi Used Inodes: -nan%
160 tasks, 3200000 files
WARNING: could not read stonewall status file
[bss017:129060] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:129060] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 14665.775 14665.764 14665.772 0.003
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.332 0.332 0.332 0.000
-- finished at 05/21/2019 23:56:53 --
- mdtest_hard_read
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss023', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 05/21/2019 23:51:31 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch1/leh015/io-500-dev/bin/mdtest "-E" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_hard" "-n" "20000" "-x" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_hard-stonewall"
Path: /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14
FS: 1117.6 TiB Used FS: 9.9% Inodes: 0.0 Mi Used Inodes: -nan%
160 tasks, 3200000 files
WARNING: could not read stonewall status file
[bss017:128911] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:128911] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 32738.979 32738.968 32738.974 0.003
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/21/2019 23:53:09 --
- mdtest_hard_stat
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss025', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 05/21/2019 23:47:40 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch1/leh015/io-500-dev/bin/mdtest "-T" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_hard" "-n" "20000" "-x" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_hard-stonewall"
Path: /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14
FS: 1117.6 TiB Used FS: 9.9% Inodes: 0.0 Mi Used Inodes: -nan%
160 tasks, 3200000 files
WARNING: could not read stonewall status file
[bss017:128612] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:128612] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 136768.693 136768.376 136768.621 0.080
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/21/2019 23:48:03 --
- mdtest_hard_write
-
--------------------------------------------------------------------------
WARNING: There are more than one active ports on host 'bss025', but the
default subnet GID prefix was detected on more than one of these
ports. If these ports are connected to different physical IB
networks, this configuration will fail in Open MPI. This version of
Open MPI requires that every physically separate IB subnet that is
used between connected MPI processes must have different subnet ID
values.
Please see this FAQ entry for more details:
http://www.open-mpi.org/faq/?category=openfabrics#ofa-default-subnet-gid
NOTE: You can turn off this warning by setting the MCA parameter
btl_openib_warn_default_gid_prefix to 0.
--------------------------------------------------------------------------
-- started at 05/21/2019 23:34:05 --
mdtest-3.3.0+dev was launched with 160 total task(s) on 10 node(s)
Command line used: /scratch1/leh015/io-500-dev/bin/mdtest "-C" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_hard" "-n" "20000" "-x" "/scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14/mdt_hard-stonewall" "-W" "0"
Path: /scratch1/leh015/io-500-dev/datafiles/io500.2019.05.21-23.18.14
FS: 1117.6 TiB Used FS: 9.9% Inodes: 0.0 Mi Used Inodes: -nan%
160 tasks, 3200000 files
[bss017:127680] 159 more processes have sent help message help-mpi-btl-openib.txt / default subnet prefix
[bss017:127680] Set MCA parameter "orte_base_help_aggregate" to 0 to see all help / error messages
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 9849.653 9849.647 9849.651 0.002
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 1279.681 1279.681 1279.681 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/21/2019 23:39:30 --
- result_summary
-
[RESULT] BW phase 1 ior_easy_write 113.373 GB/s : time 310.48 seconds
[RESULT] IOPS phase 1 mdtest_easy_write 151.681 kiops : time 319.39 seconds
[RESULT] BW phase 2 ior_hard_write 2.453 GB/s : time 314.14 seconds
[RESULT] IOPS phase 2 mdtest_hard_write 9.850 kiops : time 327.67 seconds
[RESULT] IOPS phase 3 find 1502.420 kiops : time 34.09 seconds
[RESULT] BW phase 3 ior_easy_read 93.800 GB/s : time 375.27 seconds
[RESULT] IOPS phase 4 mdtest_easy_stat 873.490 kiops : time 58.00 seconds
[RESULT] BW phase 4 ior_hard_read 56.562 GB/s : time 13.62 seconds
[RESULT] IOPS phase 5 mdtest_hard_stat 136.769 kiops : time 26.38 seconds
[RESULT] IOPS phase 6 mdtest_easy_delete 247.030 kiops : time 204.46 seconds
[RESULT] IOPS phase 7 mdtest_hard_read 32.739 kiops : time 100.87 seconds
[RESULT] IOPS phase 8 mdtest_hard_delete 14.666 kiops : time 224.25 seconds
[SCORE] Bandwidth 34.8517 GB/s : IOPS 115.562 kiops : TOTAL 63.4628