- io500
-
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.
set -euo pipefail # better error handling
# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True" # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True" # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False" # this one is optional
io500_cleanup_workdir="False" # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
setup_directories
setup_paths
setup_ior_easy # required if you want a complete score
setup_ior_hard # required if you want a complete score
setup_mdt_easy # required if you want a complete score
setup_mdt_hard # required if you want a complete score
setup_find # required if you want a complete score
setup_mdreal # optional
run_benchmarks
}
function setup_directories {
# set directories for where the benchmark files are created and where the results will go.
# If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
timestamp=`date +%Y.%m.%d-%H.%M.%S` # create a uniquifier
io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
io500_result_dir=$PWD/results/$timestamp # the directory where the output results will be kept
mkdir -p $io500_workdir $io500_result_dir
}
function setup_paths {
# Set the paths to the binaries. If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
io500_ior_cmd=$PWD/bin/ior
io500_mdtest_cmd=$PWD/bin/mdtest
io500_mdreal_cmd=$PWD/bin/md-real-io
io500_mpirun="mpirun"
io500_mpiargs="-np 760 -ppn 76 -f ./site-configs/hhmi/hostfile-10gpu --bootstrap ssh"
}
function setup_ior_easy {
io500_ior_easy_params="-u -t 1024k -b 9920000m -F --posix.odirect"
echo -n ""
}
function setup_mdt_easy {
io500_mdtest_easy_params="-u -L --posix.odirect" # unique dir per thread, files only at leaves
}
function setup_ior_hard {
io500_ior_hard_api="POSIX"
io500_ior_hard_api_specific_options="--posix.odirect"
}
function setup_mdt_hard {
io500_mdtest_hard_api="POSIX"
io500_mdtest_hard_api_specific_options="--posix.odirect"
}
function setup_find {
#
# setup the find command. This is an area where innovation is allowed.
# There are three default options provided. One is a serial find, one is python
# parallel version, one is C parallel version. Current default is to use serial.
# But it is very slow. We recommend to either customize or use the C parallel version.
# For GPFS, we recommend to use the provided mmfind wrapper described below.
# Instructions below.
# If a custom approach is used, please provide enough info so others can reproduce.
# the serial version that should run (SLOWLY) without modification
#io500_find_mpi="False"
#io500_find_cmd=$PWD/bin/sfind.sh
#io500_find_cmd_args=""
# a parallel version in C, the -s adds a stonewall
# for a real run, turn -s (stonewall) off or set it at 300 or more
# to prepare this (assuming you've run ./utilities/prepare.sh already):
# > cd build/pfind
# > ./prepare.sh
# > ./compile.sh
# > cp pfind ../../bin/
# If you use io500_find_mpi="True", then this will run with the same
# number of MPI nodes and ranks as the other phases.
# If you prefer another number, and fewer might be better here,
# Then you can set io500_find_mpi to be "False" and write a wrapper
# script for this which sets up MPI as you would like. Then change
# io500_find_cmd to point to your wrapper script.
io500_find_mpi="True"
io500_find_cmd="$PWD/bin/pfind"
# uses stonewalling, run pfind
io500_find_cmd_args="-N"
# for GPFS systems, you should probably use the provided mmfind wrapper
# if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
#io500_find_mpi="False"
#io500_find_cmd="$PWD/bin/mmfind.sh"
#io500_find_cmd_args=""
}
function setup_mdreal {
echo -n ""
}
function run_benchmarks {
# Important: source the io500_fixed.sh script. Do not change it. If you discover
# a need to change it, please email the mailing list to discuss
source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}
# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
# TODO: Please add your information using the info-creator!
# EXAMPLE:
echo io500_info_submitter='"Janelia Scientific Computing Systems"'
echo io500_info_institution='"Janelia Research Campus, Howard Hughes Medical Institute"'
echo io500_info_system='"nrs"'
echo io500_info_system_classification='"production"'
echo io500_info_storage_install_date='"9/2016"'
echo io500_info_storage_vendor='"Qumulo"'
echo io500_info_filesystem_name='"Qumulo FS"'
echo io500_info_filesystem_type='"Scale-out NAS"'
echo io500_info_filesystem_version='"2.14.0.1"'
echo io500_info_client_nodes='"10"'
echo io500_info_client_procs_per_node='"76"'
echo io500_info_client_operating_system='"Scientific-Linux"'
echo io500_info_client_operating_system_version='"7.6"'
echo io500_info_client_kernel_version='"3.10.0-957.10.1.el7.x86_64"'
echo io500_info_md_storage_devices='"13"'
echo io500_info_md_storage_type='"SSD"'
echo io500_info_md_storage_interface='"SATA"'
echo io500_info_ds_nodes='"20"'
echo io500_info_ds_storage_devices='"26"'
echo io500_info_ds_storage_type='"HDD"'
echo io500_info_ds_storage_interface='"SATA"'
echo io500_info_ds_network='"Ethernet"'
echo io500_info_ds_overhead='"12%"'
echo io500_info_whatever='"Storage nodes have both SSD and HDD. Data is written to SSD before being drained out to HDD, and is rehydrated from HDD automatically as needed. All metadata is stored permanently on SSD"'
}
main
- ior_easy_read
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Sat Nov 9 21:04:42 2019
Command line : /io500/io500-sc19/bin/ior -r -R -u -t 1024k -b 9920000m -F --posix.odirect -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/ior_file_easy -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/stonewall
Machine : Linux e11u05.int.janelia.org
TestID : 0
StartTime : Sat Nov 9 21:04:42 2019
Path : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/0
FS : 2984.3 TiB Used FS: 69.1% Inodes: 763980.0 Mi Used Inodes: 69.1%
Options:
api : POSIX
apiVersion :
test filename : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 10
tasks : 760
clients per node : 76
repetitions : 1
xfersize : 1 MiB
blocksize : 9.46 TiB
aggregate filesize : 7189.94 TiB
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
WARNING: Expected aggregate file size = 7905424179200000.
WARNING: Stat() of aggregate file size = 3048210432.
WARNING: Using actual aggregate bytes moved = 2316639928320.
read 13693 13696 0.000016 10158080000 1024.00 0.037825 161.32 0.013310 161.34 0
Max Read: 13693.45 MiB/sec (14358.62 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 13693.45 13693.45 13693.45 0.00 13693.45 13693.45 13693.45 0.00 161.34143 NA NA 0 760 76 1 2 1 1 0 0 1 10401873920000 1048576 2209320.0 POSIX 0
Finished : Sat Nov 9 21:07:24 2019
- ior_easy_write
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Sat Nov 9 20:35:41 2019
Command line : /io500/io500-sc19/bin/ior -w -u -t 1024k -b 9920000m -F --posix.odirect -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/ior_file_easy -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux e11u05.int.janelia.org
TestID : 0
StartTime : Sat Nov 9 20:35:41 2019
Path : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/0
FS : 2984.3 TiB Used FS: 69.0% Inodes: 763980.0 Mi Used Inodes: 69.0%
Options:
api : POSIX
apiVersion :
test filename : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 1
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 10
tasks : 760
clients per node : 76
repetitions : 1
xfersize : 1 MiB
blocksize : 9.46 TiB
aggregate filesize : 7189.94 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 1409 max: 2907 -- min data: 1.4 GiB mean data: 2.3 GiB time: 300.3s
WARNING: Expected aggregate file size = 7905424179200000.
WARNING: Stat() of aggregate file size = 3048210432.
WARNING: Using actual aggregate bytes moved = 2316639928320.
WARNING: maybe caused by deadlineForStonewalling
write 5575 5602 0.000038 10158080000 1024.00 1.91 394.37 0.009011 396.28 0
Max Write: 5575.08 MiB/sec (5845.90 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 5575.08 5575.08 5575.08 0.00 5575.08 5575.08 5575.08 0.00 396.28478 300.26 5872.84 0 760 76 1 2 1 1 0 0 1 10401873920000 1048576 2209320.0 POSIX 0
Finished : Sat Nov 9 20:42:22 2019
- ior_hard_read
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Sat Nov 9 21:08:04 2019
Command line : /io500/io500-sc19/bin/ior -r -R -s 1900000 -a POSIX --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/IOR_file -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/stonewall
Machine : Linux e11u05.int.janelia.org
TestID : 0
StartTime : Sat Nov 9 21:08:04 2019
Path : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard
FS : 2984.3 TiB Used FS: 69.1% Inodes: 763980.0 Mi Used Inodes: 69.1%
Options:
api : POSIX
apiVersion :
test filename : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 1900000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 10
tasks : 760
clients per node : 76
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 61.74 TiB
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
WARNING: Expected aggregate file size = 67879552000000.
WARNING: Stat() of aggregate file size = 56518658560.
WARNING: Using actual aggregate bytes moved = 56518658560.
read 5848 130941 9.18 45.91 45.91 0.027607 9.18 0.008709 9.22 0
Max Read: 5847.57 MiB/sec (6131.63 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 5847.57 5847.57 5847.57 0.00 130437.91 130437.91 130437.91 0.00 9.21757 NA NA 0 760 76 1 0 1 1 0 0 1900000 47008 47008 53900.4 POSIX 0
Finished : Sat Nov 9 21:08:13 2019
- ior_hard_write
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Sat Nov 9 20:51:38 2019
Command line : /io500/io500-sc19/bin/ior -w -s 1900000 -a POSIX --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/IOR_file -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux e11u05.int.janelia.org
TestID : 0
StartTime : Sat Nov 9 20:51:38 2019
Path : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard
FS : 2984.3 TiB Used FS: 69.1% Inodes: 763980.0 Mi Used Inodes: 69.1%
Options:
api : POSIX
apiVersion :
test filename : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 1900000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
nodes : 10
tasks : 760
clients per node : 76
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 61.74 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 791 max: 1582 -- min data: 0.0 GiB mean data: 0.1 GiB time: 300.2s
WARNING: Expected aggregate file size = 67879552000000.
WARNING: Stat() of aggregate file size = 56518658560.
WARNING: Using actual aggregate bytes moved = 56518658560.
WARNING: maybe caused by deadlineForStonewalling
write 168.53 3760 300.26 45.91 45.91 0.050615 319.78 0.001022 319.83 0
Max Write: 168.53 MiB/sec (176.71 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 168.53 168.53 168.53 0.00 3759.22 3759.22 3759.22 0.00 319.83192 300.17 170.48 0 760 76 1 0 1 1 0 0 1900000 47008 47008 53900.4 POSIX 0
Finished : Sat Nov 9 20:56:58 2019
- mdtest_easy_delete
-
-- started at 11/09/2019 21:08:33 --
mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-r' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy' '-n' '900000' '-u' '-L' '--posix.odirect' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB Used FS: 69.1% Inodes: 763980.0 Mi Used Inodes: 69.1%
Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2137 Shifting ranks by 76 for each phase.
760 tasks, 684000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 10768.460 10768.452 10768.455 0.002
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.262 0.262 0.262 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 452.325 452.325 452.325 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 3.810 3.810 3.810 0.000
-- finished at 11/09/2019 21:16:12 --
- mdtest_easy_stat
-
-- started at 11/09/2019 21:07:30 --
mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-T' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy' '-n' '900000' '-u' '-L' '--posix.odirect' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB Used FS: 69.1% Inodes: 763980.0 Mi Used Inodes: 69.1%
Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2137 Shifting ranks by 76 for each phase.
760 tasks, 684000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 171203.263 171201.951 171202.495 0.307
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 28.451 28.451 28.451 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2019 21:07:58 --
- mdtest_easy_write
-
-- started at 11/09/2019 20:42:28 --
mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-C' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy' '-n' '900000' '-u' '-L' '--posix.odirect' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB Used FS: 69.1% Inodes: 763980.0 Mi Used Inodes: 69.1%
Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2137 Shifting ranks by 76 for each phase.
760 tasks, 684000000 files
Continue stonewall hit min: 390 max: 6409 avg: 2852.8
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 9028.042 9028.012 9028.038 0.002
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 7292.039 NA
Tree creation : 0.270 0.270 0.270 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 539.525 539.523 539.524 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 297.330 NA
Tree creation : 3.708 3.708 3.708 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2019 20:51:31 --
- mdtest_hard_delete
-
-- started at 11/09/2019 21:16:30 --
mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard-stonewall' '-a' 'POSIX' '--posix.odirect' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB Used FS: 69.1% Inodes: 763980.0 Mi Used Inodes: 69.1%
Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2137 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 178.684 178.678 178.683 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 6.495 6.495 6.495 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 361.545 361.533 361.533 0.001
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.154 0.154 0.154 0.000
-- finished at 11/09/2019 21:22:32 --
- mdtest_hard_read
-
-- started at 11/09/2019 21:16:18 --
mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard-stonewall' '-a' 'POSIX' '--posix.odirect' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB Used FS: 69.1% Inodes: 763980.0 Mi Used Inodes: 69.1%
Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2137 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 9813.238 9813.048 9813.159 0.049
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 6.583 6.583 6.583 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2019 21:16:25 --
- mdtest_hard_stat
-
-- started at 11/09/2019 21:08:19 --
mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard-stonewall' '-a' 'POSIX' '--posix.odirect' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB Used FS: 69.1% Inodes: 763980.0 Mi Used Inodes: 69.1%
Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2137 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 9066.865 9059.275 9066.720 0.384
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 7.131 7.125 7.125 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2019 21:08:26 --
- mdtest_hard_write
-
-- started at 11/09/2019 20:57:04 --
mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard-stonewall' '-a' 'POSIX' '--posix.odirect' '-N' '1' '-W' '300'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB Used FS: 69.1% Inodes: 763980.0 Mi Used Inodes: 69.1%
Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank 0 Line 2137 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files
Continue stonewall hit min: 81 max: 85 avg: 82.9
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 206.861 206.860 206.861 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 207.370 NA
Tree creation : 373.824 373.824 373.824 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 312.288 312.287 312.288 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 303.718 NA
Tree creation : 0.003 0.003 0.003 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 11/09/2019 21:02:16 --
- result_summary
-
[RESULT] BW phase 1 ior_easy_write 5.444 GB/s : time 394.37 seconds
[RESULT] IOPS phase 1 mdtest_easy_write 9.028 kiops : time 539.53 seconds
[RESULT] BW phase 2 ior_hard_write 0.165 GB/s : time 319.78 seconds
[RESULT] IOPS phase 2 mdtest_hard_write 0.207 kiops : time 312.29 seconds
[RESULT] IOPS phase 3 find 35.320 kiops : time 139.72 seconds
[RESULT] BW phase 3 ior_easy_read 13.372 GB/s : time 161.32 seconds
[RESULT] IOPS phase 4 mdtest_easy_stat 171.203 kiops : time 28.45 seconds
[RESULT] BW phase 4 ior_hard_read 5.711 GB/s : time 9.18 seconds
[RESULT] IOPS phase 5 mdtest_hard_stat 9.067 kiops : time 7.13 seconds
[RESULT] IOPS phase 6 mdtest_easy_delete 10.768 kiops : time 452.33 seconds
[RESULT] IOPS phase 7 mdtest_hard_read 9.813 kiops : time 6.58 seconds
[RESULT] IOPS phase 8 mdtest_hard_delete 0.179 kiops : time 367.75 seconds
[SCORE] Bandwidth 2.87612 GB/s : IOPS 6.10646 kiops : TOTAL 4.19081