- io500
-
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.
set -euo pipefail # better error handling
# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True" # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True" # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False" # this one is optional
io500_cleanup_workdir="False" # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds
# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
setup_directories
setup_paths
setup_ior_easy # required if you want a complete score
setup_ior_hard # required if you want a complete score
setup_mdt_easy # required if you want a complete score
setup_mdt_hard # required if you want a complete score
setup_find # required if you want a complete score
setup_mdreal # optional
run_benchmarks
}
function setup_directories {
# set directories for where the benchmark files are created and where the results will go.
# If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
timestamp=`date +%Y.%m.%d-%H.%M.%S` # create a uniquifier
io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
io500_result_dir=$PWD/results/$timestamp # the directory where the output results will be kept
mkdir -p $io500_workdir $io500_result_dir
}
function setup_paths {
# Set the paths to the binaries. If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
io500_ior_cmd=$PWD/bin/ior
io500_mdtest_cmd=$PWD/bin/mdtest
io500_mdreal_cmd=$PWD/bin/md-real-io
io500_mpirun="mpirun"
io500_mpiargs="-np 40 -N 4 --allow-run-as-root -hostfile hostfile"
}
function setup_ior_easy {
#io500_ior_easy_params="-t 2048k -b \${io500_ior_easy_size}m -F"
io500_ior_easy_params="-a MPIIO -t 1024k -s 250000 -F"
echo -n ""
}
function setup_mdt_easy {
io500_mdtest_easy_params="-a POSIX -u -L" # unique dir per thread, files only at leaves
io500_mdtest_easy_files_per_proc="450000"
}
function setup_ior_hard {
io500_ior_hard_api="MPIIO"
io500_ior_hard_api_specific_options=""
io500_ior_hard_writes_per_proc="200000"
}
function setup_mdt_hard {
io500_mdtest_hard_api="POSIX"
io500_mdtest_hard_api_specific_options=""
io500_mdtest_hard_files_per_proc="75000"
}
function setup_find {
#
# setup the find command. This is an area where innovation is allowed.
# There are three default options provided. One is a serial find, one is python
# parallel version, one is C parallel version. Current default is to use serial.
# But it is very slow. We recommend to either customize or use the C parallel version.
# For GPFS, we recommend to use the provided mmfind wrapper described below.
# Instructions below.
# If a custom approach is used, please provide enough info so others can reproduce.
# the serial version that should run (SLOWLY) without modification
#io500_find_mpi="False"
#io500_find_cmd=$PWD/bin/sfind.sh
#io500_find_cmd_args=""
# a parallel version in C, the -s adds a stonewall
# for a real run, turn -s (stonewall) off or set it at 300 or more
# to prepare this (assuming you've run ./utilities/prepare.sh already):
# > cd build/pfind
# > ./prepare.sh
# > ./compile.sh
# > cp pfind ../../bin/
# If you use io500_find_mpi="True", then this will run with the same
# number of MPI nodes and ranks as the other phases.
# If you prefer another number, and fewer might be better here,
# Then you can set io500_find_mpi to be "False" and write a wrapper
# script for this which sets up MPI as you would like. Then change
# io500_find_cmd to point to your wrapper script.
#io500_find_mpi="True"
#io500_find_cmd="$PWD/bin/pfind"
# uses stonewalling, run pfind
#io500_find_cmd_args="-s 300"
# for GPFS systems, you should probably use the provided mmfind wrapper
# if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
io500_find_mpi="False"
io500_find_cmd="$PWD/bin/mmfind.sh"
io500_find_cmd_args=""
}
function setup_mdreal {
echo -n ""
}
function run_benchmarks {
# Important: source the io500_fixed.sh script. Do not change it. If you discover
# a need to change it, please email the mailing list to discuss
source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}
# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
# TODO: Please add your information using the info-creator!
# EXAMPLE:
io500_info_system_name='xxx' # e.g. Oakforest-PACS
}
main
- ior_easy_read
-
[1569567551.941235] [ca-0134:123098:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 1700.00
[1569567551.942332] [ca-0142:120210:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.942332] [ca-0142:120211:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.942351] [ca-0142:120212:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.944817] [ca-0134:123100:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.944834] [ca-0134:123101:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.945194] [ca-0134:123099:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.946097] [ca-0142:120209:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.950283] [ca-0135:122312:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.950284] [ca-0135:122310:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.950283] [ca-0135:122313:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.950284] [ca-0135:122311:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.956282] [ca-0139:119529:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 1700.00
[1569567551.956282] [ca-0139:119528:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 1700.00
[1569567551.956282] [ca-0139:119527:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 1700.00
[1569567551.956284] [ca-0139:119526:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 1700.00
[1569567551.956604] [ca-0140:119968:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.956605] [ca-0140:119969:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.956604] [ca-0140:119971:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.956604] [ca-0140:119970:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.957518] [ca-0136:122215:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.957518] [ca-0136:122213:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.957518] [ca-0136:122212:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.957518] [ca-0136:122214:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.958032] [ca-0137:124111:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.958031] [ca-0137:124109:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.958031] [ca-0137:124110:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.958032] [ca-0137:124112:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.960961] [ca-0141:122072:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.960959] [ca-0141:122071:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.960959] [ca-0141:122070:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.960960] [ca-0141:122073:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.965091] [ca-0133:127310:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.965089] [ca-0133:127309:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.965087] [ca-0133:127308:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.965089] [ca-0133:127307:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.969833] [ca-0138:121412:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.969833] [ca-0138:121409:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.969833] [ca-0138:121410:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567551.969833] [ca-0138:121411:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Fri Sep 27 14:59:12 2019
Command line : /gpfs1/ddn/io500/io-500-dev/bin/ior -r -R -a MPIIO -t 1024k -s 250000 -F -i 1 -C -Q 1 -g -G 27 -k -e -o /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/ior_file_easy -O stoneWallingStatusFile=/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/stonewall
Machine : Linux ca-0133
TestID : 0
StartTime : Fri Sep 27 14:59:12 2019
Path : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy
FS : 1150.5 TiB Used FS: 10.8% Inodes: 128.0 Mi Used Inodes: 17.5%
Options:
api : MPIIO
apiVersion : (3.1)
test filename : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 250000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 40
clients per node : 4
repetitions : 1
xfersize : 1 MiB
blocksize : 1 MiB
aggregate filesize : 9.54 TiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
read 38128 1024.00 1024.00 0.207759 262.06 0.000802 262.27 0
Max Read: 38128.16 MiB/sec (39980.27 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 38128.16 38128.16 38128.16 0.00 38128.16 38128.16 38128.16 0.00 262.27334 NA NA 0 40 4 1 1 1 1 0 0 250000 1048576 1048576 10000000.0 MPIIO 0
Finished : Fri Sep 27 15:03:34 2019
- ior_easy_write
-
[1569565930.542807] [ca-0137:119303:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.542807] [ca-0137:119305:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.542807] [ca-0137:119302:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.542807] [ca-0137:119304:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548653] [ca-0134:118295:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548653] [ca-0134:118296:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548653] [ca-0134:118297:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548665] [ca-0134:118294:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548490] [ca-0135:117524:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548491] [ca-0135:117526:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548491] [ca-0135:117525:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548498] [ca-0135:117523:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548890] [ca-0138:116622:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548890] [ca-0138:116623:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548893] [ca-0138:116621:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548890] [ca-0138:116620:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.550573] [ca-0136:117419:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.550573] [ca-0136:117421:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.550572] [ca-0136:117420:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.550572] [ca-0136:117422:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.551291] [ca-0141:117125:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.551291] [ca-0141:117127:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.551292] [ca-0141:117124:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.551292] [ca-0141:117126:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.553235] [ca-0139:114730:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.553235] [ca-0139:114733:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.553234] [ca-0139:114732:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.553234] [ca-0139:114731:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.555271] [ca-0133:122384:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.555272] [ca-0133:122387:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.555272] [ca-0133:122385:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.555271] [ca-0133:122386:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556425] [ca-0140:115164:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556425] [ca-0140:115167:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556250] [ca-0142:115420:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556250] [ca-0142:115422:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556424] [ca-0140:115165:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556424] [ca-0140:115166:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.557919] [ca-0142:115421:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569565930.558313] [ca-0142:115423:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Fri Sep 27 14:32:11 2019
Command line : /gpfs1/ddn/io500/io-500-dev/bin/ior -w -a MPIIO -t 1024k -s 250000 -F -i 1 -C -Q 1 -g -G 27 -k -e -o /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/ior_file_easy -O stoneWallingStatusFile=/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux ca-0133
TestID : 0
StartTime : Fri Sep 27 14:32:11 2019
Path : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy
FS : 1150.5 TiB Used FS: 10.0% Inodes: 128.0 Mi Used Inodes: 5.8%
Options:
api : MPIIO
apiVersion : (3.1)
test filename : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 250000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 40
clients per node : 4
repetitions : 1
xfersize : 1 MiB
blocksize : 1 MiB
aggregate filesize : 9.54 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 223343 max: 250000 -- min data: 218.1 GiB mean data: 228.2 GiB time: 300.0s
write 30652 1024.00 1024.00 0.356160 325.89 0.002398 326.25 0
Max Write: 30651.81 MiB/sec (32140.75 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 30651.81 30651.81 30651.81 0.00 30651.81 30651.81 30651.81 0.00 326.24505 300.01 31157.24 0 40 4 1 1 1 1 0 0 250000 1048576 1048576 10000000.0 MPIIO 0
Finished : Fri Sep 27 14:37:37 2019
- ior_hard_read
-
[1569568040.462221] [ca-0136:123827:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.462221] [ca-0136:123826:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.461610] [ca-0138:123018:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 1700.00
[1569568040.461612] [ca-0138:123019:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 1700.00
[1569568040.461610] [ca-0138:123017:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 1700.00
[1569568040.463181] [ca-0136:123828:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.461611] [ca-0138:123016:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 1700.00
[1569568040.463191] [ca-0136:123825:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.464030] [ca-0134:124706:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.464031] [ca-0134:124708:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.464030] [ca-0134:124709:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.464031] [ca-0134:124707:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465399] [ca-0133:128862:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465400] [ca-0133:128860:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465400] [ca-0133:128859:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465399] [ca-0133:128861:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465667] [ca-0139:121134:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465667] [ca-0139:121133:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.467886] [ca-0139:121135:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.467886] [ca-0139:121136:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468470] [ca-0141:123713:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468471] [ca-0141:123712:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468470] [ca-0141:123711:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468470] [ca-0141:123714:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468832] [ca-0142:121817:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468831] [ca-0142:121818:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468831] [ca-0142:121816:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468831] [ca-0142:121815:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.473611] [ca-0135:123921:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.473612] [ca-0135:123922:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.473612] [ca-0135:123920:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.473612] [ca-0135:123923:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.474692] [ca-0137:125751:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.474692] [ca-0137:125754:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.474692] [ca-0137:125752:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.474692] [ca-0137:125753:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.476561] [ca-0140:121577:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.476561] [ca-0140:121579:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.476561] [ca-0140:121578:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568040.476561] [ca-0140:121580:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Fri Sep 27 15:07:20 2019
Command line : /gpfs1/ddn/io500/io-500-dev/bin/ior -r -R -s 200000 -a MPIIO -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/IOR_file -O stoneWallingStatusFile=/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/stonewall
Machine : Linux ca-0133
TestID : 0
StartTime : Fri Sep 27 15:07:20 2019
Path : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard
FS : 1150.5 TiB Used FS: 10.8% Inodes: 128.0 Mi Used Inodes: 17.5%
Options:
api : MPIIO
apiVersion : (3.1)
test filename : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 200000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 40
clients per node : 4
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 350.24 GiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
read 3468.03 45.91 45.91 0.291671 103.12 0.006846 103.41 0
Max Read: 3468.03 MiB/sec (3636.50 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 3468.03 3468.03 3468.03 0.00 77359.11 77359.11 77359.11 0.00 103.41381 NA NA 0 40 4 1 0 1 1 0 0 200000 47008 47008 358642.6 MPIIO 0
Finished : Fri Sep 27 15:09:04 2019
- ior_hard_write
-
[1569566609.227755] [ca-0134:120317:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227754] [ca-0134:120318:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227754] [ca-0134:120315:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227294] [ca-0138:118652:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227293] [ca-0138:118650:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227293] [ca-0138:118649:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227292] [ca-0138:118651:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.229254] [ca-0135:119551:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.229254] [ca-0135:119552:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.229254] [ca-0135:119550:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.229254] [ca-0135:119549:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.231073] [ca-0133:124355:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.231071] [ca-0133:124353:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.231073] [ca-0133:124354:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.231071] [ca-0133:124352:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.232847] [ca-0136:119447:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.232859] [ca-0136:119444:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.232986] [ca-0136:119445:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.232998] [ca-0136:119446:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.235384] [ca-0140:117196:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.235384] [ca-0140:117195:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.235384] [ca-0140:117194:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.235384] [ca-0140:117192:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237520] [ca-0141:119155:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237518] [ca-0141:119156:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237518] [ca-0141:119154:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237719] [ca-0137:121334:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237518] [ca-0141:119157:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237717] [ca-0137:121333:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237722] [ca-0137:121332:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237720] [ca-0137:121331:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239218] [ca-0139:116745:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239216] [ca-0139:116748:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239218] [ca-0139:116746:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239217] [ca-0139:116747:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239422] [ca-0142:117453:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239422] [ca-0142:117451:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239422] [ca-0142:117450:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239422] [ca-0142:117452:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566609.245436] [ca-0134:120316:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began : Fri Sep 27 14:43:29 2019
Command line : /gpfs1/ddn/io500/io-500-dev/bin/ior -w -s 200000 -a MPIIO -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/IOR_file -O stoneWallingStatusFile=/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux ca-0133
TestID : 0
StartTime : Fri Sep 27 14:43:29 2019
Path : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard
FS : 1150.5 TiB Used FS: 10.8% Inodes: 128.0 Mi Used Inodes: 14.3%
Options:
api : MPIIO
apiVersion : (3.1)
test filename : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 200000
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 40
clients per node : 4
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 350.24 GiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 110012 max: 200000 -- min data: 4.8 GiB mean data: 7.9 GiB time: 300.0s
write 843.35 45.91 45.91 0.261790 424.98 0.015274 425.26 0
Max Write: 843.35 MiB/sec (884.32 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 843.35 843.35 843.35 0.00 18812.03 18812.03 18812.03 0.00 425.25987 300.01 1084.47 0 40 4 1 0 1 1 0 0 200000 47008 47008 358642.6 MPIIO 0
Finished : Fri Sep 27 14:50:35 2019
- mdtest_easy_delete
-
[1569568355.363689] [ca-0142:123063:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.363689] [ca-0142:123065:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.363689] [ca-0142:123064:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.363689] [ca-0142:123066:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.364429] [ca-0135:125169:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.364430] [ca-0135:125172:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.364429] [ca-0135:125170:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.364430] [ca-0135:125171:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.365877] [ca-0141:124930:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.365877] [ca-0141:124929:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.365877] [ca-0141:124931:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.365878] [ca-0141:124928:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367203] [ca-0134:125957:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367324] [ca-0134:125954:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367338] [ca-0134:125956:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367342] [ca-0134:125955:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367989] [ca-0140:122828:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367442] [ca-0138:124268:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367442] [ca-0138:124269:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367442] [ca-0138:124267:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367442] [ca-0138:124270:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.369154] [ca-0140:122826:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.369155] [ca-0140:122829:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.369160] [ca-0140:122827:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.373665] [ca-0137:126965:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.373665] [ca-0137:126967:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.373665] [ca-0137:126966:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.373665] [ca-0137:126964:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.375932] [ca-0133:130017:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.375932] [ca-0133:130018:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376047] [ca-0136:125071:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376047] [ca-0136:125073:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376047] [ca-0136:125072:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376047] [ca-0136:125074:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376650] [ca-0133:130019:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376650] [ca-0133:130020:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.379318] [ca-0139:122351:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.379318] [ca-0139:122350:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.379318] [ca-0139:122349:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568355.379318] [ca-0139:122348:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 15:12:35 --
mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-r' '-F' '-P' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy' '-n' '450000' '-a' 'POSIX' '-u' '-L' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy-stonewall' '-N' '1'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB Used FS: 10.8% Inodes: 128.0 Mi Used Inodes: 17.5%
Nodemap: 1111000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 4 for each phase.
40 tasks, 18000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 31985.106 31985.089 31985.098 0.004
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 2.697 2.697 2.697 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 431.645 431.645 431.645 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.371 0.371 0.371 0.000
-- finished at 09/27/2019 15:19:47 --
- mdtest_easy_stat
-
[1569567815.844185] [ca-0138:122269:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.844185] [ca-0138:122268:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.844185] [ca-0138:122266:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.845379] [ca-0138:122267:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.848292] [ca-0134:123957:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.848292] [ca-0134:123958:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.848292] [ca-0134:123955:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.848293] [ca-0134:123956:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.849366] [ca-0135:123170:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.849367] [ca-0135:123171:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.849367] [ca-0135:123172:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.849368] [ca-0135:123169:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.850805] [ca-0139:120388:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.851562] [ca-0139:120387:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.851563] [ca-0139:120386:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.851589] [ca-0139:120385:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853185] [ca-0137:124970:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853183] [ca-0137:124971:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853183] [ca-0137:124969:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853183] [ca-0137:124968:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853035] [ca-0142:121064:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853034] [ca-0142:121065:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853035] [ca-0142:121066:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853036] [ca-0142:121063:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855554] [ca-0136:123071:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855557] [ca-0136:123072:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855555] [ca-0136:123073:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855556] [ca-0136:123074:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855821] [ca-0141:122929:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855822] [ca-0141:122927:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855822] [ca-0141:122926:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855822] [ca-0141:122928:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.862947] [ca-0140:120826:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.862947] [ca-0140:120829:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.862948] [ca-0140:120827:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.862947] [ca-0140:120828:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.864340] [ca-0133:128133:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.864340] [ca-0133:128134:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.864341] [ca-0133:128131:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567815.864358] [ca-0133:128132:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 15:03:36 --
mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-T' '-F' '-P' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy' '-n' '450000' '-a' 'POSIX' '-u' '-L' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy-stonewall' '-N' '1'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB Used FS: 10.8% Inodes: 128.0 Mi Used Inodes: 17.5%
Nodemap: 1111000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 4 for each phase.
40 tasks, 18000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 61871.242 61871.214 61871.228 0.008
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 223.144 223.144 223.144 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 09/27/2019 15:07:19 --
- mdtest_easy_write
-
[1569566258.786410] [ca-0141:118065:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.786410] [ca-0141:118067:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.786411] [ca-0141:118066:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.786410] [ca-0141:118064:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788927] [ca-0134:119234:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788927] [ca-0134:119237:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788927] [ca-0134:119235:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788471] [ca-0135:118467:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788470] [ca-0135:118470:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788470] [ca-0135:118469:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788470] [ca-0135:118468:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.791558] [ca-0137:120245:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.791557] [ca-0137:120244:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.791557] [ca-0137:120242:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.791557] [ca-0137:120243:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.792042] [ca-0134:119236:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.792432] [ca-0136:118360:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.792433] [ca-0136:118363:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.792433] [ca-0136:118361:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.792433] [ca-0136:118362:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.794102] [ca-0138:117560:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.794102] [ca-0138:117561:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.794101] [ca-0138:117559:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.794102] [ca-0138:117558:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.797133] [ca-0142:116359:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.797134] [ca-0142:116357:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.797137] [ca-0142:116360:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.797396] [ca-0142:116358:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.800719] [ca-0140:116105:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.800719] [ca-0140:116102:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.800719] [ca-0140:116103:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.800719] [ca-0140:116104:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.802135] [ca-0139:115668:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.802136] [ca-0139:115670:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.802136] [ca-0139:115669:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.802135] [ca-0139:115671:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.803461] [ca-0133:123271:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.803461] [ca-0133:123272:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.803461] [ca-0133:123273:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569566258.803460] [ca-0133:123270:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 14:37:39 --
mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-C' '-F' '-P' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy' '-n' '450000' '-a' 'POSIX' '-u' '-L' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB Used FS: 10.7% Inodes: 128.0 Mi Used Inodes: 5.8%
Nodemap: 1111000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 4 for each phase.
40 tasks, 18000000 files
Continue stonewall hit min: 294417 max: 345155 avg: 326077.9
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 39576.723 39576.702 39576.712 0.007
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 43495.537 NA
Tree creation : 7.823 7.823 7.823 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 348.847 348.846 348.847 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 299.873 NA
Tree creation : 0.128 0.128 0.128 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 09/27/2019 14:43:28 --
- mdtest_hard_delete
-
[1569569007.272411] [ca-0136:127038:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.272411] [ca-0136:127040:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.272411] [ca-0136:127037:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.272410] [ca-0136:127039:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.274399] [ca-0135:127136:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.274400] [ca-0135:127135:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.274400] [ca-0135:127134:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.274400] [ca-0135:127137:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.275775] [ca-0141:126894:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.275776] [ca-0141:126893:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.275775] [ca-0141:126897:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.277508] [ca-0138:126229:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.277508] [ca-0138:126231:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.277507] [ca-0138:126232:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.277507] [ca-0138:126230:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.278916] [ca-0141:126895:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.280063] [ca-0142:125032:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.280063] [ca-0142:125029:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.280063] [ca-0142:125030:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.280063] [ca-0142:125031:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282401] [ca-0137:128950:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282401] [ca-0137:128949:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282401] [ca-0137:128948:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282321] [ca-0139:124336:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282401] [ca-0137:128947:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282321] [ca-0139:124334:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282321] [ca-0139:124333:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.283104] [ca-0134:127911:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282322] [ca-0139:124335:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.283103] [ca-0134:127910:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.283103] [ca-0134:127912:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.283103] [ca-0134:127909:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286792] [ca-0133:1297 :0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286792] [ca-0133:1300 :0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286791] [ca-0133:1298 :0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286792] [ca-0133:1299 :0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286059] [ca-0140:124793:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286059] [ca-0140:124792:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286059] [ca-0140:124791:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286061] [ca-0140:124790:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 15:23:27 --
mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard' '-n' '75000' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB Used FS: 10.8% Inodes: 128.0 Mi Used Inodes: 8.0%
Nodemap: 1111000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 4 for each phase.
40 tasks, 3000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 4652.834 4652.832 4652.833 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.660 0.660 0.660 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 644.769 644.768 644.768 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 1.515 1.515 1.515 0.000
-- finished at 09/27/2019 15:34:14 --
- mdtest_hard_read
-
[1569568788.834654] [ca-0136:126259:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.834654] [ca-0136:126257:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836204] [ca-0140:124012:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836206] [ca-0140:124010:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836204] [ca-0140:124009:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836204] [ca-0140:124011:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.837793] [ca-0134:127141:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.837792] [ca-0134:127140:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.837792] [ca-0134:127142:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.837792] [ca-0134:127143:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.837432] [ca-0141:126123:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836896] [ca-0138:125452:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836896] [ca-0138:125450:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836896] [ca-0138:125451:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836896] [ca-0138:125449:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838355] [ca-0136:126258:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838355] [ca-0136:126256:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838742] [ca-0139:123566:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838743] [ca-0139:123567:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838742] [ca-0139:123564:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838743] [ca-0139:123565:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.839830] [ca-0141:126124:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.839929] [ca-0141:126125:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.839979] [ca-0141:126126:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.841687] [ca-0137:128174:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.841687] [ca-0137:128173:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.841687] [ca-0137:128172:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.845199] [ca-0137:128175:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.844960] [ca-0142:124250:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.844960] [ca-0142:124249:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.844960] [ca-0142:124248:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.844960] [ca-0142:124247:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.845378] [ca-0135:126354:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.845378] [ca-0135:126353:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.845379] [ca-0135:126356:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.845379] [ca-0135:126355:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.855655] [ca-0133:494 :0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.855655] [ca-0133:492 :0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.855655] [ca-0133:493 :0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568788.855667] [ca-0133:495 :0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 15:19:49 --
mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard' '-n' '75000' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB Used FS: 10.8% Inodes: 128.0 Mi Used Inodes: 9.1%
Nodemap: 1111000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 4 for each phase.
40 tasks, 3000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 13825.912 13825.907 13825.909 0.002
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 216.984 216.984 216.984 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 09/27/2019 15:23:26 --
- mdtest_hard_stat
-
[1569568145.642760] [ca-0135:124407:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.642758] [ca-0135:124408:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.642758] [ca-0135:124406:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.642758] [ca-0135:124409:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.643118] [ca-0138:123505:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.643118] [ca-0138:123504:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.643579] [ca-0138:123507:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.643591] [ca-0138:123506:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.647049] [ca-0140:122067:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.647708] [ca-0141:124167:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.647806] [ca-0140:122069:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.647806] [ca-0140:122068:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.648201] [ca-0140:122066:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.649266] [ca-0137:126207:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.649266] [ca-0137:126205:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.649266] [ca-0137:126206:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.649266] [ca-0137:126208:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.650029] [ca-0141:124168:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.650031] [ca-0141:124169:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.650030] [ca-0141:124166:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.652984] [ca-0139:121610:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.652984] [ca-0139:121609:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.653024] [ca-0139:121612:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.653031] [ca-0139:121611:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.654912] [ca-0134:125195:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.654912] [ca-0134:125194:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.654912] [ca-0134:125193:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.654912] [ca-0134:125192:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.655448] [ca-0136:124312:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.655447] [ca-0136:124310:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.655448] [ca-0136:124313:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.655447] [ca-0136:124311:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.657362] [ca-0142:122302:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.656491] [ca-0133:129310:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.656490] [ca-0133:129309:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.656494] [ca-0133:129311:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.656490] [ca-0133:129308:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.657607] [ca-0142:122303:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.657609] [ca-0142:122301:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569568145.657610] [ca-0142:122300:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 15:09:06 --
mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard' '-n' '75000' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB Used FS: 10.8% Inodes: 128.0 Mi Used Inodes: 17.5%
Nodemap: 1111000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 4 for each phase.
40 tasks, 3000000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 14404.686 14404.679 14404.683 0.002
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 208.266 208.266 208.266 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 09/27/2019 15:12:34 --
- mdtest_hard_write
-
[1569567036.266476] [ca-0138:119964:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.266475] [ca-0138:119962:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.266475] [ca-0138:119963:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.266568] [ca-0138:119961:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.271488] [ca-0136:120770:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.271488] [ca-0136:120771:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.271488] [ca-0136:120769:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.271489] [ca-0136:120768:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272282] [ca-0139:118082:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272069] [ca-0140:118522:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272069] [ca-0140:118524:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272069] [ca-0140:118523:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272282] [ca-0139:118084:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272069] [ca-0140:118521:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272282] [ca-0139:118081:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272412] [ca-0135:120864:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272412] [ca-0135:120865:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272412] [ca-0135:120863:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.275256] [ca-0141:120487:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.275297] [ca-0141:120486:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.275308] [ca-0141:120485:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.275323] [ca-0141:120484:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.275641] [ca-0139:118083:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.276015] [ca-0135:120866:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.277615] [ca-0137:122658:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.277616] [ca-0137:122657:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.277617] [ca-0137:122655:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.277616] [ca-0137:122656:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.279305] [ca-0134:121654:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.279305] [ca-0134:121655:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.279305] [ca-0134:121652:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.279305] [ca-0134:121653:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.281669] [ca-0133:125645:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.281669] [ca-0133:125643:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.281668] [ca-0133:125642:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.281668] [ca-0133:125644:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.283640] [ca-0142:118769:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.283640] [ca-0142:118768:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.283640] [ca-0142:118767:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
[1569567036.283640] [ca-0142:118770:0] sys.c:935 UCX WARN Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 14:50:36 --
mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard' '-n' '75000' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB Used FS: 10.8% Inodes: 128.0 Mi Used Inodes: 15.4%
Nodemap: 1111000000000000000000000000000000000000
V-0: Rank 0 Line 2128 Shifting ranks by 4 for each phase.
40 tasks, 3000000 files
Continue stonewall hit min: 67449 max: 75000 avg: 69840.0
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 9178.828 9178.824 9178.826 0.001
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 9311.902 NA
Tree creation : 9016.482 9016.482 9016.482 0.000
Tree removal : 0.000 0.000 0.000 0.000
SUMMARY time: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 326.839 326.839 326.839 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
File create (stonewall) : NA NA 300.003 NA
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 09/27/2019 14:56:03 --
- result_summary
-
[RESULT] BW phase 1 ior_easy_write 29.934 GB/s : time 326.25 seconds
[RESULT] IOPS phase 1 mdtest_easy_write 39.577 kiops : time 348.85 seconds
[RESULT] BW phase 2 ior_hard_write 0.824 GB/s : time 425.26 seconds
[RESULT] IOPS phase 2 mdtest_hard_write 9.179 kiops : time 326.84 seconds
[RESULT] IOPS phase 3 find 89.690 kiops : time 187.38 seconds
[RESULT] BW phase 3 ior_easy_read 37.234 GB/s : time 262.27 seconds
[RESULT] IOPS phase 4 mdtest_easy_stat 61.871 kiops : time 223.14 seconds
[RESULT] BW phase 4 ior_hard_read 3.387 GB/s : time 103.41 seconds
[RESULT] IOPS phase 5 mdtest_hard_stat 14.405 kiops : time 208.27 seconds
[RESULT] IOPS phase 6 mdtest_easy_delete 31.985 kiops : time 431.64 seconds
[RESULT] IOPS phase 7 mdtest_hard_read 13.826 kiops : time 216.98 seconds
[RESULT] IOPS phase 8 mdtest_hard_delete 4.653 kiops : time 647.71 seconds
[SCORE] Bandwidth 7.46704 GB/s : IOPS 22.2352 kiops : TOTAL 12.8853