Hadley

Institution Nanyang Technological University
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.5.1804
Client Kernel Version 3.10.0-862.2.3.el7.x86_64

DATA SERVER

Storage Type
Volatile Memory
Storage Interface
Network
Software Version
OS Version

INFORMATION

Client Nodes 10
Client Total Procs 640
Metadata Nodes 0
Metadata Storage Devices 0
Data Nodes 0
Data Storage Devices 0

METADATA

Easy Write 39.58 kIOP/s
Easy Stat 61.87 kIOP/s
Easy Delete 31.98 kIOP/s
Hard Write 9.18 kIOP/s
Hard Read 13.83 kIOP/s
Hard Stat 14.40 kIOP/s
Hard Delete 4.65 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 40 -N 4 --allow-run-as-root -hostfile hostfile"
}

function setup_ior_easy {
  #io500_ior_easy_params="-t 2048k -b \${io500_ior_easy_size}m -F"
  io500_ior_easy_params="-a MPIIO -t 1024k -s 250000 -F"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-a POSIX -u -L" # unique dir per thread, files only at leaves
  io500_mdtest_easy_files_per_proc="450000"
}

function setup_ior_hard {
  io500_ior_hard_api="MPIIO"
  io500_ior_hard_api_specific_options=""
  io500_ior_hard_writes_per_proc="200000"
}

function setup_mdt_hard {
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
  io500_mdtest_hard_files_per_proc="75000"
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  #io500_find_mpi="True"
  #io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind
  #io500_find_cmd_args="-s 300"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  io500_find_mpi="False"
  io500_find_cmd="$PWD/bin/mmfind.sh"
  io500_find_cmd_args=""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
  # TODO: Please add your information using the info-creator!
  # EXAMPLE:
  io500_info_system_name='xxx'      # e.g. Oakforest-PACS
}

main
ior_easy_read
[1569567551.941235] [ca-0134:123098:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 1700.00
[1569567551.942332] [ca-0142:120210:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.942332] [ca-0142:120211:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.942351] [ca-0142:120212:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.944817] [ca-0134:123100:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.944834] [ca-0134:123101:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.945194] [ca-0134:123099:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.946097] [ca-0142:120209:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.950283] [ca-0135:122312:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.950284] [ca-0135:122310:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.950283] [ca-0135:122313:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.950284] [ca-0135:122311:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.956282] [ca-0139:119529:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 1700.00
[1569567551.956282] [ca-0139:119528:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 1700.00
[1569567551.956282] [ca-0139:119527:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 1700.00
[1569567551.956284] [ca-0139:119526:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 1700.00
[1569567551.956604] [ca-0140:119968:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.956605] [ca-0140:119969:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.956604] [ca-0140:119971:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.956604] [ca-0140:119970:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.957518] [ca-0136:122215:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.957518] [ca-0136:122213:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.957518] [ca-0136:122212:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.957518] [ca-0136:122214:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.958032] [ca-0137:124111:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.958031] [ca-0137:124109:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.958031] [ca-0137:124110:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.958032] [ca-0137:124112:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.960961] [ca-0141:122072:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.960959] [ca-0141:122071:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.960959] [ca-0141:122070:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.960960] [ca-0141:122073:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.965091] [ca-0133:127310:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.965089] [ca-0133:127309:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.965087] [ca-0133:127308:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.965089] [ca-0133:127307:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.969833] [ca-0138:121412:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.969833] [ca-0138:121409:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.969833] [ca-0138:121410:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567551.969833] [ca-0138:121411:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Sep 27 14:59:12 2019
Command line        : /gpfs1/ddn/io500/io-500-dev/bin/ior -r -R -a MPIIO -t 1024k -s 250000 -F -i 1 -C -Q 1 -g -G 27 -k -e -o /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/ior_file_easy -O stoneWallingStatusFile=/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/stonewall
Machine             : Linux ca-0133
TestID              : 0
StartTime           : Fri Sep 27 14:59:12 2019
Path                : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy
FS                  : 1150.5 TiB   Used FS: 10.8%   Inodes: 128.0 Mi   Used Inodes: 17.5%

Options: 
api                 : MPIIO
apiVersion          : (3.1)
test filename       : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 250000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 40
clients per node    : 4
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 1 MiB
aggregate filesize  : 9.54 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
read      38128      1024.00    1024.00    0.207759   262.06     0.000802   262.27     0   
Max Read:  38128.16 MiB/sec (39980.27 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        38128.16   38128.16   38128.16       0.00   38128.16   38128.16   38128.16       0.00  262.27334         NA            NA     0     40   4    1   1     1        1         0    0 250000  1048576  1048576 10000000.0 MPIIO      0
Finished            : Fri Sep 27 15:03:34 2019
ior_easy_write
[1569565930.542807] [ca-0137:119303:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.542807] [ca-0137:119305:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.542807] [ca-0137:119302:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.542807] [ca-0137:119304:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548653] [ca-0134:118295:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548653] [ca-0134:118296:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548653] [ca-0134:118297:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548665] [ca-0134:118294:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548490] [ca-0135:117524:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548491] [ca-0135:117526:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548491] [ca-0135:117525:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548498] [ca-0135:117523:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548890] [ca-0138:116622:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548890] [ca-0138:116623:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548893] [ca-0138:116621:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.548890] [ca-0138:116620:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.550573] [ca-0136:117419:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.550573] [ca-0136:117421:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.550572] [ca-0136:117420:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.550572] [ca-0136:117422:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.551291] [ca-0141:117125:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.551291] [ca-0141:117127:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.551292] [ca-0141:117124:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.551292] [ca-0141:117126:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.553235] [ca-0139:114730:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.553235] [ca-0139:114733:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.553234] [ca-0139:114732:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.553234] [ca-0139:114731:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.555271] [ca-0133:122384:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.555272] [ca-0133:122387:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.555272] [ca-0133:122385:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.555271] [ca-0133:122386:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556425] [ca-0140:115164:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556425] [ca-0140:115167:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556250] [ca-0142:115420:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556250] [ca-0142:115422:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556424] [ca-0140:115165:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.556424] [ca-0140:115166:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.557919] [ca-0142:115421:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569565930.558313] [ca-0142:115423:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Sep 27 14:32:11 2019
Command line        : /gpfs1/ddn/io500/io-500-dev/bin/ior -w -a MPIIO -t 1024k -s 250000 -F -i 1 -C -Q 1 -g -G 27 -k -e -o /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/ior_file_easy -O stoneWallingStatusFile=/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux ca-0133
TestID              : 0
StartTime           : Fri Sep 27 14:32:11 2019
Path                : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy
FS                  : 1150.5 TiB   Used FS: 10.0%   Inodes: 128.0 Mi   Used Inodes: 5.8%

Options: 
api                 : MPIIO
apiVersion          : (3.1)
test filename       : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 250000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 40
clients per node    : 4
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 1 MiB
aggregate filesize  : 9.54 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 223343 max: 250000 -- min data: 218.1 GiB mean data: 228.2 GiB time: 300.0s
write     30652      1024.00    1024.00    0.356160   325.89     0.002398   326.25     0   
Max Write: 30651.81 MiB/sec (32140.75 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       30651.81   30651.81   30651.81       0.00   30651.81   30651.81   30651.81       0.00  326.24505     300.01      31157.24     0     40   4    1   1     1        1         0    0 250000  1048576  1048576 10000000.0 MPIIO      0
Finished            : Fri Sep 27 14:37:37 2019
ior_hard_read
[1569568040.462221] [ca-0136:123827:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.462221] [ca-0136:123826:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.461610] [ca-0138:123018:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 1700.00
[1569568040.461612] [ca-0138:123019:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 1700.00
[1569568040.461610] [ca-0138:123017:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 1700.00
[1569568040.463181] [ca-0136:123828:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.461611] [ca-0138:123016:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 1700.00
[1569568040.463191] [ca-0136:123825:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.464030] [ca-0134:124706:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.464031] [ca-0134:124708:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.464030] [ca-0134:124709:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.464031] [ca-0134:124707:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465399] [ca-0133:128862:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465400] [ca-0133:128860:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465400] [ca-0133:128859:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465399] [ca-0133:128861:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465667] [ca-0139:121134:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.465667] [ca-0139:121133:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.467886] [ca-0139:121135:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.467886] [ca-0139:121136:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468470] [ca-0141:123713:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468471] [ca-0141:123712:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468470] [ca-0141:123711:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468470] [ca-0141:123714:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468832] [ca-0142:121817:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468831] [ca-0142:121818:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468831] [ca-0142:121816:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.468831] [ca-0142:121815:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.473611] [ca-0135:123921:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.473612] [ca-0135:123922:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.473612] [ca-0135:123920:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.473612] [ca-0135:123923:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.474692] [ca-0137:125751:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.474692] [ca-0137:125754:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.474692] [ca-0137:125752:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.474692] [ca-0137:125753:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.476561] [ca-0140:121577:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.476561] [ca-0140:121579:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.476561] [ca-0140:121578:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568040.476561] [ca-0140:121580:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Sep 27 15:07:20 2019
Command line        : /gpfs1/ddn/io500/io-500-dev/bin/ior -r -R -s 200000 -a MPIIO -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/IOR_file -O stoneWallingStatusFile=/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/stonewall
Machine             : Linux ca-0133
TestID              : 0
StartTime           : Fri Sep 27 15:07:20 2019
Path                : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard
FS                  : 1150.5 TiB   Used FS: 10.8%   Inodes: 128.0 Mi   Used Inodes: 17.5%

Options: 
api                 : MPIIO
apiVersion          : (3.1)
test filename       : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 200000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 40
clients per node    : 4
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 350.24 GiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
read      3468.03    45.91      45.91      0.291671   103.12     0.006846   103.41     0   
Max Read:  3468.03 MiB/sec (3636.50 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         3468.03    3468.03    3468.03       0.00   77359.11   77359.11   77359.11       0.00  103.41381         NA            NA     0     40   4    1   0     1        1         0    0 200000    47008    47008  358642.6 MPIIO      0
Finished            : Fri Sep 27 15:09:04 2019
ior_hard_write
[1569566609.227755] [ca-0134:120317:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227754] [ca-0134:120318:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227754] [ca-0134:120315:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227294] [ca-0138:118652:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227293] [ca-0138:118650:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227293] [ca-0138:118649:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.227292] [ca-0138:118651:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.229254] [ca-0135:119551:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.229254] [ca-0135:119552:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.229254] [ca-0135:119550:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.229254] [ca-0135:119549:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.231073] [ca-0133:124355:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.231071] [ca-0133:124353:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.231073] [ca-0133:124354:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.231071] [ca-0133:124352:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.232847] [ca-0136:119447:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.232859] [ca-0136:119444:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.232986] [ca-0136:119445:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.232998] [ca-0136:119446:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.235384] [ca-0140:117196:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.235384] [ca-0140:117195:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.235384] [ca-0140:117194:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.235384] [ca-0140:117192:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237520] [ca-0141:119155:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237518] [ca-0141:119156:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237518] [ca-0141:119154:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237719] [ca-0137:121334:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237518] [ca-0141:119157:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237717] [ca-0137:121333:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237722] [ca-0137:121332:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.237720] [ca-0137:121331:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239218] [ca-0139:116745:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239216] [ca-0139:116748:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239218] [ca-0139:116746:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239217] [ca-0139:116747:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239422] [ca-0142:117453:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239422] [ca-0142:117451:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239422] [ca-0142:117450:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.239422] [ca-0142:117452:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566609.245436] [ca-0134:120316:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Sep 27 14:43:29 2019
Command line        : /gpfs1/ddn/io500/io-500-dev/bin/ior -w -s 200000 -a MPIIO -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/IOR_file -O stoneWallingStatusFile=/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux ca-0133
TestID              : 0
StartTime           : Fri Sep 27 14:43:29 2019
Path                : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard
FS                  : 1150.5 TiB   Used FS: 10.8%   Inodes: 128.0 Mi   Used Inodes: 14.3%

Options: 
api                 : MPIIO
apiVersion          : (3.1)
test filename       : /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 200000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 40
clients per node    : 4
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 350.24 GiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 110012 max: 200000 -- min data: 4.8 GiB mean data: 7.9 GiB time: 300.0s
write     843.35     45.91      45.91      0.261790   424.98     0.015274   425.26     0   
Max Write: 843.35 MiB/sec (884.32 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write         843.35     843.35     843.35       0.00   18812.03   18812.03   18812.03       0.00  425.25987     300.01       1084.47     0     40   4    1   0     1        1         0    0 200000    47008    47008  358642.6 MPIIO      0
Finished            : Fri Sep 27 14:50:35 2019
mdtest_easy_delete
[1569568355.363689] [ca-0142:123063:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.363689] [ca-0142:123065:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.363689] [ca-0142:123064:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.363689] [ca-0142:123066:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.364429] [ca-0135:125169:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.364430] [ca-0135:125172:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.364429] [ca-0135:125170:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.364430] [ca-0135:125171:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.365877] [ca-0141:124930:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.365877] [ca-0141:124929:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.365877] [ca-0141:124931:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.365878] [ca-0141:124928:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367203] [ca-0134:125957:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367324] [ca-0134:125954:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367338] [ca-0134:125956:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367342] [ca-0134:125955:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367989] [ca-0140:122828:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367442] [ca-0138:124268:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367442] [ca-0138:124269:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367442] [ca-0138:124267:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.367442] [ca-0138:124270:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.369154] [ca-0140:122826:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.369155] [ca-0140:122829:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.369160] [ca-0140:122827:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.373665] [ca-0137:126965:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.373665] [ca-0137:126967:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.373665] [ca-0137:126966:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.373665] [ca-0137:126964:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.375932] [ca-0133:130017:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.375932] [ca-0133:130018:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376047] [ca-0136:125071:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376047] [ca-0136:125073:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376047] [ca-0136:125072:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376047] [ca-0136:125074:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376650] [ca-0133:130019:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.376650] [ca-0133:130020:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.379318] [ca-0139:122351:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.379318] [ca-0139:122350:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.379318] [ca-0139:122349:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568355.379318] [ca-0139:122348:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 15:12:35 --

mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-r' '-F' '-P' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy' '-n' '450000' '-a' 'POSIX' '-u' '-L' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy-stonewall' '-N' '1'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB   Used FS: 10.8%   Inodes: 128.0 Mi   Used Inodes: 17.5%

Nodemap: 1111000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 4 for each phase.
40 tasks, 18000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      31985.106      31985.089      31985.098          0.004
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.697          2.697          2.697          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        431.645        431.645        431.645          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.371          0.371          0.371          0.000
-- finished at 09/27/2019 15:19:47 --

mdtest_easy_stat
[1569567815.844185] [ca-0138:122269:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.844185] [ca-0138:122268:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.844185] [ca-0138:122266:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.845379] [ca-0138:122267:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.848292] [ca-0134:123957:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.848292] [ca-0134:123958:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.848292] [ca-0134:123955:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.848293] [ca-0134:123956:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.849366] [ca-0135:123170:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.849367] [ca-0135:123171:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.849367] [ca-0135:123172:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.849368] [ca-0135:123169:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.850805] [ca-0139:120388:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.851562] [ca-0139:120387:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.851563] [ca-0139:120386:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.851589] [ca-0139:120385:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853185] [ca-0137:124970:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853183] [ca-0137:124971:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853183] [ca-0137:124969:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853183] [ca-0137:124968:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853035] [ca-0142:121064:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853034] [ca-0142:121065:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853035] [ca-0142:121066:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.853036] [ca-0142:121063:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855554] [ca-0136:123071:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855557] [ca-0136:123072:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855555] [ca-0136:123073:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855556] [ca-0136:123074:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855821] [ca-0141:122929:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855822] [ca-0141:122927:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855822] [ca-0141:122926:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.855822] [ca-0141:122928:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.862947] [ca-0140:120826:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.862947] [ca-0140:120829:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.862948] [ca-0140:120827:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.862947] [ca-0140:120828:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.864340] [ca-0133:128133:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.864340] [ca-0133:128134:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.864341] [ca-0133:128131:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567815.864358] [ca-0133:128132:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 15:03:36 --

mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-T' '-F' '-P' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy' '-n' '450000' '-a' 'POSIX' '-u' '-L' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy-stonewall' '-N' '1'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB   Used FS: 10.8%   Inodes: 128.0 Mi   Used Inodes: 17.5%

Nodemap: 1111000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 4 for each phase.
40 tasks, 18000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      61871.242      61871.214      61871.228          0.008
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        223.144        223.144        223.144          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 09/27/2019 15:07:19 --

mdtest_easy_write
[1569566258.786410] [ca-0141:118065:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.786410] [ca-0141:118067:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.786411] [ca-0141:118066:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.786410] [ca-0141:118064:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788927] [ca-0134:119234:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788927] [ca-0134:119237:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788927] [ca-0134:119235:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788471] [ca-0135:118467:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788470] [ca-0135:118470:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788470] [ca-0135:118469:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.788470] [ca-0135:118468:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.791558] [ca-0137:120245:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.791557] [ca-0137:120244:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.791557] [ca-0137:120242:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.791557] [ca-0137:120243:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.792042] [ca-0134:119236:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.792432] [ca-0136:118360:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.792433] [ca-0136:118363:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.792433] [ca-0136:118361:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.792433] [ca-0136:118362:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.794102] [ca-0138:117560:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.794102] [ca-0138:117561:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.794101] [ca-0138:117559:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.794102] [ca-0138:117558:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.797133] [ca-0142:116359:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.797134] [ca-0142:116357:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.797137] [ca-0142:116360:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.797396] [ca-0142:116358:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.800719] [ca-0140:116105:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.800719] [ca-0140:116102:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.800719] [ca-0140:116103:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.800719] [ca-0140:116104:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.802135] [ca-0139:115668:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.802136] [ca-0139:115670:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.802136] [ca-0139:115669:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.802135] [ca-0139:115671:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.803461] [ca-0133:123271:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.803461] [ca-0133:123272:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.803461] [ca-0133:123273:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569566258.803460] [ca-0133:123270:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 14:37:39 --

mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-C' '-F' '-P' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy' '-n' '450000' '-a' 'POSIX' '-u' '-L' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB   Used FS: 10.7%   Inodes: 128.0 Mi   Used Inodes: 5.8%

Nodemap: 1111000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 4 for each phase.
40 tasks, 18000000 files
Continue stonewall hit min: 294417 max: 345155 avg: 326077.9 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      39576.723      39576.702      39576.712          0.007
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      43495.537             NA
   Tree creation             :          7.823          7.823          7.823          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        348.847        348.846        348.847          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.873             NA
   Tree creation             :          0.128          0.128          0.128          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 09/27/2019 14:43:28 --

mdtest_hard_delete
[1569569007.272411] [ca-0136:127038:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.272411] [ca-0136:127040:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.272411] [ca-0136:127037:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.272410] [ca-0136:127039:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.274399] [ca-0135:127136:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.274400] [ca-0135:127135:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.274400] [ca-0135:127134:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.274400] [ca-0135:127137:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.275775] [ca-0141:126894:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.275776] [ca-0141:126893:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.275775] [ca-0141:126897:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.277508] [ca-0138:126229:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.277508] [ca-0138:126231:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.277507] [ca-0138:126232:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.277507] [ca-0138:126230:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.278916] [ca-0141:126895:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.280063] [ca-0142:125032:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.280063] [ca-0142:125029:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.280063] [ca-0142:125030:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.280063] [ca-0142:125031:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282401] [ca-0137:128950:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282401] [ca-0137:128949:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282401] [ca-0137:128948:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282321] [ca-0139:124336:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282401] [ca-0137:128947:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282321] [ca-0139:124334:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282321] [ca-0139:124333:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.283104] [ca-0134:127911:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.282322] [ca-0139:124335:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.283103] [ca-0134:127910:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.283103] [ca-0134:127912:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.283103] [ca-0134:127909:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286792] [ca-0133:1297 :0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286792] [ca-0133:1300 :0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286791] [ca-0133:1298 :0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286792] [ca-0133:1299 :0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286059] [ca-0140:124793:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286059] [ca-0140:124792:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286059] [ca-0140:124791:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569569007.286061] [ca-0140:124790:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 15:23:27 --

mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard' '-n' '75000' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB   Used FS: 10.8%   Inodes: 128.0 Mi   Used Inodes: 8.0%

Nodemap: 1111000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 4 for each phase.
40 tasks, 3000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       4652.834       4652.832       4652.833          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.660          0.660          0.660          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        644.769        644.768        644.768          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.515          1.515          1.515          0.000
-- finished at 09/27/2019 15:34:14 --

mdtest_hard_read
[1569568788.834654] [ca-0136:126259:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.834654] [ca-0136:126257:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836204] [ca-0140:124012:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836206] [ca-0140:124010:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836204] [ca-0140:124009:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836204] [ca-0140:124011:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.837793] [ca-0134:127141:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.837792] [ca-0134:127140:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.837792] [ca-0134:127142:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.837792] [ca-0134:127143:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.837432] [ca-0141:126123:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836896] [ca-0138:125452:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836896] [ca-0138:125450:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836896] [ca-0138:125451:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.836896] [ca-0138:125449:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838355] [ca-0136:126258:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838355] [ca-0136:126256:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838742] [ca-0139:123566:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838743] [ca-0139:123567:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838742] [ca-0139:123564:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.838743] [ca-0139:123565:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.839830] [ca-0141:126124:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.839929] [ca-0141:126125:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.839979] [ca-0141:126126:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.841687] [ca-0137:128174:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.841687] [ca-0137:128173:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.841687] [ca-0137:128172:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.845199] [ca-0137:128175:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.844960] [ca-0142:124250:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.844960] [ca-0142:124249:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.844960] [ca-0142:124248:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.844960] [ca-0142:124247:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.845378] [ca-0135:126354:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.845378] [ca-0135:126353:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.845379] [ca-0135:126356:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.845379] [ca-0135:126355:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.855655] [ca-0133:494  :0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.855655] [ca-0133:492  :0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.855655] [ca-0133:493  :0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568788.855667] [ca-0133:495  :0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 15:19:49 --

mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard' '-n' '75000' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB   Used FS: 10.8%   Inodes: 128.0 Mi   Used Inodes: 9.1%

Nodemap: 1111000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 4 for each phase.
40 tasks, 3000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      13825.912      13825.907      13825.909          0.002
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        216.984        216.984        216.984          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 09/27/2019 15:23:26 --

mdtest_hard_stat
[1569568145.642760] [ca-0135:124407:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.642758] [ca-0135:124408:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.642758] [ca-0135:124406:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.642758] [ca-0135:124409:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.643118] [ca-0138:123505:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.643118] [ca-0138:123504:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.643579] [ca-0138:123507:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.643591] [ca-0138:123506:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.647049] [ca-0140:122067:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.647708] [ca-0141:124167:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.647806] [ca-0140:122069:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.647806] [ca-0140:122068:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.648201] [ca-0140:122066:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.649266] [ca-0137:126207:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.649266] [ca-0137:126205:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.649266] [ca-0137:126206:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.649266] [ca-0137:126208:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.650029] [ca-0141:124168:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.650031] [ca-0141:124169:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.650030] [ca-0141:124166:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.652984] [ca-0139:121610:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.652984] [ca-0139:121609:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.653024] [ca-0139:121612:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.653031] [ca-0139:121611:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.654912] [ca-0134:125195:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.654912] [ca-0134:125194:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.654912] [ca-0134:125193:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.654912] [ca-0134:125192:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.655448] [ca-0136:124312:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.655447] [ca-0136:124310:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.655448] [ca-0136:124313:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.655447] [ca-0136:124311:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.657362] [ca-0142:122302:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.656491] [ca-0133:129310:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.656490] [ca-0133:129309:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.656494] [ca-0133:129311:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.656490] [ca-0133:129308:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.657607] [ca-0142:122303:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.657609] [ca-0142:122301:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569568145.657610] [ca-0142:122300:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 15:09:06 --

mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard' '-n' '75000' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB   Used FS: 10.8%   Inodes: 128.0 Mi   Used Inodes: 17.5%

Nodemap: 1111000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 4 for each phase.
40 tasks, 3000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      14404.686      14404.679      14404.683          0.002
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        208.266        208.266        208.266          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 09/27/2019 15:12:34 --

mdtest_hard_write
[1569567036.266476] [ca-0138:119964:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.266475] [ca-0138:119962:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.266475] [ca-0138:119963:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.266568] [ca-0138:119961:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.271488] [ca-0136:120770:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.271488] [ca-0136:120771:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.271488] [ca-0136:120769:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.271489] [ca-0136:120768:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272282] [ca-0139:118082:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272069] [ca-0140:118522:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272069] [ca-0140:118524:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272069] [ca-0140:118523:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272282] [ca-0139:118084:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272069] [ca-0140:118521:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272282] [ca-0139:118081:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272412] [ca-0135:120864:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272412] [ca-0135:120865:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.272412] [ca-0135:120863:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.275256] [ca-0141:120487:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.275297] [ca-0141:120486:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.275308] [ca-0141:120485:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.275323] [ca-0141:120484:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.275641] [ca-0139:118083:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.276015] [ca-0135:120866:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.277615] [ca-0137:122658:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.277616] [ca-0137:122657:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.277617] [ca-0137:122655:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.277616] [ca-0137:122656:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.279305] [ca-0134:121654:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.279305] [ca-0134:121655:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.279305] [ca-0134:121652:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.279305] [ca-0134:121653:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.281669] [ca-0133:125645:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.281669] [ca-0133:125643:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.281668] [ca-0133:125642:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.281668] [ca-0133:125644:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.283640] [ca-0142:118769:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.283640] [ca-0142:118768:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.283640] [ca-0142:118767:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
[1569567036.283640] [ca-0142:118770:0]            sys.c:935  UCX  WARN  Conflicting CPU frequencies detected, using: 2200.00
-- started at 09/27/2019 14:50:36 --

mdtest-3.3.0+dev was launched with 40 total task(s) on 10 node(s)
Command line used: /gpfs1/ddn/io500/io-500-dev/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard' '-n' '75000' '-x' '/gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /gpfs1/ddn/io500/io-500-dev/datafiles/io500.2019.09.27-14.32.09
FS: 1150.5 TiB   Used FS: 10.8%   Inodes: 128.0 Mi   Used Inodes: 15.4%

Nodemap: 1111000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 4 for each phase.
40 tasks, 3000000 files
Continue stonewall hit min: 67449 max: 75000 avg: 69840.0 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       9178.828       9178.824       9178.826          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA       9311.902             NA
   Tree creation             :       9016.482       9016.482       9016.482          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        326.839        326.839        326.839          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.003             NA
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 09/27/2019 14:56:03 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               29.934 GB/s : time 326.25 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               39.577 kiops : time 348.85 seconds
[RESULT] BW   phase 2            ior_hard_write                0.824 GB/s : time 425.26 seconds
[RESULT] IOPS phase 2         mdtest_hard_write                9.179 kiops : time 326.84 seconds
[RESULT] IOPS phase 3                      find               89.690 kiops : time 187.38 seconds
[RESULT] BW   phase 3             ior_easy_read               37.234 GB/s : time 262.27 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat               61.871 kiops : time 223.14 seconds
[RESULT] BW   phase 4             ior_hard_read                3.387 GB/s : time 103.41 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               14.405 kiops : time 208.27 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               31.985 kiops : time 431.64 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               13.826 kiops : time 216.98 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                4.653 kiops : time 647.71 seconds
[SCORE] Bandwidth 7.46704 GB/s : IOPS 22.2352 kiops : TOTAL 12.8853