Tianhe-2E

Institution National Supercomputing Center in Changsha
Client Procs Per Node
Client Operating System Kylin Linux
Client Operating System Version 4
Client Kernel Version 3.10.0

DATA SERVER

Storage Type SSD
Volatile Memory 17.6TB
Storage Interface NVMe
Network OPA
Software Version 2.12.2
OS Version 4

INFORMATION

Client Nodes 480
Client Total Procs 5,280
Metadata Nodes 40
Metadata Storage Devices 1
Data Nodes 52
Data Storage Devices 572

METADATA

Easy Write 1,099.18 kIOP/s
Easy Stat 2,416.40 kIOP/s
Easy Delete 961.62 kIOP/s
Hard Write 554.07 kIOP/s
Hard Read 779.21 kIOP/s
Hard Stat 2,419.15 kIOP/s
Hard Delete 203.50 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

#set -euo pipefail  # better error handling

export LD_LIBRARY_PATH=/vol7/io500_20190930/mvapich2-new/lib/:$LD_LIBRARY_PATH

function usage(){
  cat << EOF
  Usage: 
    $0 node-number nodename tasks-per-node
EOF
  exit 0
}

[ $# -eq 2 ] || usage

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.

io500_run_ior_easy="True"
io500_run_md_easy="True"
io500_run_ior_hard="True"
io500_run_md_hard="True"
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False"
io500_stonewall_timer=300 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...
io500_rules="regular"

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths $1 $2
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept

  io500_ior_easy_dir=${io500_workdir}/ior_easy
  io500_ior_hard_dir=${io500_workdir}/ior_hard
  io500_mdt_easy_dir=${io500_workdir}/mdt_easy
  io500_mdt_hard_dir=${io500_workdir}/mdt_hard

  mkdir -p ${io500_workdir} ${io500_result_dir} ${io500_ior_easy_dir} ${io500_ior_hard_dir} ${io500_mdt_easy_dir} ${io500_mdt_hard_dir}

#ior hard
  lfs setstripe -c 572 -S 8M ${io500_ior_hard_dir}
#DNE1 for mdtset easy
  lfs mkdir -c 40 "$io500_mdt_easy_dir/#test-dir.0-0"
#Use DoM for mdt_easy
  lfs setstripe -L mdt -E 64K "$io500_mdt_easy_dir/#test-dir.0-0"

#Use DNE2 striped directory for mdt_hard
  lfs setdirstripe -c 40 -D $io500_mdt_hard_dir
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="yhrun"
  io500_mpiargs="-w $1 --ntasks-per-node $2"
}

function setup_ior_easy {
  # io500_ior_easy_size is the amount of data written per rank in MiB units,
  # but it can be any number as long as it is somehow used to scale the IOR
  # runtime as part of io500_ior_easy_params
  io500_ior_easy_size=40
  # 2M writes, 2 GB per proc, file per proc
  io500_ior_easy_params="-a=POSIX --posix.odirect -t 1m -C -b ${io500_ior_easy_size}g -F"
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
  io500_mdtest_easy_files_per_proc=67000
}

function setup_ior_hard {
  io500_ior_hard_api="MPIIO"
  io500_ior_hard_api_specific_options="-U /vol7/io500_20190930/mpio -H"
  io500_ior_hard_writes_per_proc=52800
}

function setup_mdt_hard {
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
  io500_mdtest_hard_files_per_proc=38888
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  io500_find_mpi="False"
  io500_find_cmd="yhrun -N 200 -n 400 $PWD/bin/pfind"
  io500_find_cmd_args="-N -P -s $io500_stonewall_timer -r $io500_result_dir/pfind_results >$io500_result_dir/find.txt 2>&1"


  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Add key/value pairs defining your system
# Feel free to add extra ones if you'd like
function extra_description {
  # top level info
  io500_info_system_name='Tianhe-2E'      # e.g. Oakforest-PACS
  io500_info_institute_name='National Supercomputing Center in Changsha'   # e.g. JCAHPC
  io500_info_storage_age_in_months='3' # not install date but age since last refresh
  io500_info_storage_install_date='08/2019'  # MM/YY
  io500_info_filesystem='Lustre'     # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
  io500_info_filesystem_version='2.12.2'
  io500_info_filesystem_vendor='National University of Defense Technology'
  # client side info
  io500_info_num_client_nodes='480'
  io500_info_procs_per_node='11'
  # server side info
  io500_info_num_metadata_server_nodes='40'
  io500_info_num_data_server_nodes='52'
  io500_info_num_data_storage_devices='572'  # if you have 5 data servers, and each has 5 drives, then this number is 25
  io500_info_num_metadata_storage_devices='40'  # if you have 2 metadata servers, and each has 5 drives, then this number is 10
  io500_info_data_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_metadata_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_storage_network='omnipath' # infiniband, omnipath, ethernet, etc
  io500_info_storage_interface='NVMe' # SAS, SATA, NVMe, etc
}
main $1 $2
ior_easy_read
yhrun: job 59571 queued and waiting for resources
yhrun: job 59571 has been allocated resources
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 10 18:17:35 2019
Command line        : /vol7/io500_20190930/bin/ior -r -R -a=POSIX --posix.odirect -t 1m -C -b 40g -F -i 1 -C -Q 1 -g -G 27 -k -e -o /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_easy/ior_file_easy -O stoneWallingStatusFile=/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_easy/stonewall
Machine             : Linux cn1
TestID              : 0
StartTime           : Sun Nov 10 18:17:35 2019
Path                : /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_easy
FS                  : 824.1 TiB   Used FS: 51.6%   Inodes: 7452.5 Mi   Used Inodes: 11.3%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
tasks               : 5280
clients per node    : 11
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 40 GiB
aggregate filesize  : 206.25 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
read      628252     41943040   1024.00    0.091296   344.08     0.089893   344.24     0   
Max Read:  628251.97 MiB/sec (658769.93 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       628251.97  628251.97  628251.97       0.00  628251.97  628251.97  628251.97       0.00  344.23895         NA            NA     0   5280  11    1   1     2        1         0    0      1 42949672960  1048576 216268800.0 POSIX      0
Finished            : Sun Nov 10 18:23:19 2019
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 10 17:49:09 2019
Command line        : /vol7/io500_20190930/bin/ior -w -a=POSIX --posix.odirect -t 1m -C -b 40g -F -i 1 -C -Q 1 -g -G 27 -k -e -o /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_easy/ior_file_easy -O stoneWallingStatusFile=/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux cn1
TestID              : 0
StartTime           : Sun Nov 10 17:49:09 2019
Path                : /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_easy
FS                  : 824.1 TiB   Used FS: 25.8%   Inodes: 7452.5 Mi   Used Inodes: 4.5%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
tasks               : 5280
clients per node    : 11
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 40 GiB
aggregate filesize  : 206.25 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 18767 max: 40960 -- min data: 36.7 GiB mean data: 79.1 GiB time: 300.0s
write     623251     41943040   1024.00    0.088608   346.87     0.067202   347.00     0   
Max Write: 623250.72 MiB/sec (653525.75 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      623250.72  623250.72  623250.72       0.00  623250.72  623250.72  623250.72       0.00  347.00128     300.02    1425401.15     0   5280  11    1   1     2        1         0    0      1 42949672960  1048576 216268800.0 POSIX      0
Finished            : Sun Nov 10 17:54:56 2019
ior_hard_read
yhrun: job 59573 queued and waiting for resources
yhrun: job 59573 has been allocated resources
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 10 18:26:02 2019
Command line        : /vol7/io500_20190930/bin/ior -r -R -s 52800 -a MPIIO -U /vol7/io500_20190930/mpio -H -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_hard/IOR_file -O stoneWallingStatusFile=/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_hard/stonewall
Machine             : Linux cn1
TestID              : 0
StartTime           : Sun Nov 10 18:26:02 2019
Path                : /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_hard
FS                  : 824.1 TiB   Used FS: 51.6%   Inodes: 7452.5 Mi   Used Inodes: 11.3%

Options: 
api                 : MPIIO
apiVersion          : (3.1)
test filename       : /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 52800
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 5280
clients per node    : 11
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 11.92 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----

hints passed to MPI_File_open() {
	romio_ds_write = disable
	romio_cb_read = disable
}

hints returned from opened file {
	direct_read = false
	direct_write = false
	romio_lustre_co_ratio = 1
	romio_lustre_coll_threshold = 0
	romio_lustre_ds_in_coll = enable
	cb_buffer_size = 16777216
	romio_cb_read = disable
	romio_cb_write = automatic
	cb_nodes = 480
	romio_no_indep_rw = false
	romio_cb_pfr = disable
	romio_cb_fr_types = aar
	romio_cb_fr_alignment = 1
	romio_cb_ds_threshold = 0
	romio_cb_alltoall = automatic
	ind_rd_buffer_size = 4194304
	ind_wr_buffer_size = 524288
	romio_ds_read = automatic
	romio_ds_write = disable
	cb_config_list = *:1
	romio_filesystem_type = LUSTRE:
	romio_aggregator_list = 0 11 22 33 44 55 66 77 88 99 110 121 132 143 154 165 176 187 198 209 220 231 242 253 264 275 286 297 308 319 330 341 352 363 374 385 396 407 418 429 440 451 462 473 484 495 506 517 528 539 550 561 572 583 594 605 616 627 638 649 660 671 682 693 704 715 726 737 748 759 770 781 792 803 814 825 836 847 858 869 880 891 902 913 924 935 946 957 968 979 990 1001 1012 1023 1034 1045 1056 1067 1078 1089 1100 1111 1122 1133 1144 1155 1166 1177 1188 1199 1210 1221 1232 1243 1254 1265 1276 1287 1298 1309 1320 1331 1342 1353 1364 1375 1386 1397 1408 1419 1430 1441 1452 1463 1474 1485 1496 1507 1518 1529 1540 1551 1562 1573 1584 1595 1606 1617 1628 1639 1650 1661 1672 1683 1694 1705 1716 1727 1738 1749 1760 1771 1782 1793 1804 1815 1826 1837 1848 1859 1870 1881 1892 1903 1914 1925 1936 1947 1958 1969 1980 1991 2002 2013 2024 2035 2046 2057 2068 2079 2090 2101 2112 2123 2134 2145 2156 2167 2178 2189 2200 2211 2222 2233 2244 2255 2266 2277 2288 2299 2310 2321 2332 2343 2354 2365 2376 2387 2398 2409 2420 2431 2442 2453 2464 
	striping_unit = 8388608
	striping_factor = 572
	romio_lustre_start_iodevice = 0
}
read      165518     45.91      45.91      8.37       67.01      0.125955   75.51      0   
Max Read:  165518.28 MiB/sec (173558.50 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       165518.28  165518.28  165518.28       0.00 3692105.51 3692105.51 3692105.51       0.00   75.50813         NA            NA     0   5280  11    1   0     1        1         0    0  52800    47008    47008 12497977.0 MPIIO      0
Finished            : Sun Nov 10 18:27:23 2019
ior_hard_write
yhrun: job 59568 queued and waiting for resources
yhrun: job 59568 has been allocated resources
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov 10 18:00:31 2019
Command line        : /vol7/io500_20190930/bin/ior -w -s 52800 -a MPIIO -U /vol7/io500_20190930/mpio -H -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_hard/IOR_file -O stoneWallingStatusFile=/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux cn1
TestID              : 0
StartTime           : Sun Nov 10 18:00:31 2019
Path                : /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_hard
FS                  : 824.1 TiB   Used FS: 50.8%   Inodes: 7452.5 Mi   Used Inodes: 9.1%

Options: 
api                 : MPIIO
apiVersion          : (3.1)
test filename       : /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 52800
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 5280
clients per node    : 11
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 11.92 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----

hints passed to MPI_File_open() {
	romio_ds_write = disable
	romio_cb_read = disable
}

hints returned from opened file {
	direct_read = false
	direct_write = false
	romio_lustre_co_ratio = 1
	romio_lustre_coll_threshold = 0
	romio_lustre_ds_in_coll = enable
	cb_buffer_size = 16777216
	romio_cb_read = disable
	romio_cb_write = automatic
	cb_nodes = 480
	romio_no_indep_rw = false
	romio_cb_pfr = disable
	romio_cb_fr_types = aar
	romio_cb_fr_alignment = 1
	romio_cb_ds_threshold = 0
	romio_cb_alltoall = automatic
	ind_rd_buffer_size = 4194304
	ind_wr_buffer_size = 524288
	romio_ds_read = automatic
	romio_ds_write = disable
	cb_config_list = *:1
	romio_filesystem_type = LUSTRE:
	romio_aggregator_list = 0 11 22 33 44 55 66 77 88 99 110 121 132 143 154 165 176 187 198 209 220 231 242 253 264 275 286 297 308 319 330 341 352 363 374 385 396 407 418 429 440 451 462 473 484 495 506 517 528 539 550 561 572 583 594 605 616 627 638 649 660 671 682 693 704 715 726 737 748 759 770 781 792 803 814 825 836 847 858 869 880 891 902 913 924 935 946 957 968 979 990 1001 1012 1023 1034 1045 1056 1067 1078 1089 1100 1111 1122 1133 1144 1155 1166 1177 1188 1199 1210 1221 1232 1243 1254 1265 1276 1287 1298 1309 1320 1331 1342 1353 1364 1375 1386 1397 1408 1419 1430 1441 1452 1463 1474 1485 1496 1507 1518 1529 1540 1551 1562 1573 1584 1595 1606 1617 1628 1639 1650 1661 1672 1683 1694 1705 1716 1727 1738 1749 1760 1771 1782 1793 1804 1815 1826 1837 1848 1859 1870 1881 1892 1903 1914 1925 1936 1947 1958 1969 1980 1991 2002 2013 2024 2035 2046 2057 2068 2079 2090 2101 2112 2123 2134 2145 2156 2167 2178 2189 2200 2211 2222 2233 2244 2255 2266 2277 2288 2299 2310 2321 2332 2343 2354 2365 2376 2387 2398 2409 2420 2431 2442 2453 2464 
	striping_unit = 8388608
	striping_factor = 572
	romio_lustre_start_iodevice = 0
}
stonewalling pairs accessed min: 24402 max: 52800 -- min data: 1.1 GiB mean data: 2.0 GiB time: 300.1s
write     32639      45.91      45.91      4.02       378.88     0.035581   382.91     0   
Max Write: 32639.05 MiB/sec (34224.53 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       32639.05   32639.05   32639.05       0.00  728057.53  728057.53  728057.53       0.00  382.91479     300.11      36410.16     0   5280  11    1   0     1        1         0    0  52800    47008    47008 12497977.0 MPIIO      0
Finished            : Sun Nov 10 18:06:59 2019
mdtest_easy_delete
yhrun: job 59575 queued and waiting for resources
yhrun: job 59575 has been allocated resources
-- started at 11/10/2019 18:28:50 --

mdtest-3.3.0+dev was launched with 5280 total task(s) on 480 node(s)
Command line used: /vol7/io500_20190930/bin/mdtest '-r' '-F' '-P' '-d' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_easy' '-n' '67000' '-u' '-L' '-x' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_easy-stonewall' '-N' '1'
Path: /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07
FS: 824.1 TiB   Used FS: 51.6%   Inodes: 7452.5 Mi   Used Inodes: 11.3%

Nodemap: 111111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2162 Shifting ranks by 11 for each phase.
5280 tasks, 353760000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     961619.852     961562.734     961589.468          4.908
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.396          3.396          3.396          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        367.901        367.879        367.891          0.002
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.294          0.294          0.294          0.000
-- finished at 11/10/2019 18:34:58 --

mdtest_easy_stat
yhrun: job 59572 queued and waiting for resources
yhrun: job 59572 has been allocated resources
-- started at 11/10/2019 18:23:27 --

mdtest-3.3.0+dev was launched with 5280 total task(s) on 480 node(s)
Command line used: /vol7/io500_20190930/bin/mdtest '-T' '-F' '-P' '-d' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_easy' '-n' '67000' '-u' '-L' '-x' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_easy-stonewall' '-N' '1'
Path: /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07
FS: 824.1 TiB   Used FS: 51.6%   Inodes: 7452.5 Mi   Used Inodes: 11.3%

Nodemap: 111111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2162 Shifting ranks by 11 for each phase.
5280 tasks, 353760000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    2416395.834    2416236.342    2416314.580         14.413
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        146.410        146.400        146.405          0.001
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/10/2019 18:25:54 --

mdtest_easy_write
yhrun: job 59567 queued and waiting for resources
yhrun: job 59567 has been allocated resources
-- started at 11/10/2019 17:55:01 --

mdtest-3.3.0+dev was launched with 5280 total task(s) on 480 node(s)
Command line used: /vol7/io500_20190930/bin/mdtest '-C' '-F' '-P' '-d' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_easy' '-n' '67000' '-u' '-L' '-x' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07
FS: 824.1 TiB   Used FS: 50.8%   Inodes: 7452.5 Mi   Used Inodes: 4.5%

Nodemap: 111111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2162 Shifting ranks by 11 for each phase.
5280 tasks, 353760000 files
Continue stonewall hit min: 60000 max: 67000 avg: 66839.9 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :    1099176.779    1099064.102    1099100.851          7.029
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA    1175476.381             NA
   Tree creation             :          6.313          6.313          6.313          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        321.874        321.841        321.863          0.002
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.231             NA
   Tree creation             :          0.158          0.158          0.158          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/10/2019 18:00:23 --

mdtest_hard_delete
yhrun: job 59577 queued and waiting for resources
yhrun: job 59577 has been allocated resources
-- started at 11/10/2019 18:38:54 --

mdtest-3.3.0+dev was launched with 5280 total task(s) on 480 node(s)
Command line used: /vol7/io500_20190930/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_hard' '-n' '38888' '-x' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07
FS: 824.1 TiB   Used FS: 51.6%   Inodes: 7452.5 Mi   Used Inodes: 9.0%

Nodemap: 111111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2162 Shifting ranks by 11 for each phase.
5280 tasks, 205328640 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     203500.853     203478.597     203490.595          5.649
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.166          3.166          3.166          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        858.979        858.885        858.928          0.024
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.316          0.316          0.316          0.000
-- finished at 11/10/2019 18:53:14 --

mdtest_hard_read
yhrun: job 59576 queued and waiting for resources
yhrun: job 59576 has been allocated resources
-- started at 11/10/2019 18:35:04 --

mdtest-3.3.0+dev was launched with 5280 total task(s) on 480 node(s)
Command line used: /vol7/io500_20190930/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_hard' '-n' '38888' '-x' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07
FS: 824.1 TiB   Used FS: 51.6%   Inodes: 7452.5 Mi   Used Inodes: 9.0%

Nodemap: 111111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2162 Shifting ranks by 11 for each phase.
5280 tasks, 205328640 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     779212.258     779045.101     779147.431         43.396
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        224.357        224.308        224.327          0.012
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/10/2019 18:38:48 --

mdtest_hard_stat
yhrun: job 59574 queued and waiting for resources
yhrun: job 59574 has been allocated resources
-- started at 11/10/2019 18:27:31 --

mdtest-3.3.0+dev was launched with 5280 total task(s) on 480 node(s)
Command line used: /vol7/io500_20190930/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_hard' '-n' '38888' '-x' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07
FS: 824.1 TiB   Used FS: 51.6%   Inodes: 7452.5 Mi   Used Inodes: 11.3%

Nodemap: 111111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2162 Shifting ranks by 11 for each phase.
5280 tasks, 205328640 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    2419147.595    2419021.208    2419092.870         13.387
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         72.254         72.250         72.252          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/10/2019 18:28:43 --

mdtest_hard_write
yhrun: job 59569 queued and waiting for resources
yhrun: job 59569 has been allocated resources
-- started at 11/10/2019 18:07:07 --

mdtest-3.3.0+dev was launched with 5280 total task(s) on 480 node(s)
Command line used: /vol7/io500_20190930/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_hard' '-n' '38888' '-x' '/vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /vol7/io500_20190930/datafiles/io500.2019.11.10-17.49.07
FS: 824.1 TiB   Used FS: 51.5%   Inodes: 7452.5 Mi   Used Inodes: 9.1%

Nodemap: 111111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2162 Shifting ranks by 11 for each phase.
5280 tasks, 205328640 files
Continue stonewall hit min: 31341 max: 33103 avg: 32305.2 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     554065.562     554040.549     554054.424          2.721
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     569312.977             NA
   Tree creation             :          2.348          2.348          2.348          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        315.471        315.457        315.463          0.002
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.610             NA
   Tree creation             :          0.426          0.426          0.426          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/10/2019 18:12:23 --

result_summary
[RESULT] BW   phase 1            ior_easy_write              608.644 GB/s : time 347.00 seconds
[RESULT] IOPS phase 1         mdtest_easy_write             1099.180 kiops : time 321.87 seconds
[RESULT] BW   phase 2            ior_hard_write               31.874 GB/s : time 382.91 seconds
[RESULT] IOPS phase 2         mdtest_hard_write              554.066 kiops : time 315.47 seconds
[RESULT] IOPS phase 3                      find             1603.060 kiops : time 303.33 seconds
[RESULT] BW   phase 3             ior_easy_read              613.527 GB/s : time 344.24 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             2416.400 kiops : time 146.41 seconds
[RESULT] BW   phase 4             ior_hard_read              161.639 GB/s : time  75.51 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat             2419.150 kiops : time  72.25 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              961.620 kiops : time 367.90 seconds
[RESULT] IOPS phase 7          mdtest_hard_read              779.212 kiops : time 224.36 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete              203.501 kiops : time 865.04 seconds
[SCORE] Bandwidth 209.433 GB/s : IOPS 982.776 kiops : TOTAL 453.68