nrs

Institution Janelia Research Campus, HHMI
Client Procs Per Node
Client Operating System Scientific-Linux
Client Operating System Version 7.6
Client Kernel Version 3.10.0-957.10.1.el7.x86_64

DATA SERVER

Storage Type HDD
Volatile Memory 128GB
Storage Interface SATA
Network 40GbE
Software Version 2.14.0.1
OS Version 7.6

INFORMATION

Client Nodes 10
Client Total Procs 760
Metadata Storage Devices 13
Data Nodes 20
Data Storage Devices 520

METADATA

Easy Write 9.03 kIOP/s
Easy Stat 171.20 kIOP/s
Easy Delete 10.77 kIOP/s
Hard Write 0.21 kIOP/s
Hard Read 9.81 kIOP/s
Hard Stat 9.07 kIOP/s
Hard Delete 0.18 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="-np 760 -ppn 76 -f ./site-configs/hhmi/hostfile-10gpu --bootstrap ssh"
}

function setup_ior_easy {
  io500_ior_easy_params="-u -t 1024k -b 9920000m -F --posix.odirect"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L --posix.odirect" # unique dir per thread, files only at leaves
}

function setup_ior_hard {
  io500_ior_hard_api="POSIX"
  io500_ior_hard_api_specific_options="--posix.odirect"
}

function setup_mdt_hard {
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options="--posix.odirect"
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind
  io500_find_cmd_args="-N"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
  # TODO: Please add your information using the info-creator!
  # EXAMPLE:
  echo io500_info_submitter='"Janelia Scientific Computing Systems"'
  echo io500_info_institution='"Janelia Research Campus, Howard Hughes Medical Institute"'
  echo io500_info_system='"nrs"'
  echo io500_info_system_classification='"production"'
  echo io500_info_storage_install_date='"9/2016"'
  echo io500_info_storage_vendor='"Qumulo"'
  echo io500_info_filesystem_name='"Qumulo FS"'
  echo io500_info_filesystem_type='"Scale-out NAS"'
  echo io500_info_filesystem_version='"2.14.0.1"'
  echo io500_info_client_nodes='"10"'
  echo io500_info_client_procs_per_node='"76"'
  echo io500_info_client_operating_system='"Scientific-Linux"'
  echo io500_info_client_operating_system_version='"7.6"'
  echo io500_info_client_kernel_version='"3.10.0-957.10.1.el7.x86_64"'
  echo io500_info_md_storage_devices='"13"'
  echo io500_info_md_storage_type='"SSD"'
  echo io500_info_md_storage_interface='"SATA"'
  echo io500_info_ds_nodes='"20"'
  echo io500_info_ds_storage_devices='"26"'
  echo io500_info_ds_storage_type='"HDD"'
  echo io500_info_ds_storage_interface='"SATA"'
  echo io500_info_ds_network='"Ethernet"'
  echo io500_info_ds_overhead='"12%"'
  echo io500_info_whatever='"Storage nodes have both SSD and HDD. Data is written to SSD before being drained out to HDD, and is rehydrated from HDD automatically as needed. All metadata is stored permanently on SSD"'
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Nov  9 21:04:42 2019
Command line        : /io500/io500-sc19/bin/ior -r -R -u -t 1024k -b 9920000m -F --posix.odirect -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/ior_file_easy -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/stonewall
Machine             : Linux e11u05.int.janelia.org
TestID              : 0
StartTime           : Sat Nov  9 21:04:42 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/0
FS                  : 2984.3 TiB   Used FS: 69.1%   Inodes: 763980.0 Mi   Used Inodes: 69.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 760
clients per node    : 76
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 7189.94 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 7905424179200000.
WARNING: Stat() of aggregate file size      = 3048210432.
WARNING: Using actual aggregate bytes moved = 2316639928320.
read      13693      13696      0.000016    10158080000 1024.00    0.037825   161.32     0.013310   161.34     0   
Max Read:  13693.45 MiB/sec (14358.62 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        13693.45   13693.45   13693.45       0.00   13693.45   13693.45   13693.45       0.00  161.34143         NA            NA     0    760  76    1   2     1        1         0    0      1 10401873920000  1048576 2209320.0 POSIX      0
Finished            : Sat Nov  9 21:07:24 2019
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Nov  9 20:35:41 2019
Command line        : /io500/io500-sc19/bin/ior -w -u -t 1024k -b 9920000m -F --posix.odirect -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/ior_file_easy -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux e11u05.int.janelia.org
TestID              : 0
StartTime           : Sat Nov  9 20:35:41 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/0
FS                  : 2984.3 TiB   Used FS: 69.0%   Inodes: 763980.0 Mi   Used Inodes: 69.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 760
clients per node    : 76
repetitions         : 1
xfersize            : 1 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 7189.94 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 1409 max: 2907 -- min data: 1.4 GiB mean data: 2.3 GiB time: 300.3s
WARNING: Expected aggregate file size       = 7905424179200000.
WARNING: Stat() of aggregate file size      = 3048210432.
WARNING: Using actual aggregate bytes moved = 2316639928320.
WARNING: maybe caused by deadlineForStonewalling
write     5575       5602       0.000038    10158080000 1024.00    1.91       394.37     0.009011   396.28     0   
Max Write: 5575.08 MiB/sec (5845.90 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        5575.08    5575.08    5575.08       0.00    5575.08    5575.08    5575.08       0.00  396.28478     300.26       5872.84     0    760  76    1   2     1        1         0    0      1 10401873920000  1048576 2209320.0 POSIX      0
Finished            : Sat Nov  9 20:42:22 2019
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Nov  9 21:08:04 2019
Command line        : /io500/io500-sc19/bin/ior -r -R -s 1900000 -a POSIX --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/IOR_file -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/stonewall
Machine             : Linux e11u05.int.janelia.org
TestID              : 0
StartTime           : Sat Nov  9 21:08:04 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard
FS                  : 2984.3 TiB   Used FS: 69.1%   Inodes: 763980.0 Mi   Used Inodes: 69.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 760
clients per node    : 76
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 61.74 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 67879552000000.
WARNING: Stat() of aggregate file size      = 56518658560.
WARNING: Using actual aggregate bytes moved = 56518658560.
read      5848       130941     9.18        45.91      45.91      0.027607   9.18       0.008709   9.22       0   
Max Read:  5847.57 MiB/sec (6131.63 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         5847.57    5847.57    5847.57       0.00  130437.91  130437.91  130437.91       0.00    9.21757         NA            NA     0    760  76    1   0     1        1         0    0 1900000    47008    47008   53900.4 POSIX      0
Finished            : Sat Nov  9 21:08:13 2019
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sat Nov  9 20:51:38 2019
Command line        : /io500/io500-sc19/bin/ior -w -s 1900000 -a POSIX --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/IOR_file -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux e11u05.int.janelia.org
TestID              : 0
StartTime           : Sat Nov  9 20:51:38 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard
FS                  : 2984.3 TiB   Used FS: 69.1%   Inodes: 763980.0 Mi   Used Inodes: 69.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 760
clients per node    : 76
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 61.74 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 791 max: 1582 -- min data: 0.0 GiB mean data: 0.1 GiB time: 300.2s
WARNING: Expected aggregate file size       = 67879552000000.
WARNING: Stat() of aggregate file size      = 56518658560.
WARNING: Using actual aggregate bytes moved = 56518658560.
WARNING: maybe caused by deadlineForStonewalling
write     168.53     3760       300.26      45.91      45.91      0.050615   319.78     0.001022   319.83     0   
Max Write: 168.53 MiB/sec (176.71 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write         168.53     168.53     168.53       0.00    3759.22    3759.22    3759.22       0.00  319.83192     300.17        170.48     0    760  76    1   0     1        1         0    0 1900000    47008    47008   53900.4 POSIX      0
Finished            : Sat Nov  9 20:56:58 2019
mdtest_easy_delete
-- started at 11/09/2019 21:08:33 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-r' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy' '-n' '900000' '-u' '-L' '--posix.odirect' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB   Used FS: 69.1%   Inodes: 763980.0 Mi   Used Inodes: 69.1%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 76 for each phase.
760 tasks, 684000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      10768.460      10768.452      10768.455          0.002
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.262          0.262          0.262          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        452.325        452.325        452.325          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.810          3.810          3.810          0.000
-- finished at 11/09/2019 21:16:12 --

mdtest_easy_stat
-- started at 11/09/2019 21:07:30 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-T' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy' '-n' '900000' '-u' '-L' '--posix.odirect' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB   Used FS: 69.1%   Inodes: 763980.0 Mi   Used Inodes: 69.1%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 76 for each phase.
760 tasks, 684000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     171203.263     171201.951     171202.495          0.307
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         28.451         28.451         28.451          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/09/2019 21:07:58 --

mdtest_easy_write
-- started at 11/09/2019 20:42:28 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-C' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy' '-n' '900000' '-u' '-L' '--posix.odirect' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB   Used FS: 69.1%   Inodes: 763980.0 Mi   Used Inodes: 69.1%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 76 for each phase.
760 tasks, 684000000 files
Continue stonewall hit min: 390 max: 6409 avg: 2852.8 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       9028.042       9028.012       9028.038          0.002
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA       7292.039             NA
   Tree creation             :          0.270          0.270          0.270          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        539.525        539.523        539.524          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        297.330             NA
   Tree creation             :          3.708          3.708          3.708          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/09/2019 20:51:31 --

mdtest_hard_delete
-- started at 11/09/2019 21:16:30 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard-stonewall' '-a' 'POSIX' '--posix.odirect' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB   Used FS: 69.1%   Inodes: 763980.0 Mi   Used Inodes: 69.1%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        178.684        178.678        178.683          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          6.495          6.495          6.495          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        361.545        361.533        361.533          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.154          0.154          0.154          0.000
-- finished at 11/09/2019 21:22:32 --

mdtest_hard_read
-- started at 11/09/2019 21:16:18 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard-stonewall' '-a' 'POSIX' '--posix.odirect' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB   Used FS: 69.1%   Inodes: 763980.0 Mi   Used Inodes: 69.1%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :       9813.238       9813.048       9813.159          0.049
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          6.583          6.583          6.583          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/09/2019 21:16:25 --

mdtest_hard_stat
-- started at 11/09/2019 21:08:19 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard-stonewall' '-a' 'POSIX' '--posix.odirect' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB   Used FS: 69.1%   Inodes: 763980.0 Mi   Used Inodes: 69.1%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :       9066.865       9059.275       9066.720          0.384
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          7.131          7.125          7.125          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/09/2019 21:08:26 --

mdtest_hard_write
-- started at 11/09/2019 20:57:04 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35/mdt_hard-stonewall' '-a' 'POSIX' '--posix.odirect' '-N' '1' '-W' '300'
Path: /io500/io500-sc19/datafiles/io500.2019.11.09-20.35.35
FS: 2984.3 TiB   Used FS: 69.1%   Inodes: 763980.0 Mi   Used Inodes: 69.1%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files
Continue stonewall hit min: 81 max: 85 avg: 82.9 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        206.861        206.860        206.861          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        207.370             NA
   Tree creation             :        373.824        373.824        373.824          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        312.288        312.287        312.288          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        303.718             NA
   Tree creation             :          0.003          0.003          0.003          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/09/2019 21:02:16 --

result_summary
[RESULT] BW   phase 1            ior_easy_write                5.444 GB/s : time 394.37 seconds
[RESULT] IOPS phase 1         mdtest_easy_write                9.028 kiops : time 539.53 seconds
[RESULT] BW   phase 2            ior_hard_write                0.165 GB/s : time 319.78 seconds
[RESULT] IOPS phase 2         mdtest_hard_write                0.207 kiops : time 312.29 seconds
[RESULT] IOPS phase 3                      find               35.320 kiops : time 139.72 seconds
[RESULT] BW   phase 3             ior_easy_read               13.372 GB/s : time 161.32 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              171.203 kiops : time  28.45 seconds
[RESULT] BW   phase 4             ior_hard_read                5.711 GB/s : time   9.18 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat                9.067 kiops : time   7.13 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               10.768 kiops : time 452.33 seconds
[RESULT] IOPS phase 7          mdtest_hard_read                9.813 kiops : time   6.58 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                0.179 kiops : time 367.75 seconds
[SCORE] Bandwidth 2.87612 GB/s : IOPS 6.10646 kiops : TOTAL 4.19081