DGX-2H SuperPOD

Institution NVIDIA
Client Procs Per Node
Client Operating System NVIDIA DGX-OS
Client Operating System Version 4.02
Client Kernel Version 4.15.0-50-generic

DATA SERVER

Storage Type NVMe
Volatile Memory 1.92TB
Storage Interface Mellanox EDR InfiniBand
Network Mellanox EDR InfiniBand
Software Version Exascaler
OS Version Exa5.0

INFORMATION

Client Nodes 10
Client Total Procs 400
Metadata Nodes 10
Metadata Storage Devices 2
Data Nodes 10
Data Storage Devices 21

METADATA

Easy Write 799.58 kIOP/s
Easy Stat 2,000.81 kIOP/s
Easy Delete 485.87 kIOP/s
Hard Write 446.27 kIOP/s
Hard Read 472.52 kIOP/s
Hard Stat 881.03 kIOP/s
Hard Delete 584.58 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=/lustre/0/titi
  io500_result_dir=$PWD/results/$timestamp.10x40ppn      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
  mkdir -p $io500_workdir/ior_hard $io500_workdir/ior_easy
  lfs setdirstripe -c 40 -i 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39 $io500_workdir/mdt_easy
  lfs setdirstripe -c 40 -i 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39 $io500_workdir/mdt_hard
  lfs setdirstripe -c 40 -D $io500_workdir/mdt_easy
  lfs setdirstripe -c 40 -D $io500_workdir/mdt_hard

  lfs setstripe -L mdt -E 1M $io500_workdir/mdt_easy
  #lfs setstripe -L mdt -E 1M $io500_workdir/mdt_hard
  lfs setstripe -C 400 -S 16M $io500_workdir/ior_hard
  
  timestamp_file=$io500_workdir/timestampfile
  touch $timestamp_file
  
  sync
  export TIMESTAMP=$timestamp

  io500_workdir=$PWD/datafiles
}


function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="--map-by ppr:20:socket -np 400 --oversubscribe -hostfile host.10 --allow-run-as-root --mca oob_tcp_if_include ib1 --mca btl self,vader,openib --mca btl_tcp_if_exclude docker0 --mca btl_openib_warn_default_gid_prefix 0 --mca btl_openib_if_exclude mlx5_0,mlx5_5,mlx5_6 --mca plm_base_verbose 0 enroot-unshare ./run2_mount.sh"
}

function setup_ior_easy {
  # io500_ior_easy_size is the amount of data written per rank in MiB units,
  # but it can be any number as long as it is somehow used to scale the IOR
  # runtime as part of io500_ior_easy_params
  io500_ior_easy_size=225008
  # 2M writes, 2 GB per proc, file per proc
  io500_ior_easy_params="-t 16m --posix.odirect -a POSIX"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
  io500_mdtest_easy_files_per_proc=700000
}

function setup_ior_hard {
  io500_ior_hard_writes_per_proc=210000
  io500_ior_hard_other_options="--posix.odirect" #e.g., -E to keep precreated files using lfs setstripe, or -a MPIIO
  io500_ior_hard_api="POSIX"
  io500_ior_hard_api_specific_options=""
}

function setup_mdt_hard {
  io500_mdtest_hard_files_per_proc=375000
  io500_mdtest_hard_other_options=""
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind 
  io500_find_cmd_args="-q 200000"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  io500_mdreal_params="-P=5000 -I=1000"
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Add key/value pairs defining your system
# Feel free to add extra ones if you'd like
function extra_description {
  io500_info_system_name=SuperPOD
  io500_info_institute_name=NVIDIA
  io500_info_filesystem=EXAScaler
  io500_info_filesystem_vendor=DDN
  io500_info_filesystem_version=5
  io500_info_metadata_storage_type=NVMe
  io500_info_num_client_nodes=10
  io500_info_num_data_server_nodes=10
  io500_info_num_data_storage_devices=230
  io500_info_num_metadata_server_nodes=10
  io500_info_num_metadata_storage_devices=10
  io500_info_procs_per_node=40
  io500_info_storage_age_in_months=1
  io500_info_storage_install_date=11/05
  io500_info_storage_interface=NVMe
  io500_info_storage_network=infiniband
  io500_info_whatever='DGX-2 Servers'
}

main

ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Nov 11 20:14:10 2019
Command line        : /home/lcapps/io-500-dev.l2+mount/bin/ior -r -R -t 16m --posix.odirect -a POSIX -b 225008m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /home/lcapps/io-500-dev.l2+mount/datafiles/ior_easy/ior_file_easy -O stoneWallingStatusFile=/home/lcapps/io-500-dev.l2+mount/datafiles/ior_easy/stonewall
Machine             : Linux circe-n010
TestID              : 0
StartTime           : Mon Nov 11 20:14:10 2019
Path                : /home/lcapps/io-500-dev.l2+mount/datafiles/ior_easy
FS                  : 308.8 TiB   Used FS: 29.1%   Inodes: 1151.9 Mi   Used Inodes: 34.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /home/lcapps/io-500-dev.l2+mount/datafiles/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 16 MiB
blocksize           : 219.73 GiB
aggregate filesize  : 85.83 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
read      293035     230408192  16384      0.082855   306.84     0.077480   307.14     0   
Max Read:  293035.37 MiB/sec (307269.85 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       293035.37  293035.37  293035.37       0.00   18314.71   18314.71   18314.71       0.00  307.14108         NA            NA     0    400  40    1   1     1        1         0    0      1 235937988608 16777216 90003200.0 POSIX      0
Finished            : Mon Nov 11 20:19:17 2019
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Nov 11 19:43:33 2019
Command line        : /home/lcapps/io-500-dev.l2+mount/bin/ior -w -t 16m --posix.odirect -a POSIX -b 225008m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /home/lcapps/io-500-dev.l2+mount/datafiles/ior_easy/ior_file_easy -O stoneWallingStatusFile=/home/lcapps/io-500-dev.l2+mount/datafiles/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux circe-n010
TestID              : 0
StartTime           : Mon Nov 11 19:43:33 2019
Path                : /home/lcapps/io-500-dev.l2+mount/datafiles/ior_easy
FS                  : 308.8 TiB   Used FS: 0.0%   Inodes: 1151.9 Mi   Used Inodes: 0.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /home/lcapps/io-500-dev.l2+mount/datafiles/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 16 MiB
blocksize           : 219.73 GiB
aggregate filesize  : 85.83 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 12906 max: 14063 -- min data: 201.7 GiB mean data: 210.0 GiB time: 300.1s
write     279717     230408192  16384      0.091398   321.46     0.145003   321.77     0   
Max Write: 279716.88 MiB/sec (293304.41 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      279716.88  279716.88  279716.88       0.00   17482.30   17482.30   17482.30       0.00  321.76535     300.12     286674.43     0    400  40    1   1     1        1         0    0      1 235937988608 16777216 90003200.0 POSIX      0
Finished            : Mon Nov 11 19:48:55 2019
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Nov 11 20:21:36 2019
Command line        : /home/lcapps/io-500-dev.l2+mount/bin/ior -r -R -s 210000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /home/lcapps/io-500-dev.l2+mount/datafiles/ior_hard/IOR_file -O stoneWallingStatusFile=/home/lcapps/io-500-dev.l2+mount/datafiles/ior_hard/stonewall
Machine             : Linux circe-n010
TestID              : 0
StartTime           : Mon Nov 11 20:21:36 2019
Path                : /home/lcapps/io-500-dev.l2+mount/datafiles/ior_hard
FS                  : 308.8 TiB   Used FS: 29.1%   Inodes: 1151.9 Mi   Used Inodes: 34.1%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /home/lcapps/io-500-dev.l2+mount/datafiles/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 210000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 3.59 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
read      67165      45.91      45.91      0.080681   55.78      0.096383   56.07      0   
Max Read:  67165.34 MiB/sec (70427.97 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        67165.34   67165.34   67165.34       0.00 1498212.33 1498212.33 1498212.33       0.00   56.06682         NA            NA     0    400  40    1   0     1        1         0    0 210000    47008    47008 3765747.0 POSIX      0
Finished            : Mon Nov 11 20:22:32 2019
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Mon Nov 11 19:54:29 2019
Command line        : /home/lcapps/io-500-dev.l2+mount/bin/ior -w -s 210000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /home/lcapps/io-500-dev.l2+mount/datafiles/ior_hard/IOR_file -O stoneWallingStatusFile=/home/lcapps/io-500-dev.l2+mount/datafiles/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux circe-n010
TestID              : 0
StartTime           : Mon Nov 11 19:54:29 2019
Path                : /home/lcapps/io-500-dev.l2+mount/datafiles/ior_hard
FS                  : 308.8 TiB   Used FS: 27.8%   Inodes: 1151.9 Mi   Used Inodes: 21.7%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /home/lcapps/io-500-dev.l2+mount/datafiles/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 210000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 400
clients per node    : 40
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 3.59 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 185912 max: 210000 -- min data: 8.1 GiB mean data: 8.9 GiB time: 300.1s
write     11427      45.91      45.91      0.070627   329.28     0.144621   329.55     0   
Max Write: 11426.79 MiB/sec (11981.85 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       11426.79   11426.79   11426.79       0.00  254889.66  254889.66  254889.66       0.00  329.55436     300.12      12169.27     0    400  40    1   0     1        1         0    0 210000    47008    47008 3765747.0 POSIX      0
Finished            : Mon Nov 11 19:59:59 2019
mdtest_easy_delete
-- started at 11/11/2019 20:25:29 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /home/lcapps/io-500-dev.l2+mount/bin/mdtest '-r' '-F' '-P' '-d' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_easy' '-n' '700000' '-u' '-L' '-x' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_easy-stonewall' '-N' '1'
Path: /home/lcapps/io-500-dev.l2+mount/datafiles
FS: 308.8 TiB   Used FS: 29.1%   Inodes: 1151.9 Mi   Used Inodes: 34.1%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 40 for each phase.
400 tasks, 280000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     485873.776     485873.134     485873.382          0.148
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          3.144          3.144          3.144          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        539.117        539.116        539.117          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.318          0.318          0.318          0.000
-- finished at 11/11/2019 20:34:30 --

mdtest_easy_stat
-- started at 11/11/2019 20:19:20 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /home/lcapps/io-500-dev.l2+mount/bin/mdtest '-T' '-F' '-P' '-d' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_easy' '-n' '700000' '-u' '-L' '-x' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_easy-stonewall' '-N' '1'
Path: /home/lcapps/io-500-dev.l2+mount/datafiles
FS: 308.8 TiB   Used FS: 29.1%   Inodes: 1151.9 Mi   Used Inodes: 34.1%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 40 for each phase.
400 tasks, 280000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    2000813.454    1998290.245    1999570.884        528.832
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        131.083        130.918        130.999          0.035
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/11/2019 20:21:31 --

mdtest_easy_write
-- started at 11/11/2019 19:48:58 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /home/lcapps/io-500-dev.l2+mount/bin/mdtest '-C' '-F' '-P' '-d' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_easy' '-n' '700000' '-u' '-L' '-x' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /home/lcapps/io-500-dev.l2+mount/datafiles
FS: 308.8 TiB   Used FS: 27.8%   Inodes: 1151.9 Mi   Used Inodes: 0.0%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 40 for each phase.
400 tasks, 280000000 files
Continue stonewall hit min: 603719 max: 654856 avg: 628138.1 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     799577.035     799576.384     799576.724          0.114
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     839652.300             NA
   Tree creation             :          2.161          2.161          2.161          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        327.601        327.601        327.601          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.237             NA
   Tree creation             :          0.463          0.463          0.463          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/11/2019 19:54:26 --

mdtest_hard_delete
-- started at 11/11/2019 20:39:54 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /home/lcapps/io-500-dev.l2+mount/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_hard' '-n' '375000' '-x' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /home/lcapps/io-500-dev.l2+mount/datafiles
FS: 308.8 TiB   Used FS: 29.1%   Inodes: 1151.9 Mi   Used Inodes: 12.4%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 40 for each phase.
400 tasks, 150000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     584584.433     584583.424     584583.997          0.250
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.354          0.354          0.354          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        256.395        256.395        256.395          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.823          2.823          2.823          0.000
-- finished at 11/11/2019 20:44:13 --

mdtest_hard_read
-- started at 11/11/2019 20:34:33 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /home/lcapps/io-500-dev.l2+mount/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_hard' '-n' '375000' '-x' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /home/lcapps/io-500-dev.l2+mount/datafiles
FS: 308.8 TiB   Used FS: 29.1%   Inodes: 1151.9 Mi   Used Inodes: 12.4%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 40 for each phase.
400 tasks, 150000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :     472519.670     472349.901     472454.691         29.867
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        317.316        317.202        317.246          0.020
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/11/2019 20:39:51 --

mdtest_hard_stat
-- started at 11/11/2019 20:22:35 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /home/lcapps/io-500-dev.l2+mount/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_hard' '-n' '375000' '-x' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /home/lcapps/io-500-dev.l2+mount/datafiles
FS: 308.8 TiB   Used FS: 29.1%   Inodes: 1151.9 Mi   Used Inodes: 34.1%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 40 for each phase.
400 tasks, 150000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     881028.362     880258.625     880631.871        168.725
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        170.273        170.124        170.201          0.033
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/11/2019 20:25:26 --

mdtest_hard_write
-- started at 11/11/2019 20:00:02 --

mdtest-3.3.0+dev was launched with 400 total task(s) on 10 node(s)
Command line used: /home/lcapps/io-500-dev.l2+mount/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_hard' '-n' '375000' '-x' '/home/lcapps/io-500-dev.l2+mount/datafiles/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /home/lcapps/io-500-dev.l2+mount/datafiles
FS: 308.8 TiB   Used FS: 29.0%   Inodes: 1151.9 Mi   Used Inodes: 21.7%

Nodemap: 1111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 40 for each phase.
400 tasks, 150000000 files
Continue stonewall hit min: 320512 max: 374711 avg: 348198.4 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     446267.824     446122.626     446201.835         28.091
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     466898.149             NA
   Tree creation             :          0.679          0.679          0.679          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        335.971        335.862        335.912          0.021
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        298.308             NA
   Tree creation             :          1.473          1.473          1.473          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/11/2019 20:05:40 --

result_summary
[RESULT] BW   phase 1            ior_easy_write              273.161 GB/s : time 321.77 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              799.577 kiops : time 327.60 seconds
[RESULT] BW   phase 2            ior_hard_write               11.159 GB/s : time 329.55 seconds
[RESULT] IOPS phase 2         mdtest_hard_write              446.268 kiops : time 335.97 seconds
[RESULT] IOPS phase 3                      find              815.970 kiops : time 506.31 seconds
[RESULT] BW   phase 3             ior_easy_read              286.167 GB/s : time 307.14 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             2000.810 kiops : time 131.08 seconds
[RESULT] BW   phase 4             ior_hard_read               65.591 GB/s : time  56.07 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat              881.028 kiops : time 170.27 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              485.874 kiops : time 539.12 seconds
[RESULT] IOPS phase 7          mdtest_hard_read              472.520 kiops : time 317.32 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete              584.584 kiops : time 262.78 seconds
[SCORE] Bandwidth 86.9718 GB/s : IOPS 715.755 kiops : TOTAL 249.501