EXA5-GCP-PD-STD

Institution Google Cloud
Client Procs Per Node
Client Operating System CentOS 7.7
Client Operating System Version
Client Kernel Version

DATA SERVER

Storage Type Persistent Disk Standard
Volatile Memory 60GB
Storage Interface
Network
Software Version EXA5 - Lustre 2.12.3
OS Version CentOS 7.7

INFORMATION

Client Nodes 200
Client Total Procs 1,600
Metadata Nodes 5
Metadata Storage Devices 1
Data Nodes 100
Data Storage Devices 1

METADATA

Easy Write 167.63 kIOP/s
Easy Stat 583.86 kIOP/s
Easy Delete 91.02 kIOP/s
Hard Write 105.95 kIOP/s
Hard Read 77.63 kIOP/s
Hard Stat 197.42 kIOP/s
Hard Delete 51.28 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir

  mkdir -p $io500_workdir/ior_hard
  lfs setstripe -c 100 -S 32M $io500_workdir/ior_hard
  
  lfs mkdir -i 0,1,2,3,4 -c 5 $io500_workdir/mdt_easy
  lfs setdirstripe -D -c 5 $io500_workdir/mdt_easy
  
  
  lfs mkdir -i 0,1,2,3,4 -c 5 $io500_workdir/mdt_hard
  lfs setdirstripe -D -c 5 $io500_workdir/mdt_hard
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="srun"
  io500_mpiargs=""
}

function setup_ior_easy {
  io500_ior_easy_size=7440
  io500_ior_easy_params="-t 2048k -b ${io500_ior_easy_size}m -F"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
  io500_mdtest_easy_files_per_proc=32400
}

function setup_ior_hard {
  io500_ior_hard_api="POSIX"
  io500_ior_hard_api_specific_options=""
  io500_ior_hard_writes_per_proc=41450
}

function setup_mdt_hard {
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
  io500_mdtest_hard_files_per_proc=22320
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind
  io500_find_cmd_args="-C"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
  # TODO: Please add your information using the info-creator!
  # EXAMPLE:
  io500_info_system_name='GCP'      # e.g. Oakforest-PACS
}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Wed Nov  6 04:31:28 2019
Command line        : /lustre-pdstd/io-500-dev/bin/ior -r -R -t 2048k -b 7440m -F -b 7440m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_easy/ior_file_easy -O stoneWallingStatusFile=/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_easy/stonewall
Machine             : Linux slurm-io500-2019-compute1
TestID              : 0
StartTime           : Wed Nov  6 04:31:28 2019
Path                : /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_easy
FS                  : 967.9 TiB   Used FS: 7.7%   Inodes: 1950.0 Mi   Used Inodes: 8.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 200
tasks               : 1600
clients per node    : 8
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 7.27 GiB
aggregate filesize  : 11.35 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 12482248704000.
WARNING: Stat() of aggregate file size      = 7801405440.
WARNING: Using actual aggregate bytes moved = 12482248704000.
read      84480      42270      0.037850    7618560    2048.00    0.039595   140.81     0.057586   140.91     0   
Max Read:  84479.52 MiB/sec (88583.20 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        84479.52   84479.52   84479.52       0.00   42239.76   42239.76   42239.76       0.00  140.90989         NA            NA     0   1600   8    1   2     1        1         0    0      1 7801405440  2097152 11904000.0 POSIX      0
Finished            : Wed Nov  6 04:33:49 2019
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Wed Nov  6 04:07:04 2019
Command line        : /lustre-pdstd/io-500-dev/bin/ior -w -t 2048k -b 7440m -F -b 7440m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_easy/ior_file_easy -O stoneWallingStatusFile=/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux slurm-io500-2019-compute1
TestID              : 0
StartTime           : Wed Nov  6 04:07:04 2019
Path                : /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_easy
FS                  : 967.9 TiB   Used FS: 6.2%   Inodes: 1949.9 Mi   Used Inodes: 3.7%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 200
tasks               : 1600
clients per node    : 8
repetitions         : 1
xfersize            : 2 MiB
blocksize           : 7.27 GiB
aggregate filesize  : 11.35 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 3720 max: 3720 -- min data: 7.3 GiB mean data: 7.3 GiB time: 298.8s
WARNING: Expected aggregate file size       = 12482248704000.
WARNING: Stat() of aggregate file size      = 7801405440.
WARNING: Using actual aggregate bytes moved = 12482248704000.
WARNING: maybe caused by deadlineForStonewalling
write     39607      19807      0.080317    7618560    2048.00    0.040603   300.51     0.009049   300.55     0   
Max Write: 39606.77 MiB/sec (41530.71 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       39606.77   39606.77   39606.77       0.00   19803.38   19803.38   19803.38       0.00  300.55470     298.77      39843.73     0   1600   8    1   2     1        1         0    0      1 7801405440  2097152 11904000.0 POSIX      0
Finished            : Wed Nov  6 04:12:05 2019
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Wed Nov  6 04:35:32 2019
Command line        : /lustre-pdstd/io-500-dev/bin/ior -r -R -s 41450 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_hard/IOR_file -O stoneWallingStatusFile=/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_hard/stonewall
Machine             : Linux slurm-io500-2019-compute1
TestID              : 0
StartTime           : Wed Nov  6 04:35:32 2019
Path                : /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_hard
FS                  : 967.9 TiB   Used FS: 7.7%   Inodes: 1950.0 Mi   Used Inodes: 8.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 41450
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 200
tasks               : 1600
clients per node    : 8
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 2.84 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
read      3401.03    75870      874.11      45.91      45.91      0.032146   874.13     0.035978   874.19     0   
Max Read:  3401.03 MiB/sec (3566.23 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         3401.03    3401.03    3401.03       0.00   75864.40   75864.40   75864.40       0.00  874.19136         NA            NA     0   1600   8    1   0     1        1         0    0  41450    47008    47008 2973147.0 POSIX      0
Finished            : Wed Nov  6 04:50:07 2019
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Wed Nov  6 04:17:32 2019
Command line        : /lustre-pdstd/io-500-dev/bin/ior -w -s 41450 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_hard/IOR_file -O stoneWallingStatusFile=/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux slurm-io500-2019-compute1
TestID              : 0
StartTime           : Wed Nov  6 04:17:32 2019
Path                : /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_hard
FS                  : 967.9 TiB   Used FS: 7.4%   Inodes: 1949.2 Mi   Used Inodes: 6.3%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 41450
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 200
tasks               : 1600
clients per node    : 8
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 2.84 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 33158 max: 41450 -- min data: 1.5 GiB mean data: 1.8 GiB time: 300.0s
write     8672       193473     303.62      45.91      45.91      0.054349   342.79     0.018257   342.86     0   
Max Write: 8671.62 MiB/sec (9092.85 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        8671.62    8671.62    8671.62       0.00  193431.97  193431.97  193431.97       0.00  342.85956     300.05       9646.01     0   1600   8    1   0     1        1         0    0  41450    47008    47008 2973147.0 POSIX      0
Finished            : Wed Nov  6 04:23:22 2019
mdtest_easy_delete
-- started at 11/06/2019 04:53:18 --

mdtest-3.3.0+dev was launched with 1600 total task(s) on 200 node(s)
Command line used: /lustre-pdstd/io-500-dev/bin/mdtest '-r' '-F' '-P' '-d' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_easy' '-n' '32400' '-u' '-L' '-x' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_easy-stonewall' '-N' '1'
Path: /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01
FS: 967.9 TiB   Used FS: 7.7%   Inodes: 1950.0 Mi   Used Inodes: 8.0%

Nodemap: 1111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 8 for each phase.
1600 tasks, 51840000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      91020.113      91019.677      91019.929          0.096
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.896          0.896          0.896          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        569.547        569.544        569.546          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.116          1.116          1.116          0.000
-- finished at 11/06/2019 05:02:49 --

mdtest_easy_stat
-- started at 11/06/2019 04:34:02 --

mdtest-3.3.0+dev was launched with 1600 total task(s) on 200 node(s)
Command line used: /lustre-pdstd/io-500-dev/bin/mdtest '-T' '-F' '-P' '-d' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_easy' '-n' '32400' '-u' '-L' '-x' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_easy-stonewall' '-N' '1'
Path: /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01
FS: 967.9 TiB   Used FS: 7.7%   Inodes: 1950.0 Mi   Used Inodes: 8.0%

Nodemap: 1111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 8 for each phase.
1600 tasks, 51840000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     583861.211     583824.632     583842.738          6.678
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         88.794         88.788         88.791          0.001
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/06/2019 04:35:30 --

mdtest_easy_write
-- started at 11/06/2019 04:12:20 --

mdtest-3.3.0+dev was launched with 1600 total task(s) on 200 node(s)
Command line used: /lustre-pdstd/io-500-dev/bin/mdtest '-C' '-F' '-P' '-d' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_easy' '-n' '32400' '-u' '-L' '-x' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01
FS: 967.9 TiB   Used FS: 7.4%   Inodes: 1949.9 Mi   Used Inodes: 3.7%

Nodemap: 1111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 8 for each phase.
1600 tasks, 51840000 files
Continue stonewall hit min: 27508 max: 32400 avg: 31899.7 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     167627.480     167625.581     167627.040          0.226
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     169864.171             NA
   Tree creation             :          2.967          2.967          2.967          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        309.261        309.257        309.258          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.473             NA
   Tree creation             :          0.337          0.337          0.337          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/06/2019 04:17:30 --

mdtest_hard_delete
-- started at 11/06/2019 05:10:33 --

mdtest-3.3.0+dev was launched with 1600 total task(s) on 200 node(s)
Command line used: /lustre-pdstd/io-500-dev/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_hard' '-n' '22320' '-x' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01
FS: 967.9 TiB   Used FS: 7.7%   Inodes: 1950.0 Mi   Used Inodes: 5.5%

Nodemap: 1111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 8 for each phase.
1600 tasks, 35712000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      51276.757      51274.399      51276.275          0.438
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.888          0.888          0.888          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        696.488        696.456        696.462          0.006
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.126          1.126          1.126          0.000
-- finished at 11/06/2019 05:22:11 --

mdtest_hard_read
-- started at 11/06/2019 05:02:51 --

mdtest-3.3.0+dev was launched with 1600 total task(s) on 200 node(s)
Command line used: /lustre-pdstd/io-500-dev/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_hard' '-n' '22320' '-x' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01
FS: 967.9 TiB   Used FS: 7.7%   Inodes: 1949.9 Mi   Used Inodes: 5.5%

Nodemap: 1111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 8 for each phase.
1600 tasks, 35712000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      77625.899      77624.667      77625.292          0.203
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        460.060        460.053        460.056          0.001
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/06/2019 05:10:31 --

mdtest_hard_stat
-- started at 11/06/2019 04:50:14 --

mdtest-3.3.0+dev was launched with 1600 total task(s) on 200 node(s)
Command line used: /lustre-pdstd/io-500-dev/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_hard' '-n' '22320' '-x' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01
FS: 967.9 TiB   Used FS: 7.7%   Inodes: 1950.0 Mi   Used Inodes: 8.0%

Nodemap: 1111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 8 for each phase.
1600 tasks, 35712000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :     197416.406     196923.027     196946.193         64.037
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        181.350        180.897        181.329          0.059
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/06/2019 04:53:16 --

mdtest_hard_write
-- started at 11/06/2019 04:23:26 --

mdtest-3.3.0+dev was launched with 1600 total task(s) on 200 node(s)
Command line used: /lustre-pdstd/io-500-dev/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_hard' '-n' '22320' '-x' '/lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /lustre-pdstd/io-500-dev/datafiles/io500.2019.11.06-04.07.01
FS: 967.9 TiB   Used FS: 7.7%   Inodes: 1949.2 Mi   Used Inodes: 6.3%

Nodemap: 1111111100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 8 for each phase.
1600 tasks, 35712000 files
Continue stonewall hit min: 19072 max: 22320 avg: 21529.6 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     105949.244     105820.006     105944.246         11.847
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     114835.410             NA
   Tree creation             :         33.245         33.245         33.245          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        337.479        337.067        337.083          0.038
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.972             NA
   Tree creation             :          0.030          0.030          0.030          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/06/2019 04:29:07 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               38.679 GB/s : time 300.51 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              167.627 kiops : time 309.26 seconds
[RESULT] BW   phase 2            ior_hard_write                8.469 GB/s : time 342.79 seconds
[RESULT] IOPS phase 2         mdtest_hard_write              105.949 kiops : time 337.48 seconds
[RESULT] IOPS phase 3                      find              641.470 kiops : time 137.35 seconds
[RESULT] BW   phase 3             ior_easy_read               82.500 GB/s : time 140.81 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              583.861 kiops : time  88.79 seconds
[RESULT] BW   phase 4             ior_hard_read                3.321 GB/s : time 874.13 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat              197.416 kiops : time 181.35 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               91.020 kiops : time 569.55 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               77.626 kiops : time 460.06 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               51.277 kiops : time 699.25 seconds
[SCORE] Bandwidth 17.3087 GB/s : IOPS 162.058 kiops : TOTAL 52.9624