ofsdev

Institution Clemson University
Client Procs Per Node
Client Operating System Oracle Linux
Client Operating System Version 7.5
Client Kernel Version 3.10.0-862.14.4.el7.x86_64

DATA SERVER

Storage Type HDD
Volatile Memory 16GB
Storage Interface SAS
Network InfiniBand FDR
Software Version 2.10.5
OS Version 7.5

INFORMATION

Client Nodes 10
Client Total Procs 40
Metadata Nodes 16
Metadata Storage Devices 2
Data Nodes 16
Data Storage Devices 12

METADATA

Easy Write 18.20 kIOP/s
Easy Stat 55.97 kIOP/s
Easy Delete 20.15 kIOP/s
Hard Write 15.69 kIOP/s
Hard Read 19.43 kIOP/s
Hard Stat 23.87 kIOP/s
Hard Delete 8.14 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.
#set -x

if [ "$1" == "" ]
then
	SCALE=1
else
	SCALE=$1
fi


NP=$(( $SCALE * 10 ))

echo "$SCALE processes per node for $NP processes."

set -euo pipefail  # better error handling

export OFS_MOUNT=/mnt/lustre/jburto2

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"     
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"  
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="True"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_stonewall_timer=300 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...


# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths    
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it. 
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$OFS_MOUNT/io500/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept

  mkdir -p $io500_workdir $io500_result_dir
  mkdir -p ${io500_workdir}/ior_easy ${io500_workdir}/ior_hard 

# for ior_easy, large chunks, as few targets as will allow the files to be evenly spread.
  lfs setstripe -c $(( 8 / ${SCALE}  )) ${io500_workdir}/ior_easy  # turn off striping for ior_easy
# stripe across all OSTs for ior_hard, 64k chunksize
# best pattern is minimal chunksize to fit one I/O in, regardless of RAID stripe.
  lfs setstripe -c -1 -S 64k ${io500_workdir}/ior_hard 
# Enable DNE2
# https://lustre.ornl.gov/ecosystem-2016/documents/papers/LustreEco2016-Simmons-DNE.pdf 
  lfs setdirstripe -c 16 ${io500_workdir}/mdt_easy
  lfs setdirstripe -c 16 ${io500_workdir}/mdt_hard
  echo "Stripes set"
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpi_prefix="/usr/lib64/openmpi"
  #io500_mpi_prefix="/home/jburto2/openmpi/1.10.7"
  io500_mpirun="$io500_mpi_prefix/bin/mpirun"

  # Run OpenMPI over IB to keep the ethernet network clear for data. Map by node to balance processes.
  # The I/O 500 benchmarks are not heavy on interprocess communication.
  io500_mpiargs="-np $NP --mca btl_tcp_if_exclude ib0 --mca btl ^openib --map-by node --machinefile /home/jburto2/pvfs10nodelistmpi --prefix $io500_mpi_prefix"
}

function setup_ior_easy {
# 4M writes, 416 GB per proc, file per proc. 
  io500_ior_easy_size=$((416 * 1024 / $SCALE))
  io500_ior_easy_params="-t 4m -b ${io500_ior_easy_size}m -F -a MPIIO"
   
}

function setup_mdt_easy {
# one level, 11 directories, unique dir per thread, files only at leaves.
# BeeGFS doesn't have distributed directories, so more directories = better distribution. 
#  io500_mdtest_easy_params="-z 1 -b 6 -u -L" 
  io500_mdtest_easy_params="-u -L" 
  io500_mdtest_easy_files_per_proc=$((2000000 / $SCALE ))
}

function setup_ior_hard {
  if [ "$SCALE" == "1" ] 
  then
  	io500_ior_hard_writes_per_proc=200000
  else	
  	io500_ior_hard_writes_per_proc=$(( 200000 / $SCALE ))
  fi

  io500_ior_hard_other_options=" -a MPIIO"

}

function setup_mdt_hard {
  io500_mdtest_hard_files_per_proc="$(( 800000 / $SCALE ))"
  io500_mdtest_files_per_proc=$(( 800000 / $SCALE )) 
  io500_mdtest_hard_other_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/ 
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script. 
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  io500_find_cmd_args="-s 10000 -r $io500_result_dir/pfind_results"
  
  # for GPFS systems, you should probably use the provided mmfind wrapper 
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  io500_mdreal_params="-P=5000 -I=1000"
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./bin/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Add key/value pairs defining your system 
# Feel free to add extra ones if you'd like
function extra_description {
  # top level info
  io500_info_system_name='Palmetto ofstest'      # e.g. Oakforest-PACS
  io500_info_institute_name='Clemson University'   # e.g. JCAHPC
  io500_info_storage_age_in_months='0' # not install date but age since last refresh
  io500_info_storage_install_date='4/12'  # MM/YY
  io500_info_filesysem='Lustre'     # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
  io500_info_filesystem_version='2.10.5'
  # client side info
  io500_info_num_client_nodes="10"
  io500_info_procs_per_node="${SCALE}"
  # server side info
  io500_info_num_metadata_server_nodes='16'
  io500_info_num_data_server_nodes='16'
  io500_info_num_data_storage_devices='160'  # if you have 5 data servers, and each has 5 drives, then this number is 25
  io500_info_num_metadata_storage_devices='32'  # if you have 2 metadata servers, and each has 5 drives, then this number is 10
  io500_info_data_storage_type='HDD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_metadata_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_storage_network='infiniband' # infiniband, omnipath, ethernet, etc
  io500_info_storage_interface='SAS' # SAS, SATA, NVMe, etc
  # miscellaneous
  io500_info_whatever='infiniband'
}

main
ior_easy_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Thu Nov  8 17:40:27 2018
Command line        : /home/jburto2/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 4m -b 106496m -F -a MPIIO -o /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_easy/stonewall
Machine             : Linux pvfs017-ib0.palmetto.clemson.edu
TestID              : 0
StartTime           : Thu Nov  8 17:40:27 2018
Path                : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_easy
FS                  : 144.2 TiB   Used FS: 19.1%   Inodes: 290.9 Mi   Used Inodes: 3.6%

Options: 
api                 : MPIIO
apiVersion          : (3.0)
test filename       : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 40
clients per node    : 4
repetitions         : 1
xfersize            : 4 MiB
blocksize           : 104 GiB
aggregate filesize  : 4.06 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      5356       109051904  4096       0.009937   795.35     0.158370   795.37     0   
Max Read:  5355.83 MiB/sec (5615.99 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         5355.83    5355.83    5355.83       0.00    1338.96    1338.96    1338.96       0.00  795.36555     0     40   4    1   1     1        1         0    0      1 111669149696  4194304 4259840.0 MPIIO      0
Finished            : Thu Nov  8 17:53:43 2018
ior_easy_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Thu Nov  8 17:00:55 2018
Command line        : /home/jburto2/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 4m -b 106496m -F -a MPIIO -o /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux pvfs017-ib0.palmetto.clemson.edu
TestID              : 0
StartTime           : Thu Nov  8 17:00:55 2018
Path                : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_easy
FS                  : 144.2 TiB   Used FS: 16.2%   Inodes: 288.6 Mi   Used Inodes: 0.0%

Options: 
api                 : MPIIO
apiVersion          : (3.0)
test filename       : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 40
clients per node    : 4
repetitions         : 1
xfersize            : 4 MiB
blocksize           : 104 GiB
aggregate filesize  : 4.06 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 14573 max: 26624 -- min data: 56.9 GiB mean data: 84.9 GiB time: 300.1s
write     10074      109051904  4096       0.005834   422.80     0.049696   422.85     0   
Max Write: 10074.16 MiB/sec (10563.52 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       10074.16   10074.16   10074.16       0.00    2518.54    2518.54    2518.54       0.00  422.84820     0     40   4    1   1     1        1         0    0      1 111669149696  4194304 4259840.0 MPIIO      0
Finished            : Thu Nov  8 17:07:58 2018
ior_hard_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Thu Nov  8 17:55:33 2018
Command line        : /home/jburto2/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 50000 -a MPIIO -o /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_hard/stonewall
Machine             : Linux pvfs017-ib0.palmetto.clemson.edu
TestID              : 0
StartTime           : Thu Nov  8 17:55:33 2018
Path                : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_hard
FS                  : 144.2 TiB   Used FS: 19.1%   Inodes: 290.9 Mi   Used Inodes: 3.6%

Options: 
api                 : MPIIO
apiVersion          : (3.0)
test filename       : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 50000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 40
clients per node    : 4
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 87.56 GiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      1853.27    45.91      45.91      0.017307   48.36      0.146088   48.38      0   
Max Read:  1853.27 MiB/sec (1943.30 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         1853.27    1853.27    1853.27       0.00   41339.67   41339.67   41339.67       0.00   48.37968     0     40   4    1   0     1        1         0    0  50000    47008    47008   89660.6 MPIIO      0
Finished            : Thu Nov  8 17:56:22 2018
ior_hard_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Thu Nov  8 17:19:54 2018
Command line        : /home/jburto2/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 50000 -a MPIIO -o /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux pvfs017-ib0.palmetto.clemson.edu
TestID              : 0
StartTime           : Thu Nov  8 17:19:54 2018
Path                : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_hard
FS                  : 144.2 TiB   Used FS: 19.0%   Inodes: 290.9 Mi   Used Inodes: 2.0%

Options: 
api                 : MPIIO
apiVersion          : (3.0)
test filename       : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 50000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 40
clients per node    : 4
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 87.56 GiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 10725 max: 50000 -- min data: 0.5 GiB mean data: 1.4 GiB time: 300.3s
write     142.46     45.91      45.91      0.016351   629.21     0.141537   629.37     0   
Max Write: 142.46 MiB/sec (149.38 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write         142.46     142.46     142.46       0.00    3177.80    3177.80    3177.80       0.00  629.36716     0     40   4    1   0     1        1         0    0  50000    47008    47008   89660.6 MPIIO      0
Finished            : Thu Nov  8 17:30:24 2018
mdtest_easy_delete
-- started at 11/08/2018 17:59:56 --

mdtest-1.9.3 was launched with 40 total task(s) on 10 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-r" "-F" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_easy" "-n" "500000" "-u" "-L" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_easy-stonewall"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53
FS: 144.2 TiB   Used FS: 19.1%   Inodes: 290.9 Mi   Used Inodes: 3.6%

40 tasks, 20000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :      20147.172      20147.172      20147.172          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          2.881          2.881          2.881          0.000

-- finished at 11/08/2018 18:04:54 --
mdtest_easy_stat
-- started at 11/08/2018 17:53:44 --

mdtest-1.9.3 was launched with 40 total task(s) on 10 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-T" "-F" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_easy" "-n" "500000" "-u" "-L" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_easy-stonewall"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53
FS: 144.2 TiB   Used FS: 19.1%   Inodes: 290.9 Mi   Used Inodes: 3.6%

40 tasks, 20000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :      55972.991      55972.991      55972.991          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/08/2018 17:55:31 --
mdtest_easy_write
-- started at 11/08/2018 17:07:59 --

mdtest-1.9.3 was launched with 40 total task(s) on 10 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-C" "-F" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_easy" "-n" "500000" "-u" "-L" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_easy-stonewall" "-W" "300"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53
FS: 144.2 TiB   Used FS: 19.0%   Inodes: 290.9 Mi   Used Inodes: 0.0%

40 tasks, 20000000 files
stonewall rank 30: 133284 of 149680 
stonewall rank 10: 133083 of 149680 
Continue stonewall hit min: 133083 max: 149680 avg: 141974.5 
stonewall rank 0: 133597 of 149680 
stonewall rank 20: 133311 of 149680 
stonewall rank 4: 145258 of 149680 
stonewall rank 14: 141364 of 149680 
stonewall rank 24: 143575 of 149680 
stonewall rank 34: 141265 of 149680 
stonewall rank 8: 145647 of 149680 
stonewall rank 38: 146323 of 149680 
stonewall rank 18: 146409 of 149680 
stonewall rank 28: 145085 of 149680 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :      60905.863      60905.863      60905.863          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :        103.702        103.702        103.702          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/08/2018 17:13:28 --
stonewall rank 22: 141303 of 149680 
stonewall rank 32: 140035 of 149680 
stonewall rank 12: 139913 of 149680 
stonewall rank 2: 142149 of 149680 
stonewall rank 36: 148029 of 149680 
stonewall rank 16: 148370 of 149680 
stonewall rank 26: 149146 of 149680 
stonewall rank 31: 144561 of 149680 
stonewall rank 1: 142987 of 149680 
stonewall rank 11: 144348 of 149680 
stonewall rank 21: 142721 of 149680 
stonewall rank 35: 147554 of 149680 
stonewall rank 15: 147382 of 149680 
stonewall rank 5: 148212 of 149680 
stonewall rank 25: 148556 of 149680 
stonewall rank 19: 139965 of 149680 
stonewall rank 9: 139416 of 149680 
stonewall rank 39: 139990 of 149680 
stonewall rank 29: 139422 of 149680 
stonewall rank 13: 135632 of 149680 
stonewall rank 23: 137826 of 149680 
stonewall rank 3: 137598 of 149680 
stonewall rank 33: 135789 of 149680 
stonewall rank 17: 139347 of 149680 
stonewall rank 37: 139264 of 149680 
stonewall rank 27: 140713 of 149680 
stonewall rank 7: 140870 of 149680 
mdtest_hard_delete
-- started at 11/08/2018 18:09:16 --

mdtest-1.9.3 was launched with 40 total task(s) on 10 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-r" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_hard" "-n" "200000" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_hard-stonewall"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53
FS: 144.2 TiB   Used FS: 19.1%   Inodes: 288.6 Mi   Used Inodes: 1.7%

40 tasks, 8000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :       8139.034       8139.034       8139.034          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          7.082          7.082          7.082          0.000

-- finished at 11/08/2018 18:19:32 --
mdtest_hard_read
-- started at 11/08/2018 18:04:55 --

mdtest-1.9.3 was launched with 40 total task(s) on 10 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-E" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_hard" "-n" "200000" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_hard-stonewall"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53
FS: 144.2 TiB   Used FS: 19.1%   Inodes: 287.0 Mi   Used Inodes: 1.7%

40 tasks, 8000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :      19428.006      19428.006      19428.006          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/08/2018 18:09:14 --
mdtest_hard_stat
-- started at 11/08/2018 17:56:23 --

mdtest-1.9.3 was launched with 40 total task(s) on 10 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-T" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_hard" "-n" "200000" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_hard-stonewall"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53
FS: 144.2 TiB   Used FS: 19.1%   Inodes: 290.9 Mi   Used Inodes: 3.6%

40 tasks, 8000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :      23870.575      23870.575      23870.575          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/08/2018 17:59:54 --
mdtest_hard_write
-- started at 11/08/2018 17:30:25 --

mdtest-1.9.3 was launched with 40 total task(s) on 10 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-C" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_hard" "-n" "200000" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53/mdt_hard-stonewall" "-W" "300"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-17.00.53
FS: 144.2 TiB   Used FS: 19.1%   Inodes: 290.9 Mi   Used Inodes: 2.0%

40 tasks, 8000000 files
stonewall rank 30: 119182 of 125500 
stonewall rank 20: 117429 of 125500 
Continue stonewall hit min: 117429 max: 125500 avg: 121680.1 
stonewall rank 0: 117765 of 125500 
stonewall rank 10: 119223 of 125500 
stonewall rank 38: 121514 of 125500 
stonewall rank 24: 125315 of 125500 
stonewall rank 28: 120951 of 125500 
stonewall rank 8: 121000 of 125500 
stonewall rank 14: 124737 of 125500 
stonewall rank 18: 121661 of 125500 
stonewall rank 34: 124924 of 125500 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :      25062.887      25062.887      25062.887          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :        906.364        906.364        906.364          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/08/2018 17:35:45 --
stonewall rank 16: 120761 of 125500 
stonewall rank 32: 122005 of 125500 
stonewall rank 12: 121969 of 125500 
stonewall rank 22: 122528 of 125500 
stonewall rank 2: 122412 of 125500 
stonewall rank 6: 121857 of 125500 
stonewall rank 36: 120761 of 125500 
stonewall rank 26: 121830 of 125500 
stonewall rank 1: 118289 of 125500 
stonewall rank 21: 118358 of 125500 
stonewall rank 11: 118990 of 125500 
stonewall rank 31: 118931 of 125500 
stonewall rank 19: 123026 of 125500 
stonewall rank 5: 122046 of 125500 
stonewall rank 29: 122594 of 125500 
stonewall rank 25: 122188 of 125500 
stonewall rank 35: 121327 of 125500 
stonewall rank 15: 121523 of 125500 
stonewall rank 9: 122737 of 125500 
stonewall rank 39: 122894 of 125500 
stonewall rank 3: 122222 of 125500 
stonewall rank 33: 121856 of 125500 
stonewall rank 13: 121777 of 125500 
stonewall rank 23: 122216 of 125500 
stonewall rank 27: 123547 of 125500 
stonewall rank 7: 123631 of 125500 
stonewall rank 17: 122956 of 125500 
stonewall rank 37: 122772 of 125500 
result_summary
[RESULT] BW   phase 1            ior_easy_write                9.838 GB/s : time 422.85 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               60.906 kiops : time 714.23 seconds
[RESULT] BW   phase 2            ior_hard_write                0.139 GB/s : time 629.37 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               25.063 kiops : time 571.04 seconds
[RESULT] IOPS phase 3                      find              367.620 kiops : time  29.94 seconds
[RESULT] BW   phase 3             ior_easy_read                5.230 GB/s : time 795.37 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat               55.973 kiops : time 108.73 seconds
[RESULT] BW   phase 4             ior_hard_read                1.810 GB/s : time  48.38 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               23.871 kiops : time 211.97 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               20.147 kiops : time 299.99 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               19.428 kiops : time 260.40 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                8.139 kiops : time 618.45 seconds
[SCORE] Bandwidth 1.89722 GB/s : IOPS 35.2589 kiops : TOTAL 8.17887