ofsdev

Institution Clemson University
Client Procs Per Node
Client Operating System Oracle Linux
Client Operating System Version 7.5
Client Kernel Version 3.10.0-862.14.4.el7.x86_64

DATA SERVER

Storage Type HDD
Volatile Memory 16GB
Storage Interface SAS
Network InfiniBand FDR
Software Version 2.10.5
OS Version 7.5

INFORMATION

Client Nodes 16
Client Total Procs 32
Metadata Nodes 16
Metadata Storage Devices 2
Data Nodes 16
Data Storage Devices 12

METADATA

Easy Write 16.25 kIOP/s
Easy Stat 37.97 kIOP/s
Easy Delete 18.96 kIOP/s
Hard Write 14.03 kIOP/s
Hard Read 15.11 kIOP/s
Hard Stat 34.61 kIOP/s
Hard Delete 11.45 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.
#set -x

if [ "$1" == "" ]
then
	SCALE=1
else
	SCALE=$1
fi


NP=$(( $SCALE * 16 ))

echo "$SCALE processes per node for $NP processes."

set -euo pipefail  # better error handling

export OFS_MOUNT=/mnt/lustre/jburto2

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"     
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"  
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="True"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_stonewall_timer=300 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...


# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths    
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it. 
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$OFS_MOUNT/io500/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept

  mkdir -p $io500_workdir $io500_result_dir
  mkdir -p ${io500_workdir}/ior_easy ${io500_workdir}/ior_hard 
  #mkdir -p ${io500_workdir}/mdt_easy ${io500_workdir}/mdt_hard

# for ior_easy, large chunks, as few targets as will allow the files to be evenly spread.
  lfs setstripe -c 1 ${io500_workdir}/ior_easy  # turn off striping for ior_easy
# stripe across all OSTs for ior_hard, 64k chunksize
  lfs setstripe -c -1 -S 256k ${io500_workdir}/ior_hard 
#  lfs setstripe -c -1 -S 64k ${io500_workdir}/ior_hard 
# Enable DNE2
# https://lustre.ornl.gov/ecosystem-2016/documents/papers/LustreEco2016-Simmons-DNE.pdf 
  lfs setdirstripe -c 16 ${io500_workdir}/mdt_easy
  lfs setdirstripe -c 16 ${io500_workdir}/mdt_hard
  echo "Stripes set"
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpi_prefix="/usr/lib64/openmpi"
  #io500_mpi_prefix="/home/jburto2/openmpi/1.10.7"
  io500_mpirun="$io500_mpi_prefix/bin/mpirun"

  # Run OpenMPI over IB to keep the ethernet network clear for data. Map by node to balance processes.
  # The I/O 500 benchmarks are not heavy on interprocess communication.
  io500_mpiargs="-np $NP --mca btl_tcp_if_exclude ib0 --mca btl ^openib --map-by node --machinefile /home/jburto2/pvfsnodelistmpi --prefix $io500_mpi_prefix"
}

function setup_ior_easy {
# 2M writes, 240 GB per proc, file per proc. 
  io500_ior_easy_size=$((300 * 1024 / $SCALE))
  io500_ior_easy_params="-t 4m -b ${io500_ior_easy_size}m -F -a MPIIO"
   
}

function setup_mdt_easy {
# one level, 11 directories, unique dir per thread, files only at leaves.
# BeeGFS doesn't have distributed directories, so more directories = better distribution. 
#  io500_mdtest_easy_params="-z 1 -b 6 -u -L" 
  io500_mdtest_easy_params="-u -L" 
  io500_mdtest_easy_files_per_proc=$((1280000 / $SCALE ))
}

function setup_ior_hard {
  if [ "$SCALE" == "1" ] 
  then
  	io500_ior_hard_writes_per_proc=128000
  else	
  	io500_ior_hard_writes_per_proc=$(( 128000 / $SCALE ))
  fi

  io500_ior_hard_other_options=" -a MPIIO"

}

function setup_mdt_hard {
  io500_mdtest_hard_files_per_proc="$(( 500000 / $SCALE ))"
  io500_mdtest_files_per_proc=$(( 500000 / $SCALE )) 
  io500_mdtest_hard_other_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/ 
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script. 
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  io500_find_cmd_args="-s 10000 -r $io500_result_dir/pfind_results"
  
  # for GPFS systems, you should probably use the provided mmfind wrapper 
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  io500_mdreal_params="-P=5000 -I=1000"
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./bin/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Add key/value pairs defining your system 
# Feel free to add extra ones if you'd like
function extra_description {
  # top level info
  io500_info_system_name='Palmetto ofstest'      # e.g. Oakforest-PACS
  io500_info_institute_name='Clemson University'   # e.g. JCAHPC
  io500_info_storage_age_in_months='0' # not install date but age since last refresh
  io500_info_storage_install_date='4/12'  # MM/YY
  io500_info_filesysem='Lustre'     # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
  io500_info_filesystem_version='2.10.5'
  # client side info
  io500_info_num_client_nodes="$(( ${SCALE} * 16 ))"
  io500_info_procs_per_node="1"
  # server side info
  io500_info_num_metadata_server_nodes='16'
  io500_info_num_data_server_nodes='16'
  io500_info_num_data_storage_devices='160'  # if you have 5 data servers, and each has 5 drives, then this number is 25
  io500_info_num_metadata_storage_devices='32'  # if you have 2 metadata servers, and each has 5 drives, then this number is 10
  io500_info_data_storage_type='HDD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_metadata_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_storage_network='infiniband' # infiniband, omnipath, ethernet, etc
  io500_info_storage_interface='SAS' # SAS, SATA, NVMe, etc
  # miscellaneous
  io500_info_whatever='infiniband'
}

main
ior_easy_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Thu Nov  8 13:40:21 2018
Command line        : /home/jburto2/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 4m -b 153600m -F -a MPIIO -o /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_easy/stonewall
Machine             : Linux pvfs017-ib0.palmetto.clemson.edu
TestID              : 0
StartTime           : Thu Nov  8 13:40:21 2018
Path                : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_easy
FS                  : 144.2 TiB   Used FS: 8.8%   Inodes: 290.9 Mi   Used Inodes: 3.4%

Options: 
api                 : MPIIO
apiVersion          : (3.0)
test filename       : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 32
clients per node    : 2
repetitions         : 1
xfersize            : 4 MiB
blocksize           : 150 GiB
aggregate filesize  : 4.69 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 5153960755200.
WARNING: Stat() of aggregate file size      = 5028601397248.
WARNING: Using actual aggregate bytes moved = 5028601397248.
read      7484       157286400  4096       0.010441   640.77     0.018999   640.79     0   
Max Read:  7484.01 MiB/sec (7847.55 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         7484.01    7484.01    7484.01       0.00    1871.00    1871.00    1871.00       0.00  640.78612     0     32   2    1   1     1        1         0    0      1 161061273600  4194304 4795648.0 MPIIO      0
Finished            : Thu Nov  8 13:51:01 2018
ior_easy_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Thu Nov  8 13:06:43 2018
Command line        : /home/jburto2/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 4m -b 153600m -F -a MPIIO -o /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux pvfs017-ib0.palmetto.clemson.edu
TestID              : 0
StartTime           : Thu Nov  8 13:06:43 2018
Path                : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_easy
FS                  : 144.2 TiB   Used FS: 5.6%   Inodes: 289.6 Mi   Used Inodes: 0.0%

Options: 
api                 : MPIIO
apiVersion          : (3.0)
test filename       : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 32
clients per node    : 2
repetitions         : 1
xfersize            : 4 MiB
blocksize           : 150 GiB
aggregate filesize  : 4.69 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 30079 max: 37466 -- min data: 117.5 GiB mean data: 131.8 GiB time: 300.2s
WARNING: Expected aggregate file size       = 5153960755200.
WARNING: Stat() of aggregate file size      = 5028601397248.
WARNING: Using actual aggregate bytes moved = 5028601397248.
WARNING: maybe caused by deadlineForStonewalling
write     13617      157286400  4096       0.003649   352.15     0.019788   352.17     0   
Max Write: 13617.28 MiB/sec (14278.75 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       13617.28   13617.28   13617.28       0.00    3404.32    3404.32    3404.32       0.00  352.17371     0     32   2    1   1     1        1         0    0      1 161061273600  4194304 4795648.0 MPIIO      0
Finished            : Thu Nov  8 13:12:36 2018
ior_hard_read
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Thu Nov  8 13:53:36 2018
Command line        : /home/jburto2/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 64000 -a MPIIO -o /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_hard/stonewall
Machine             : Linux pvfs017-ib0.palmetto.clemson.edu
TestID              : 0
StartTime           : Thu Nov  8 13:53:36 2018
Path                : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_hard
FS                  : 144.2 TiB   Used FS: 8.8%   Inodes: 290.9 Mi   Used Inodes: 3.4%

Options: 
api                 : MPIIO
apiVersion          : (3.0)
test filename       : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 64000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 32
clients per node    : 2
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 89.66 GiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      173.46     45.91      45.91      0.016086   529.28     0.078929   529.30     0   
Max Read:  173.46 MiB/sec (181.89 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read          173.46     173.46     173.46       0.00    3869.24    3869.24    3869.24       0.00  529.30263     0     32   2    1   0     1        1         0    0  64000    47008    47008   91812.5 MPIIO      0
Finished            : Thu Nov  8 14:02:25 2018
ior_hard_write
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began               : Thu Nov  8 13:23:05 2018
Command line        : /home/jburto2/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 64000 -a MPIIO -o /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux pvfs017-ib0.palmetto.clemson.edu
TestID              : 0
StartTime           : Thu Nov  8 13:23:05 2018
Path                : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_hard
FS                  : 144.2 TiB   Used FS: 8.8%   Inodes: 290.9 Mi   Used Inodes: 1.9%

Options: 
api                 : MPIIO
apiVersion          : (3.0)
test filename       : /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 64000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 32
clients per node    : 2
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 89.66 GiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 25897 max: 64000 -- min data: 1.1 GiB mean data: 2.3 GiB time: 300.1s
write     198.43     45.91      45.91      0.016409   462.57     0.169441   462.70     0   
Max Write: 198.43 MiB/sec (208.07 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write         198.43     198.43     198.43       0.00    4426.24    4426.24    4426.24       0.00  462.69518     0     32   2    1   0     1        1         0    0  64000    47008    47008   91812.5 MPIIO      0
Finished            : Thu Nov  8 13:30:48 2018
mdtest_easy_delete
-- started at 11/08/2018 14:04:47 --

mdtest-1.9.3 was launched with 32 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-r" "-F" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_easy" "-n" "640000" "-u" "-L" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_easy-stonewall"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41
FS: 144.2 TiB   Used FS: 8.8%   Inodes: 290.9 Mi   Used Inodes: 3.4%

32 tasks, 20480000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :      18957.789      18957.789      18957.789          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          2.992          2.992          2.992          0.000

-- finished at 11/08/2018 14:09:50 --
mdtest_easy_stat
-- started at 11/08/2018 13:51:03 --

mdtest-1.9.3 was launched with 32 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-T" "-F" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_easy" "-n" "640000" "-u" "-L" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_easy-stonewall"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41
FS: 144.2 TiB   Used FS: 8.8%   Inodes: 290.9 Mi   Used Inodes: 3.4%

32 tasks, 20480000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :      37967.080      37967.080      37967.080          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/08/2018 13:53:34 --
mdtest_easy_write
-- started at 11/08/2018 13:12:37 --

mdtest-1.9.3 was launched with 32 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-C" "-F" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_easy" "-n" "640000" "-u" "-L" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_easy-stonewall" "-W" "300"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41
FS: 144.2 TiB   Used FS: 8.8%   Inodes: 290.9 Mi   Used Inodes: 0.0%

32 tasks, 20480000 files
errno 25, Inappropriate ioctl for device 
stonewall rank 16: 170890 of 179222 
Continue stonewall hit min: 148636 max: 179222 avg: 163526.8 
stonewall rank 0: 170424 of 179222 
stonewall rank 12: 154579 of 179222 
stonewall rank 20: 166247 of 179222 
stonewall rank 28: 155466 of 179222 
stonewall rank 8: 168913 of 179222 
stonewall rank 24: 168302 of 179222 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :      58111.869      58111.869      58111.869          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :         88.437         88.437         88.437          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/08/2018 13:18:30 --
stonewall rank 2: 171495 of 179222 
stonewall rank 18: 171133 of 179222 
stonewall rank 26: 169761 of 179222 
stonewall rank 22: 153595 of 179222 
stonewall rank 10: 172322 of 179222 
stonewall rank 6: 154876 of 179222 
stonewall rank 30: 148799 of 179222 
stonewall rank 14: 149625 of 179222 
stonewall rank 1: 153487 of 179222 
stonewall rank 17: 156995 of 179222 
stonewall rank 21: 171886 of 179222 
stonewall rank 9: 155195 of 179222 
stonewall rank 5: 171994 of 179222 
stonewall rank 25: 155348 of 179222 
stonewall rank 29: 172977 of 179222 
stonewall rank 13: 173964 of 179222 
stonewall rank 3: 170751 of 179222 
stonewall rank 19: 172089 of 179222 
stonewall rank 7: 153359 of 179222 
stonewall rank 23: 152527 of 179222 
stonewall rank 11: 148636 of 179222 
stonewall rank 27: 150691 of 179222 
stonewall rank 31: 174003 of 179222 
stonewall rank 15: 173306 of 179222 
mdtest_hard_delete
-- started at 11/08/2018 14:15:10 --

mdtest-1.9.3 was launched with 32 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-r" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_hard" "-n" "250000" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_hard-stonewall"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41
FS: 144.2 TiB   Used FS: 8.8%   Inodes: 289.3 Mi   Used Inodes: 1.6%

32 tasks, 8000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :      11453.276      11453.276      11453.276          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          7.425          7.425          7.425          0.000

-- finished at 11/08/2018 14:22:06 --
mdtest_hard_read
-- started at 11/08/2018 14:09:52 --

mdtest-1.9.3 was launched with 32 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-E" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_hard" "-n" "250000" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_hard-stonewall"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41
FS: 144.2 TiB   Used FS: 8.8%   Inodes: 287.3 Mi   Used Inodes: 1.6%

32 tasks, 8000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :      15109.044      15109.044      15109.044          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/08/2018 14:15:08 --
mdtest_hard_stat
-- started at 11/08/2018 14:02:27 --

mdtest-1.9.3 was launched with 32 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-T" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_hard" "-n" "250000" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_hard-stonewall"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41
FS: 144.2 TiB   Used FS: 8.8%   Inodes: 290.9 Mi   Used Inodes: 3.4%

32 tasks, 8000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :      34610.272      34610.272      34610.272          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/08/2018 14:04:45 --
mdtest_hard_write
-- started at 11/08/2018 13:30:50 --

mdtest-1.9.3 was launched with 32 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest "-C" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_hard" "-n" "250000" "-x" "/mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41/mdt_hard-stonewall" "-W" "300"
Path: /mnt/lustre/jburto2/io500/datafiles/io500.2018.11.08-13.06.41
FS: 144.2 TiB   Used FS: 8.8%   Inodes: 290.9 Mi   Used Inodes: 1.9%

32 tasks, 8000000 files
stonewall rank 16: 142857 of 149109 
Continue stonewall hit min: 132126 max: 149109 avg: 137195.6 
stonewall rank 0: 142174 of 149109 
stonewall rank 20: 143605 of 149109 
stonewall rank 24: 136028 of 149109 
stonewall rank 8: 136759 of 149109 
stonewall rank 28: 135187 of 149109 
stonewall rank 12: 134848 of 149109 
stonewall rank 18: 143403 of 149109 
stonewall rank 2: 144220 of 149109 
stonewall rank 26: 135300 of 149109 
stonewall rank 6: 135469 of 149109 
stonewall rank 10: 134873 of 149109 
stonewall rank 22: 135134 of 149109 
stonewall rank 14: 132126 of 149109 
stonewall rank 30: 132538 of 149109 

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :      23480.619      23480.619      23480.619          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :        823.702        823.702        823.702          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 11/08/2018 13:36:30 --
stonewall rank 17: 141023 of 149109 
stonewall rank 1: 140068 of 149109 
stonewall rank 9: 134456 of 149109 
stonewall rank 25: 135196 of 149109 
stonewall rank 5: 138411 of 149109 
stonewall rank 21: 137980 of 149109 
stonewall rank 13: 136893 of 149109 
stonewall rank 29: 136853 of 149109 
stonewall rank 3: 136542 of 149109 
stonewall rank 19: 137859 of 149109 
stonewall rank 11: 132574 of 149109 
stonewall rank 7: 135029 of 149109 
stonewall rank 23: 134475 of 149109 
stonewall rank 27: 133392 of 149109 
stonewall rank 15: 132862 of 149109 
stonewall rank 31: 133017 of 149109 
result_summary
[RESULT] BW   phase 1            ior_easy_write               13.298 GB/s : time 352.17 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               58.112 kiops : time 626.36 seconds
[RESULT] BW   phase 2            ior_hard_write                0.194 GB/s : time 462.70 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               23.481 kiops : time 539.36 seconds
[RESULT] IOPS phase 3                      find              338.390 kiops : time  31.05 seconds
[RESULT] BW   phase 3             ior_easy_read                7.309 GB/s : time 640.79 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat               37.967 kiops : time 152.85 seconds
[RESULT] BW   phase 4             ior_hard_read                0.169 GB/s : time 529.30 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               34.610 kiops : time 139.78 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               18.958 kiops : time 304.69 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               15.109 kiops : time 317.81 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               11.453 kiops : time 418.52 seconds
[SCORE] Bandwidth 1.33646 GB/s : IOPS 34.4643 kiops : TOTAL 6.78676