ofsdev

Institution Clemson University
Client Procs Per Node
Client Operating System Oracle Linux
Client Operating System Version 7.4
Client Kernel Version 4.1.12-103.9.4.el7uek.x86_64

DATA SERVER

Storage Type HDD
Volatile Memory 16GB
Storage Interface SAS
Network InfiniBand FDR
Software Version 7.0
OS Version 7.5

INFORMATION

Client Nodes 16
Client Total Procs 16
Metadata Nodes 16
Metadata Storage Devices 2
Data Nodes 16
Data Storage Devices 12

METADATA

Easy Write 37.13 kIOP/s
Easy Stat 129.05 kIOP/s
Easy Delete 63.88 kIOP/s
Hard Write 6.58 kIOP/s
Hard Read 11.36 kIOP/s
Hard Stat 73.13 kIOP/s
Hard Delete 7.70 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.
#set -x

if [ "$1" == "" ]
then
	SCALE=1
else
	SCALE=$1
fi


NP=$(( $SCALE * 16 ))

echo "$SCALE processes per node for $NP processes."

set -euo pipefail  # better error handling

export OFS_MOUNT=/mnt/beegfs/jburto2

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"     
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"  
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="True"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths    
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it. 
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$OFS_MOUNT/io500/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept

  mkdir -p $io500_workdir $io500_result_dir
  mkdir -p ${io500_workdir}/ior_easy ${io500_workdir}/ior_hard 
  mkdir -p ${io500_workdir}/mdt_easy ${io500_workdir}/mdt_hard 
# for ior_easy. 
# 1 targets
  beegfs-ctl --setpattern --numtargets=1 --chunksize=2048k ${io500_workdir}/ior_easy
# stripe across all OSTs for ior_hard, 256k chunksize
  beegfs-ctl --setpattern --numtargets=16 --chunksize=256k ${io500_workdir}/ior_hard 
# turn off striping and use small chunks for mdtest
  beegfs-ctl --setpattern --numtargets=1 --chunksize=256k ${io500_workdir}/mdt_easy 
  beegfs-ctl --setpattern --numtargets=1 --chunksize=256k ${io500_workdir}/mdt_hard 
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpi_prefix="/usr/lib64/openmpi"
  io500_mpirun="$io500_mpi_prefix/bin/mpirun"
  
  io500_mpiargs="-np $NP --mca btl_tcp_if_exclude ib0 --mca btl ^openib --map-by node --machinefile /home/jburto2/pvfsnodelistmpi --prefix $io500_mpi_prefix"
}

function setup_ior_easy {
# 1M writes, 300 GB per proc, file per proc. 
  io500_ior_easy_size=$((320 * 1024 / $SCALE))
  io500_ior_easy_params="-t 4m -b ${io500_ior_easy_size}m -F -a POSIX"
   
}

function setup_mdt_easy {
# one level, 11 directories, unique dir per thread, files only at leaves.
# BeeGFS doesn't have distributed directories, so more directories = better distribution. 
  io500_mdtest_easy_params="-z 1 -b 16 -u -L" 
  io500_mdtest_easy_files_per_proc=1000000 
}

function setup_ior_hard {
  if [ "$SCALE" == "1" ] 
  then
	# One process per node is significantly faster because of buffering.
  	io500_ior_hard_writes_per_proc=2500000
  else	
  	io500_ior_hard_writes_per_proc=$(( 180000 / $SCALE ))
  fi

  io500_ior_hard_other_options=" -a POSIX"

}

function setup_mdt_hard {
# Multiple directories might improve mdt_hard slightly, but this test is storage bound, not md bound.
  io500_mdtest_hard_files_per_proc="$(( 150000 / $SCALE ))"
  io500_mdtest_files_per_proc=$(( 150000 / $SCALE )) 

}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/ 
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script. 
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  io500_find_cmd_args="-s 10000 -r $io500_result_dir/pfind_results"
  
  # for GPFS systems, you should probably use the provided mmfind wrapper 
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  io500_mdreal_params="-P=5000 -I=1000"
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./bin/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Add key/value pairs defining your system 
# Feel free to add extra ones if you'd like
function extra_description {
  # top level info
  io500_info_system_name='Palmetto ofstest'      # e.g. Oakforest-PACS
  io500_info_institute_name='Clemson University'   # e.g. JCAHPC
  io500_info_storage_age_in_months='0' # not install date but age since last refresh
  io500_info_storage_install_date='4/12'  # MM/YY
  io500_info_filesysem='BeeGFS'     # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
  io500_info_filesystem_version='7'
  # client side info
  io500_info_num_client_nodes='16'
  io500_info_procs_per_node="${SCALE}"
  # server side info
  io500_info_num_metadata_server_nodes='16'
  io500_info_num_data_server_nodes='16'
  io500_info_num_data_storage_devices='192'  # if you have 5 data servers, and each has 5 drives, then this number is 25
  io500_info_num_metadata_storage_devices='32'  # if you have 2 metadata servers, and each has 5 drives, then this number is 10
  io500_info_data_storage_type='HDD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_metadata_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_storage_network='infiniband' # infiniband, omnipath, ethernet, etc
  io500_info_storage_interface='SAS' # SAS, SATA, NVMe, etc
  # miscellaneous
  io500_info_whatever='infiniband'
}

main
ior_easy_read
IOR-3.1.0: MPI Coordinated Test of Parallel I/O

Began: Tue Aug 28 12:19:58 2018
Command line used: /home/jburto2/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 4m -b 327680m -F -a POSIX -o /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/ior_easy/ior_file_easy
Machine: Linux pvfs017.ofsdev.clemson.edu

Test 0 started: Tue Aug 28 12:19:58 2018
Summary:
	api                = POSIX
	test filename      = /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/ior_easy/ior_file_easy
	access             = file-per-process
	ordering in a file = sequential offsets
	ordering inter file= constant task offsets = 1
	clients            = 16 (1 per node)
	repetitions        = 1
	xfersize           = 4 MiB
	blocksize          = 320 GiB
	aggregate filesize = 5120 GiB

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      12316      335544320  4096       0.004306   425.67     0.025804   425.68     0   

Max Read:  12316.44 MiB/sec (12914.72 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
read        12316.44   12316.44   12316.44       0.00    3079.11    3079.11    3079.11       0.00  425.68158 0 16 1 1 1 1 1 0 0 1 343597383680 4194304 5497558138880 POSIX 0

Finished: Tue Aug 28 12:27:04 2018
ior_easy_write
IOR-3.1.0: MPI Coordinated Test of Parallel I/O

Began: Tue Aug 28 11:51:39 2018
Command line used: /home/jburto2/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 4m -b 327680m -F -a POSIX -o /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/ior_easy/ior_file_easy
Machine: Linux pvfs017.ofsdev.clemson.edu

Test 0 started: Tue Aug 28 11:51:39 2018
Summary:
	api                = POSIX
	test filename      = /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/ior_easy/ior_file_easy
	access             = file-per-process
	ordering in a file = sequential offsets
	ordering inter file= constant task offsets = 1
	clients            = 16 (1 per node)
	repetitions        = 1
	xfersize           = 4 MiB
	blocksize          = 320 GiB
	aggregate filesize = 5120 GiB

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
write     15594      335544320  4096       0.003899   336.18     0.038996   336.21     0   

Max Write: 15593.87 MiB/sec (16351.36 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
write       15593.87   15593.87   15593.87       0.00    3898.47    3898.47    3898.47       0.00  336.21410 0 16 1 1 1 1 1 0 0 1 343597383680 4194304 5497558138880 POSIX 0

Finished: Tue Aug 28 11:57:16 2018
ior_hard_read
IOR-3.1.0: MPI Coordinated Test of Parallel I/O

Began: Tue Aug 28 12:29:11 2018
Command line used: /home/jburto2/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 2500000 -a POSIX -o /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/ior_hard/IOR_file
Machine: Linux pvfs017.ofsdev.clemson.edu

Test 0 started: Tue Aug 28 12:29:11 2018
Summary:
	api                = POSIX
	test filename      = /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/ior_hard/IOR_file
	access             = single-shared-file
	ordering in a file = sequential offsets
	ordering inter file= constant task offsets = 1
	clients            = 16 (1 per node)
	repetitions        = 1
	xfersize           = 47008 bytes
	blocksize          = 47008 bytes
	aggregate filesize = 1751.18 GiB

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
read      4193       45.91      45.91      0.028882   427.61     0.092489   427.68     0   

Max Read:  4192.86 MiB/sec (4396.53 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
read         4192.86    4192.86    4192.86       0.00   93527.35   93527.35   93527.35       0.00  427.68239 0 16 1 1 0 1 1 0 0 2500000 47008 47008 1880320000000 POSIX 0

Finished: Tue Aug 28 12:36:19 2018
ior_hard_write
IOR-3.1.0: MPI Coordinated Test of Parallel I/O

Began: Tue Aug 28 12:04:46 2018
Command line used: /home/jburto2/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 2500000 -a POSIX -o /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/ior_hard/IOR_file
Machine: Linux pvfs017.ofsdev.clemson.edu

Test 0 started: Tue Aug 28 12:04:46 2018
Summary:
	api                = POSIX
	test filename      = /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/ior_hard/IOR_file
	access             = single-shared-file
	ordering in a file = sequential offsets
	ordering inter file= constant task offsets = 1
	clients            = 16 (1 per node)
	repetitions        = 1
	xfersize           = 47008 bytes
	blocksize          = 47008 bytes
	aggregate filesize = 1751.18 GiB

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
write     3946       45.91      45.91      0.120683   454.22     0.072259   454.41     0   

Max Write: 3946.25 MiB/sec (4137.94 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
write        3946.25    3946.25    3946.25       0.00   88026.38   88026.38   88026.38       0.00  454.40923 0 16 1 1 0 1 1 0 0 2500000 47008 47008 1880320000000 POSIX 0

Finished: Tue Aug 28 12:12:20 2018
mdtest_easy_delete
-- started at 08/28/2018 12:36:55 --

mdtest-1.9.3 was launched with 16 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest -r -F -d /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/mdt_easy -n 1000000 -z 1 -b 16 -u -L
Path: /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37
FS: 145.4 TiB   Used FS: 4.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

16 tasks, 16000000 files

SUMMARY: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :      63881.583      63881.583      63881.583          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          7.020          7.020          7.020          0.000

-- finished at 08/28/2018 12:41:08 --
mdtest_easy_stat
-- started at 08/28/2018 12:27:05 --

mdtest-1.9.3 was launched with 16 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest -T -F -d /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/mdt_easy -n 1000000 -z 1 -b 16 -u -L
Path: /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37
FS: 145.4 TiB   Used FS: 4.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

16 tasks, 16000000 files

SUMMARY: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :     129050.376     129050.376     129050.376          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 08/28/2018 12:29:09 --
mdtest_easy_write
-- started at 08/28/2018 11:57:32 --

mdtest-1.9.3 was launched with 16 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest -C -F -d /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/mdt_easy -n 1000000 -z 1 -b 16 -u -L
Path: /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37
FS: 145.4 TiB   Used FS: 3.4%   Inodes: 0.0 Mi   Used Inodes: -nan%

16 tasks, 16000000 files

SUMMARY: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :      37129.315      37129.315      37129.315          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :         32.292         32.292         32.292          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 08/28/2018 12:04:44 --
mdtest_hard_delete
-- started at 08/28/2018 12:44:42 --

mdtest-1.9.3 was launched with 16 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest -r -t -F -w 3901 -e 3901 -d /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/mdt_hard -n 150000
Path: /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37
FS: 145.4 TiB   Used FS: 4.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

16 tasks, 2400000 files

SUMMARY: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :       7697.120       7697.120       7697.120          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.395          0.395          0.395          0.000

-- finished at 08/28/2018 12:49:57 --
mdtest_hard_read
-- started at 08/28/2018 12:41:09 --

mdtest-1.9.3 was launched with 16 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest -E -t -F -w 3901 -e 3901 -d /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/mdt_hard -n 150000
Path: /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37
FS: 145.4 TiB   Used FS: 4.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

16 tasks, 2400000 files

SUMMARY: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :      11361.401      11361.401      11361.401          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 08/28/2018 12:44:41 --
mdtest_hard_stat
-- started at 08/28/2018 12:36:20 --

mdtest-1.9.3 was launched with 16 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest -T -t -F -w 3901 -e 3901 -d /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/mdt_hard -n 150000
Path: /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37
FS: 145.4 TiB   Used FS: 4.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

16 tasks, 2400000 files

SUMMARY: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :          0.000          0.000          0.000          0.000
   File stat         :      73130.070      73130.070      73130.070          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :          0.000          0.000          0.000          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 08/28/2018 12:36:53 --
mdtest_hard_write
-- started at 08/28/2018 12:12:22 --

mdtest-1.9.3 was launched with 16 total task(s) on 16 node(s)
Command line used: /home/jburto2/io-500-dev/bin/mdtest -C -t -F -w 3901 -e 3901 -d /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37/mdt_hard -n 150000
Path: /mnt/beegfs/jburto2/io500/datafiles/io500.2018.08.28-11.51.37
FS: 145.4 TiB   Used FS: 4.6%   Inodes: 0.0 Mi   Used Inodes: -nan%

16 tasks, 2400000 files

SUMMARY: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation     :       6575.286       6575.286       6575.286          0.000
   File stat         :          0.000          0.000          0.000          0.000
   File read         :          0.000          0.000          0.000          0.000
   File removal      :          0.000          0.000          0.000          0.000
   Tree creation     :         41.137         41.137         41.137          0.000
   Tree removal      :          0.000          0.000          0.000          0.000

-- finished at 08/28/2018 12:18:27 --
result_summary
[RESULT] BW   phase 1            ior_easy_write               15.228 GB/s : time 336.21 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               37.129 kiops : time 445.20 seconds
[RESULT] BW   phase 2            ior_hard_write                3.854 GB/s : time 454.41 seconds
[RESULT] IOPS phase 2         mdtest_hard_write                6.575 kiops : time 369.74 seconds
[RESULT] IOPS phase 3                      find              213.370 kiops : time  86.23 seconds
[RESULT] BW   phase 3             ior_easy_read               12.027 GB/s : time 425.68 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              129.050 kiops : time 125.65 seconds
[RESULT] BW   phase 4             ior_hard_read                4.095 GB/s : time 427.68 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               73.130 kiops : time  34.50 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               63.882 kiops : time 254.61 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               11.361 kiops : time 212.98 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                7.697 kiops : time 319.04 seconds
[SCORE] Bandwidth 7.33208 GB/s : IOPS 35.8793 kiops : TOTAL 16.2194