Frontera
| Institution | TACC |
|---|---|
| Client Procs Per Node | |
| Client Operating System | CentOS |
| Client Operating System Version | 7.7 |
| Client Kernel Version | 3.1 |
DATA SERVER
| Storage Type | NVMe-SSD |
|---|---|
| Volatile Memory | 128GB |
| Storage Interface | Infiniband |
| Network | IB-HDR |
| Software Version | 1.4.1 |
| OS Version | CentOS 7.7 |
INFORMATION
| Client Nodes | 10 |
|---|---|
| Client Total Procs | 280 |
| Metadata Nodes | 2 |
| Metadata Storage Devices | 468 |
| Data Nodes | 32 |
| Data Storage Devices | 22 |
METADATA
| Easy Write | 34.65 kIOP/s |
|---|---|
| Easy Stat | 259.21 kIOP/s |
| Easy Delete | 24.09 kIOP/s |
| Hard Write | 6.54 kIOP/s |
| Hard Read | 174.36 kIOP/s |
| Hard Stat | 144.32 kIOP/s |
| Hard Delete | 20.57 kIOP/s |
Submitted Files
- io500
-
#!/bin/bash # # INSTRUCTIONS: # This script takes its parameters from the same .ini file as io500 binary. # # The only parts of the script that may need to be modified are: # - setup_paths() to configure the binary locations and MPI parameters # - setup_directories() to create/tune the IOR/mdtest output directories # # Please visit https://vi4io.org/io500-info-creator/ to help generate the # "system-information.txt" file, by pasting the output of the info-creator. # This file contains details of your system hardware for your submission. # Set the paths to the binaries and how to launch MPI jobs. # If you ran ./prepare.sh successfully, then binaries are in ./bin/ function setup_paths { ##CNODES="flash-32,flash-33,flash-34,flash-35,flash-36,flash-37,flash-38,flash-39,flash-40,flash-41" CNODES="flash-42,flash-43,flash-44,flash-45,flash-47,flash-48,flash-50,flash-51,flash-52,flash-53" #CNODES="flash-32,flash-33,flash-37,flash-38,flash-44,flash-45,flash-50,flash-51,flash-56,flash-57" #CNODES="flash-32,flash-37,flash-40,flash-41,flash-42,flash-45,flash-47,flash-48,flash-63" #BINDING="-genv MV2_CPU_MAPPING 0,28:14,42:1,29:15,43:2,30:16,44:3,31:17,45:4,32:18,46:5,33:19,47:6,34:20,48:7,35:21,49:8,36:22,50:9,37:23,51:10,38:24,52:11,39:25,53:12,40:26,54:13,41:27,55" #BINDING="-genv MV2_CPU_MAPPING 0,28:1,29:2,30:3,31:4,32:5,33:6,34:7,35:8,36:9,37:10,38:11,39:12,40:13,41:14,42:15,43:16,44:17,45:18,46:19,47:20,48:21,49:22,50:23,51:24,52:25,53:26,54:27,55" #BINDING="-genv MV2_CPU_MAPPING 0:28:1:29:2:30:3:31:4:32:5:33:6:34:7:35:8:36:9:37:10:38:11:39:12:40:13:41:14:42:15:43:16:44:17:45:18:46:19:47:20:48:21:49:22:50:23:51:24:52:25:53:26:54:27:55" BINDING="-genv MV2_CPU_MAPPING 0:1:14:15:2:3:16:17:4:5:18:19:6:7:20:21:8:9:22:23:10:11:24:25:12:13:26:27" #BINDING="-genv MV2_CPU_MAPPING 0,28:1,29:2,30:3,31:4,32:5,33:6,34:7,35:8,36:9,37:10,38:11,39:12,40:13,41" #BINDING="-genv MV2_CPU_MAPPING 0,28,14,42,1,29,15,43,2,30,16,44,3,31,17,45,4,32,18,46,5,33,19,47,6,34,20,48,7,35,21,49,8,36,22,50,9,37,23,51,10,38,24,52,11,39,25,53,12,40,26,54,13,41,27,55" io500_ior_cmd=$PWD/bin/ior io500_mdtest_cmd=$PWD/bin/mdtest io500_mpirun="/opt/ddn/mvapich/bin/mpirun" #io500_mpiargs="-np 110 -ppn 11 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_NUM_HCAS 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_BFS_PATH /scratch2/flash -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so" #io500_mpiargs="-np 280 -ppn 28 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_NUM_HCAS 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so" #io500_mpiargs="-np 120 -ppn 12 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so" #io500_mpiargs="-np 280 -ppn 28 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_BFS_PATH /scratch2/flash -genv IM_CLIENT_ENABLE_SGL 1 -genv IM_CLIENT_NO_MKNOD_CREATE 1 -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so" #w/o mmd #io500_mpiargs="-np 280 -ppn 28 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 $BINDING --hosts $CNODES -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 " #mmd #io500_mpiargs="-ppn 14 -np 140 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 --hosts $CNODES -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 -env IM_CLIENT_NET_IF ib0-verbs $PWD/io500 $io500_ini --timestamp $timestamp : -np 140 -env IM_CLIENT_NET_IF ib1-verbs " #taskset io500_mpiargs="-np 140 -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so -genv IM_CLIENT_BFS_PATH /scratch2/flash/ -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_USE_RDMA_CM 0 --hosts $CNODES -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 -env IM_CLIENT_NET_IF ib0-verbs taskset -c 0-13 $PWD/io500 $io500_ini --timestamp $timestamp : -np 140 -env IM_CLIENT_NET_IF ib1-verbs taskset -c 14-27 " #io500_mpiargs="-np 28 -ppn 28 -genv LD_LIBRARY_PATH /opt/ddn/ime/lib:/opt/ddn/mvapich/lib64:\$LD_LIBRARY_PATH -genv IM_CLIENT_PGEOM 1+0 -genv MV2_HOMOGENEOUS_CLUSTER 1 -genv MV2_NUM_HCAS 1 -genv MV2_USE_RDMA_CM 0 $BINDING -genv IM_CLIENT_NET_IF ib0-verbs,ib1-verbs -genv IM_CLIENT_MIN_CONNECTIONS 0 -genv IM_CLIENT_ENABLE_SGL 1 -genv LD_PRELOAD /flash/io500/posix_2_ime/libposix2ime.so" } # Set directories where benchmark files are created and where the results go. # If you want to set up stripe tuning on your output directories or anything # similar, then this is the right place to do it. function setup_directories { local workdir local resultdir local ts mkdir -p $io500_workdir $io500_resultdir eval $(grep "^BFSDIR" /etc/ddn/ime/ime-fuse.conf) eval $(grep "^MNTDIR" /etc/ddn/ime/ime-fuse.conf) bfspath=$(echo "$BFSDIR${io500_workdir#"$MNTDIR"}") # lfs setstripe -L mdt -E 1M $bfspath # lfs setdirstripe -D -c -1 $bfspath # Example commands to create output directories for Lustre. Creating # top-level directories is allowed, but not the whole directory tree. #if (( $(lfs df $io500_workdir | grep -c MDT) > 1 )); then # lfs setdirstripe -D -c -1 $io500_workdir #fi #lfs setstripe -c 1 $io500_workdir #mkdir $io500_workdir/ior-easy $io500_workdir/ior-hard #mkdir $io500_workdir/mdtest-easy $io500_workdir/mdtest-hard #local osts=$(lfs df $io500_workdir | grep -c OST) # Try overstriping for ior-hard to improve scaling, or use wide striping #lfs setstripe -C $((osts * 4)) $io500_workdir/ior-hard || # lfs setstripe -c -1 $io500_workdir/ior-hard # Try to use DoM if available, otherwise use default for small files #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-easy || true #DoM? #lfs setstripe -E 64k -L mdt $io500_workdir/mdtest-hard || true #DoM? } # ***** YOU SHOULD NOT EDIT ANYTHING BELOW THIS LINE ***** set -eo pipefail # better error handling io500_ini="${1:-""}" if [[ -z "$io500_ini" ]]; then echo "error: ini file must be specified. usage: $0" exit 1 fi if [[ ! -s "$io500_ini" ]]; then echo "error: ini file '$io500_ini' not found or empty" exit 2 fi function get_ini_section_param() { local section="$1" local param="$2" local inside=false while read LINE; do LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE) $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue ! $inside && continue #echo $LINE | awk -F = "/^$param/ { print \$2 }" if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/" return fi done < $io500_ini echo "" } function get_ini_global_param() { local param="$1" local default="$2" local val val=$(get_ini_section_param global $param | sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/') echo "${val:-$default}" } function run_benchmarks { $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp } create_tarball() { local sourcedir=$(dirname $io500_resultdir) local fname=$(basename ${io500_resultdir}) local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz cp -v $0 $io500_ini $io500_resultdir tar czf $tarball -C $sourcedir $fname echo "Created result tarball $tarball" } function main { # These commands extract the 'datadir' and 'resultdir' from .ini file timestamp=$(date +%Y.%m.%d-%H.%M.%S) # create a uniquifier [ $(get_ini_global_param timestamp-datadir True) != "False" ] && ts="$timestamp" || ts="io500" # working directory where the test files will be created export io500_workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts [ $(get_ini_global_param timestamp-resultdir True) != "False" ] && ts="$timestamp" || ts="io500" # the directory where the output results will be kept export io500_resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts setup_directories setup_paths run_benchmarks if [[ ! -s "system-information.txt" ]]; then echo "Warning: please create a 'system-information.txt' description by" echo "copying the information from https://vi4io.org/io500-info-creator/" else cp "system-information.txt" $io500_resultdir fi create_tarball } main - ior-easy-read
-
WARNING: fsync() not supported in selected backend. Using value of 0. IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O Began : Mon Oct 26 08:27:03 2020 Command line : ./ior -C -Q 1 -g -G 271 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-easy.stonewall -t 1M -b 9920000m -r -R -a IME Machine : Linux flash-42 TestID : 0 StartTime : Mon Oct 26 08:27:03 2020 Path : /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy FS : 10835.6 TiB Used FS: 12.7% Inodes: 1144.4 Mi Used Inodes: 28.9% Options: api : IME apiVersion : 1.4.1 (141015) test filename : /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy access : single-shared-file type : independent segments : 1 ordering in a file : sequential ordering inter file : constant task offset task offset : 1 nodes : 10 tasks : 280 clients per node : 28 repetitions : 1 xfersize : 1 MiB blocksize : 9.46 TiB aggregate filesize : 2648.93 TiB Results: access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 2912524697600000 WARNING: Stat() of aggregate file size = 2902345410150400 WARNING: Using actual aggregate bytes moved = 62324211712000 read 191757 191782 0.000031 10158080000 1024.00 0.038910 309.92 0.005796 309.96 0
- ior-easy-write
-
WARNING: fsync() not supported in selected backend. Using value of 0. IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O Began : Mon Oct 26 08:05:42 2020 Command line : ./ior -C -Q 1 -g -G 271 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-easy.stonewall -t 1M -b 9920000m -w -D 300 -O stoneWallingWearOut=1 -a IME Machine : Linux flash-42 TestID : 0 StartTime : Mon Oct 26 08:05:45 2020 WARNING: Backend returned error during statfs. Options: api : IME apiVersion : 1.4.1 (141015) test filename : /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy access : single-shared-file type : independent segments : 1 ordering in a file : sequential ordering inter file : constant task offset task offset : 1 nodes : 10 tasks : 280 clients per node : 28 repetitions : 1 xfersize : 1 MiB blocksize : 9.46 TiB aggregate filesize : 2648.93 TiB stonewallingTime : 300 stoneWallingWearOut : 1 Results: access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ---- stonewalling pairs accessed min: 206542 max: 212275 -- min data: 201.7 GiB mean data: 204.2 GiB time: 300.0s WARNING: Expected aggregate file size = 2912524697600000 WARNING: Stat() of aggregate file size = 2902345410150400 WARNING: Using actual aggregate bytes moved = 62324211712000 WARNING: Maybe caused by deadlineForStonewalling write 194209 194275 0.000031 10158080000 1024.00 0.035703 305.94 0.068537 306.05 0
- ior-hard-read
-
WARNING: fsync() not supported in selected backend. Using value of 0. IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O Began : Mon Oct 26 08:32:55 2020 Command line : ./ior -C -Q 1 -g -G 27 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a IME Machine : Linux flash-42 TestID : 0 StartTime : Mon Oct 26 08:32:55 2020 Path : /flash/datafiles/2020.10.26-08.05.37/ior-hard/file FS : 10835.6 TiB Used FS: 12.7% Inodes: 1144.4 Mi Used Inodes: 28.9% Options: api : IME apiVersion : 1.4.1 (141015) test filename : /flash/datafiles/2020.10.26-08.05.37/ior-hard/file access : single-shared-file type : independent segments : 10000000 ordering in a file : sequential ordering inter file : constant task offset task offset : 1 nodes : 10 tasks : 280 clients per node : 28 repetitions : 1 xfersize : 47008 bytes blocksize : 47008 bytes aggregate filesize : 119.71 TiB Results: access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ---- stonewalling pairs accessed min: 4851523 max: 4851523 -- min data: 212.4 GiB mean data: 212.4 GiB time: 415.9s WARNING: Expected aggregate file size = 131622400000000 WARNING: Stat() of aggregate file size = 63856910091520 WARNING: Using actual aggregate bytes moved = 63856910091520 read 146329 3264376 416.12 45.91 45.91 0.003747 416.14 0.040449 416.18 0
- ior-hard-write
-
WARNING: fsync() not supported in selected backend. Using value of 0. IOR-3.4.0+dev: MPI Coordinated Test of Parallel I/O Began : Mon Oct 26 08:16:01 2020 Command line : ./ior -C -Q 1 -g -G 27 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a IME Machine : Linux flash-42 TestID : 0 StartTime : Mon Oct 26 08:16:01 2020 WARNING: Backend returned error during statfs. Options: api : IME apiVersion : 1.4.1 (141015) test filename : /flash/datafiles/2020.10.26-08.05.37/ior-hard/file access : single-shared-file type : independent segments : 10000000 ordering in a file : sequential ordering inter file : constant task offset task offset : 1 nodes : 10 tasks : 280 clients per node : 28 repetitions : 1 xfersize : 47008 bytes blocksize : 47008 bytes aggregate filesize : 119.71 TiB stonewallingTime : 300 stoneWallingWearOut : 1 Results: access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ---- stonewalling pairs accessed min: 4501129 max: 4851523 -- min data: 197.1 GiB mean data: 204.9 GiB time: 300.0s WARNING: Expected aggregate file size = 131622400000000 WARNING: Stat() of aggregate file size = 63856910091520 WARNING: Using actual aggregate bytes moved = 63856910091520 WARNING: Maybe caused by deadlineForStonewalling write 194609 4342120 310.53 45.91 45.91 0.034108 312.85 0.047459 312.93 0
- mdtest-easy-delete
-
-- started at 10/26/2020 08:40:06 -- mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s) Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-easy' '-x' './results/2020.10.26-08.05.37/mdtest-easy.stonewall' '-r' '-a' 'POSIX' Path: /flash/datafiles/2020.10.26-08.05.37 FS: 1222.6 TiB Used FS: 10.7% Inodes: 1144.4 Mi Used Inodes: 28.9% Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000 V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase. 280 tasks, 280000000 files SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 24087.462 24087.398 24087.421 0.016 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 1.000 1.000 1.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 442.550 442.549 442.550 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 1.000 1.000 1.000 0.000 -- finished at 10/26/2020 08:47:30 --
- mdtest-easy-stat
-
-- started at 10/26/2020 08:32:13 -- mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s) Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-easy' '-x' './results/2020.10.26-08.05.37/mdtest-easy.stonewall' '-T' '-a' 'IME' Path: /flash/datafiles/2020.10.26-08.05.37 FS: 10835.6 TiB Used FS: 12.7% Inodes: 1144.4 Mi Used Inodes: 28.9% Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000 V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase. 280 tasks, 280000000 files SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 259212.035 259211.333 259211.663 0.160 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 41.124 41.124 41.124 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 -- finished at 10/26/2020 08:32:55 --
- mdtest-easy-write
-
-- started at 10/26/2020 08:10:53 -- mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s) Command line used: ./mdtest '-n' '1000000' '-u' '-L' '-F' '-P' '-N' '1' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-easy' '-x' './results/2020.10.26-08.05.37/mdtest-easy.stonewall' '-C' '-Y' '-W' '300' '-a' 'IME' Path: /flash/datafiles/2020.10.26-08.05.37 FS: 10835.6 TiB Used FS: 12.7% Inodes: 1144.4 Mi Used Inodes: 27.9% Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000 V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase. 280 tasks, 280000000 files Continue stonewall hit min: 36683 max: 38071 avg: 37265.1 SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 34650.332 34650.239 34650.273 0.024 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 File create (stonewall) : NA NA 34760.625 NA Tree creation : 185.695 185.695 185.695 0.000 Tree removal : 0.000 0.000 0.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 307.642 307.641 307.642 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 File create (stonewall) : NA NA 300.174 NA Tree creation : 0.005 0.005 0.005 0.000 Tree removal : 0.000 0.000 0.000 0.000 -- finished at 10/26/2020 08:16:01 --
- mdtest-hard-delete
-
-- started at 10/26/2020 08:47:42 -- mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s) Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-hard' '-x' './results/2020.10.26-08.05.37/mdtest-hard.stonewall' '-r' '-a' 'IME' Path: /flash/datafiles/2020.10.26-08.05.37 FS: 10835.6 TiB Used FS: 12.7% Inodes: 1144.4 Mi Used Inodes: 28.1% Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000 V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase. 280 tasks, 280000000 files SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 20565.437 20565.082 20565.123 0.029 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 17.441 17.441 17.441 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 104.212 104.210 104.211 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.057 0.057 0.057 0.000 -- finished at 10/26/2020 08:49:26 --
- mdtest-hard-read
-
-- started at 10/26/2020 08:47:30 -- mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s) Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-hard' '-x' './results/2020.10.26-08.05.37/mdtest-hard.stonewall' '-E' '-X' '-a' 'IME' Path: /flash/datafiles/2020.10.26-08.05.37 FS: 10835.6 TiB Used FS: 12.7% Inodes: 1144.4 Mi Used Inodes: 28.1% Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000 V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase. 280 tasks, 280000000 files SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 174357.878 174344.497 174348.117 5.548 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 12.292 12.292 12.292 0.000 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 -- finished at 10/26/2020 08:47:42 --
- mdtest-hard-stat
-
-- started at 10/26/2020 08:39:51 -- mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s) Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-hard' '-x' './results/2020.10.26-08.05.37/mdtest-hard.stonewall' '-T' '-a' 'IME' Path: /flash/datafiles/2020.10.26-08.05.37 FS: 10835.6 TiB Used FS: 12.7% Inodes: 1144.4 Mi Used Inodes: 28.9% Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000 V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase. 280 tasks, 280000000 files SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 144316.009 144315.395 144315.651 0.117 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 14.850 14.850 14.850 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 -- finished at 10/26/2020 08:40:06 --
- mdtest-hard-write
-
-- started at 10/26/2020 08:21:14 -- mdtest-3.4.0+dev was launched with 280 total task(s) on 10 node(s) Command line used: ./mdtest '-n' '1000000' '-t' '-w' '3901' '-e' '3901' '-P' '-N' '1' '-F' '-d' '/flash/datafiles/2020.10.26-08.05.37/mdtest-hard' '-x' './results/2020.10.26-08.05.37/mdtest-hard.stonewall' '-C' '-Y' '-W' '300' '-a' 'POSIX' Path: /flash/datafiles/2020.10.26-08.05.37 FS: 1222.6 TiB Used FS: 10.7% Inodes: 1144.4 Mi Used Inodes: 28.8% Nodemap: 1000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000100000000010000000001000000000 V-0: Rank 0 Line 2216 Shifting ranks by 1 for each phase. 280 tasks, 280000000 files Continue stonewall hit min: 7364 max: 7654 avg: 7561.3 SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 6542.175 6542.157 6542.164 0.004 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 File create (stonewall) : NA NA 7055.864 NA Tree creation : 645.575 645.575 645.575 0.000 Tree removal : 0.000 0.000 0.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 327.586 327.585 327.586 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 File create (stonewall) : NA NA 300.059 NA Tree creation : 0.002 0.002 0.002 0.000 Tree removal : 0.000 0.000 0.000 0.000 -- finished at 10/26/2020 08:26:44 --
- result
-
version = io500-sc20-6-gf1c40e1e4342 config-hash = 1C74EB35 result-dir = ./results/2020.10.26-08.05.37 ; START 2020-10-26 08:05:41 [ior-easy-write] t_start = 2020-10-26 08:05:42 exe = ./ior -C -Q 1 -g -G 271 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-easy.stonewall -t 1M -b 9920000m -w -D 300 -O stoneWallingWearOut=1 -a IME throughput-stonewall = 52319.40 score = 189.657067 t_delta = 310.7931 t_end = 2020-10-26 08:10:53 [mdtest-easy-write] t_start = 2020-10-26 08:10:53 exe = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /flash/datafiles/2020.10.26-08.05.37/mdtest-easy -x ./results/2020.10.26-08.05.37/mdtest-easy.stonewall -C -Y -W 300 -a IME rate-stonewall = 34.760625 score = 34.650264 t_delta = 307.8700 t_end = 2020-10-26 08:16:01 [timestamp] t_start = 2020-10-26 08:16:01 t_delta = 0.0032 t_end = 2020-10-26 08:16:01 [ior-hard-write] t_start = 2020-10-26 08:16:01 exe = ./ior -C -Q 1 -g -G 27 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -w -D 300 -a IME throughput-stonewall = 51337.45 score = 190.047767 t_delta = 313.0129 t_end = 2020-10-26 08:21:14 [mdtest-hard-write] t_start = 2020-10-26 08:21:14 exe = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /flash/datafiles/2020.10.26-08.05.37/mdtest-hard -x ./results/2020.10.26-08.05.37/mdtest-hard.stonewall -C -Y -W 300 -a POSIX rate-stonewall = 7.055864 score = 6.542162 t_delta = 330.1941 t_end = 2020-10-26 08:26:44 [find] t_start = 2020-10-26 08:26:44 exe = ./pfind /flash/datafiles/2020.10.26-08.05.37 -newer ./results/2020.10.26-08.05.37/timestampfile -size 3901c -name *01* -C -q 50000 nproc = 140 found = 55896 total-files = 12803002 score = 665.439184 t_delta = 19.4988 t_end = 2020-10-26 08:27:03 [ior-easy-read] t_start = 2020-10-26 08:27:03 exe = ./ior -C -Q 1 -g -G 271 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-easy/ior_file_easy -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-easy.stonewall -t 1M -b 9920000m -r -R -a IME score = 187.262238 t_delta = 310.1218 t_end = 2020-10-26 08:32:13 [mdtest-easy-stat] t_start = 2020-10-26 08:32:13 exe = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /flash/datafiles/2020.10.26-08.05.37/mdtest-easy -x ./results/2020.10.26-08.05.37/mdtest-easy.stonewall -T -a IME score = 259.211637 t_delta = 41.1936 t_end = 2020-10-26 08:32:55 [ior-hard-read] t_start = 2020-10-26 08:32:55 exe = ./ior -C -Q 1 -g -G 27 -k -e -o /flash/datafiles/2020.10.26-08.05.37/ior-hard/file -O stoneWallingStatusFile=./results/2020.10.26-08.05.37/ior-hard.stonewall -O stoneWallingWearOut=1 -t 47008 -b 47008 -s 10000000 -r -R -a IME score = 142.899363 t_delta = 416.3154 t_end = 2020-10-26 08:39:51 [mdtest-hard-stat] t_start = 2020-10-26 08:39:51 exe = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /flash/datafiles/2020.10.26-08.05.37/mdtest-hard -x ./results/2020.10.26-08.05.37/mdtest-hard.stonewall -T -a IME score = 144.315536 t_delta = 14.9572 t_end = 2020-10-26 08:40:06 [mdtest-easy-delete] t_start = 2020-10-26 08:40:06 exe = ./mdtest -n 1000000 -u -L -F -P -N 1 -d /flash/datafiles/2020.10.26-08.05.37/mdtest-easy -x ./results/2020.10.26-08.05.37/mdtest-easy.stonewall -r -a POSIX score = 24.087414 t_delta = 443.9772 t_end = 2020-10-26 08:47:30 [mdtest-hard-read] t_start = 2020-10-26 08:47:30 exe = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /flash/datafiles/2020.10.26-08.05.37/mdtest-hard -x ./results/2020.10.26-08.05.37/mdtest-hard.stonewall -E -X -a IME score = 174.357658 t_delta = 12.3219 t_end = 2020-10-26 08:47:42 [mdtest-hard-delete] t_start = 2020-10-26 08:47:42 exe = ./mdtest -n 1000000 -t -w 3901 -e 3901 -P -N 1 -F -d /flash/datafiles/2020.10.26-08.05.37/mdtest-hard -x ./results/2020.10.26-08.05.37/mdtest-hard.stonewall -r -a IME score = 20.565134 t_delta = 104.3508 t_end = 2020-10-26 08:49:27 [SCORE] MD = 68.546190 BW = 176.229284 SCORE = 109.908352 hash = 894133ED ; END 2020-10-26 08:49:27
- result_summary
-
IO500 version io500-sc20-6-gf1c40e1e4342 [RESULT] ior-easy-write 189.657067 GiB/s : time 310.793 seconds [RESULT] mdtest-easy-write 34.650264 kIOPS : time 307.870 seconds [RESULT] ior-hard-write 190.047767 GiB/s : time 313.013 seconds [RESULT] mdtest-hard-write 6.542162 kIOPS : time 330.194 seconds [RESULT] find 665.439184 kIOPS : time 19.499 seconds [RESULT] ior-easy-read 187.262238 GiB/s : time 310.122 seconds [RESULT] mdtest-easy-stat 259.211637 kIOPS : time 41.194 seconds [RESULT] ior-hard-read 142.899363 GiB/s : time 416.315 seconds [RESULT] mdtest-hard-stat 144.315536 kIOPS : time 14.957 seconds [RESULT] mdtest-easy-delete 24.087414 kIOPS : time 443.977 seconds [RESULT] mdtest-hard-read 174.357658 kIOPS : time 12.322 seconds [RESULT] mdtest-hard-delete 20.565134 kIOPS : time 104.351 seconds [SCORE] Bandwidth 176.229284 GiB/s : IOPS 68.546190 kiops : TOTAL 109.908352