AI400X
| Institution | DDN |
|---|---|
| Client Procs Per Node | |
| Client Operating System | CentOS |
| Client Operating System Version | 8.1 |
| Client Kernel Version | 4.18.0-147.8.1.el8_1.x86_64 |
DATA SERVER
| Storage Type | NVMe |
|---|---|
| Volatile Memory | 3.84TB |
| Storage Interface | NVMe |
| Network | IB-EDR |
| Software Version | 2.13.54 |
| OS Version | CentOS7.7 |
INFORMATION
| Client Nodes | 40 |
|---|---|
| Client Total Procs | 1,280 |
METADATA
| Easy Write | 378.84 kIOP/s |
|---|---|
| Easy Stat | 1,069.15 kIOP/s |
| Easy Delete | 237.28 kIOP/s |
| Hard Write | 131.45 kIOP/s |
| Hard Read | 350.85 kIOP/s |
| Hard Stat | 1,108.62 kIOP/s |
| Hard Delete | 99.59 kIOP/s |
Submitted Files
- io500
-
#!/bin/bash #SBATCH --job-name=io500-ai400 #SBATCH -P 32n #SBATCH --nodes=10 #SBATCH --ntasks-per-node=16 #SBATCH -o io_500_out_%j #SBATCH -e io_500_err_%J #SBATCH --dependency=singleton LUSTRE_MGS=10.0.11.224@o2ib10 LUSTRE_MDS=es400nv-vm[1-4],es400xmd-vm[1-4] LUSTRE_OSS=es400nv-vm[1-4],es400xmd-vm[1-4] #LUSTRE_MDS=es400nv-vm[1-4] #LUSTRE_OSS=es400nv-vm[1-4] #LUSTRE_CLIENT=${SLURM_JOB_NODELIST} LUSTRE_CLIENT=ec[01-40] FSNAME=/ai400 MNT=/ai400_0 ROOT=`pwd` #module purge #module load mpi/gcc/openmpi/4.0.4 export PATH=/usr/mpi/gcc/openmpi-4.0.3rc4/bin:$PATH PDSH="pdsh" # Lustre MDS/OSS Setting $PDSH -w ${LUSTRE_MDS} "echo 128 > /sys/module/mdt/parameters/max_mod_rpcs_per_client" $PDSH -w ${LUSTRE_OSS},${LUSTRE_MDS} "sysctl -w vm.min_free_kbytes=524288" $PDSH -w ${LUSTRE_OSS} lctl set_param \ osd-ldiskfs.*.read_cache_enable=0 \ osd-ldiskfs.*.writethrough_cache_enable=0 \ obdfilter.*.brw_size=16 \ obdfilter.*.precreate_batch=1024 # ReMount Lustre Client #$PDSH -w ${LUSTRE_CLIENT} umount -t lustre -a #$PDSH -w ${LUSTRE_CLIENT} mount -t lustre ${LUSTRE_MGS}:${FSNAME} ${MNT} $PDSH -w ${LUSTRE_CLIENT},${LUSTRE_MDS},${LUSTRE_OSS} lctl get_param version sleep 2 # Lustre Client Setting $PDSH -w ${LUSTRE_CLIENT} lctl set_param \ osc.*.max_pages_per_rpc=16M \ osc.*.max_rpcs_in_flight=16 \ osc.*.max_dirty_mb=512 \ osc.*.checksums=0 \ osc.*.short_io_bytes=65536 \ llite.*.max_read_ahead_mb=2048 \ llite.*.max_read_ahead_per_file_mb=16 \ llite.*.max_cached_mb=8192 \ ldlm.namespaces.*.lru_size=0 \ ldlm.namespaces.*.lru_max_age=5000 \ mdc.*.max_rpcs_in_flight=128 \ mdc.*.max_mod_rpcs_in_flight=127 sleep 2 # Cleanup & TRIM to all OSTs $PDSH -w ${LUSTRE_CLIENT} lctl set_param ldlm.namespaces.*.lru_size=clear #$PDSH -w ${LUSTRE_OSS} fstrim -av $PDSH -w ${LUSTRE_MDS},${LUSTRE_OSS} "echo 3 > /proc/sys/vm/drop_caches" $PDSH -w ${LUSTRE_CLIENT} "cpupower frequency-set -g performance" # # INSTRUCTIONS: # This script takes its parameters from the same .ini file as io500 binary. function setup_paths { # Set the paths to the binaries and how to launch MPI jobs. # If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/ io500_ior_cmd=$PWD/bin/ior io500_mdtest_cmd=$PWD/bin/mdtest io500_mdreal_cmd=$PWD/bin/md-real-io #io500_mpirun="/work/tools/mpi/gcc/openmpi/4.0.4/bin/mpirun" io500_mpirun="mpirun" #io500_mpiargs="--bind-to core --map-by node --allow-run-as-root -np 160 -hostfile ./hostfile -npernode 16 singularity.sh --bind /work -B /usr/mpi -B /usr/lib64 -B /sys/class/infiniband_verbs -B /bin -B /sbin -B /etc/libibverbs.d centos8.sif " #io500_mpiargs="--bind-to core --map-by node --allow-run-as-root -np 640 -hostfile ./hostfile -npernode 16 singularity.sh --bind /work -B /usr/mpi -B /usr/lib64 -B /sys/class/infiniband_verbs -B /bin -B /sbin -B /etc/libibverbs.d centos8.sif " io500_mpiargs=" --oversubscribe --bind-to core:overload-allowed --map-by core --allow-run-as-root -np 1280 -hostfile ./hostfile -npernode 32" } function setup_directories { local workdir local resultdir local ts # set directories where benchmark files are created and where the results go # If you want to set up stripe tuning on your output directories or anything # similar, then this is the right place to do it. This creates the output # directories for both the app run and the script run. timestamp=$(date +%Y.%m.%d-%H.%M.%S) # create a uniquifier [ $(get_ini_global_param timestamp-datadir True) != "False" ] && ts="$timestamp" || ts="io500" # directory where the data will be stored workdir=$(get_ini_global_param datadir $PWD/datafiles)/$ts io500_workdir=$workdir-scr [ $(get_ini_global_param timestamp-resultdir True) != "False" ] && ts="$timestamp" || ts="io500" # the directory where the output results will be kept resultdir=$(get_ini_global_param resultdir $PWD/results)/$ts io500_result_dir=$resultdir-scr mkdir -p $workdir-{scr,app} $resultdir-{scr,app} lfs setdirstripe -c 16 -i 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 $workdir-app/mdtest-{easy,hard} lfs setdirstripe -c 16 -i 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 $workdir-scr/mdt_{easy,hard} #lfs setdirstripe -c 8 -i 0,1,2,3,4,5,6,7 $workdir-app/mdtest-{easy,hard} #lfs setdirstripe -c 8 -i 0,1,2,3,4,5,6,7 $workdir-scr/mdt_{easy,hard} #lfs setdirstripe -c 4 -i 0,1,2,3 $workdir-app/mdtest-{easy,hard} #lfs setdirstripe -c 4 -i 0,1,2,3 $workdir-scr/mdt_{easy,hard} lfs setdirstripe -c 16 -D $workdir-app/mdtest-{easy,hard} lfs setdirstripe -c 16 -D $workdir-scr/mdt_{easy,hard} lfs setstripe -L mdt -E 1M $workdir-app/mdtest-{easy,hard} lfs setstripe -L mdt -E 1M $workdir-scr/mdt_{easy,hard} mkdir $workdir-app/ior-hard $workdir-scr/ior_hard lfs setstripe -C 640 -S 16m $workdir-app/ior-hard $workdir-scr/ior_hard } # you should not edit anything below this line set -eo pipefail # better error handling io500_ini="${1:-""}" if [[ -z "$io500_ini" ]]; then echo "error: ini file must be specified. usage: $0" exit 1 fi if [[ ! -s "$io500_ini" ]]; then echo "error: ini file '$io500_ini' not found or empty" exit 2 fi function get_ini_section_param() { local section="$1" local param="$2" local inside=false while read LINE; do LINE=$(sed -e 's/ *#.*//' -e '1s/ *= */=/' <<<$LINE) $inside && [[ "$LINE" =~ "[.*]" ]] && inside=false && break [[ -n "$section" && "$LINE" =~ "[$section]" ]] && inside=true && continue ! $inside && continue #echo $LINE | awk -F = "/^$param/ { print \$2 }" if [[ $(echo $LINE | grep "^$param *=" ) != "" ]] ; then # echo "$section : $param : $inside : $LINE" >> parsed.txt # debugging echo $LINE | sed -e "s/[^=]*=[ \t]*\(.*\)/\1/" return fi done < $io500_ini echo "" } function get_ini_param() { local section="$1" local param="$2" local default="$3" # try and get the most-specific param first, then more generic params val=$(get_ini_section_param $section $param) [ -n "$val" ] || val="$(get_ini_section_param ${section%-*} $param)" [ -n "$val" ] || val="$(get_ini_section_param global $param)" echo "${val:-$default}" | sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/' } function get_ini_run_param() { local section="$1" local default="$2" local val val=$(get_ini_section_param $section noRun) # logic is reversed from "noRun=TRUE" to "run=False" [[ $val = [Tt][Rr][Uu][Ee] ]] && echo "False" || echo "$default" } function get_ini_global_param() { local param="$1" local default="$2" local val val=$(get_ini_section_param global $param | sed -e 's/[Ff][Aa][Ll][Ss][Ee]/False/' -e 's/[Tt][Rr][Uu][Ee]/True/') echo "${val:-$default}" } # does the write phase and enables the subsequent read io500_run_ior_easy="$(get_ini_run_param ior-easy True)" # does the creat phase and enables the subsequent stat io500_run_md_easy="$(get_ini_run_param mdtest-easy True)" # does the write phase and enables the subsequent read io500_run_ior_hard="$(get_ini_run_param ior-hard True)" # does the creat phase and enables the subsequent read io500_run_md_hard="$(get_ini_run_param mdtest-hard True)" io500_run_find="$(get_ini_run_param find True)" io500_run_ior_easy_read="$(get_ini_run_param ior-easy-read True)" io500_run_md_easy_stat="$(get_ini_run_param mdtest-easy-stat True)" io500_run_ior_hard_read="$(get_ini_run_param ior-hard-read True)" io500_run_md_hard_stat="$(get_ini_run_param mdtest-easy-stat True)" io500_run_md_hard_read="$(get_ini_run_param mdtest-easy-stat True)" # turn this off if you want to just run find by itself io500_run_md_easy_delete="$(get_ini_run_param mdtest-easy-delete True)" # turn this off if you want to just run find by itself io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)" io500_run_md_hard_delete="$(get_ini_run_param mdtest-hard-delete True)" io500_run_mdreal="$(get_ini_run_param mdreal False)" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches io500_clean_cache="$(get_ini_global_param drop-caches False)" io500_clean_cache_cmd="$(get_ini_global_param drop-caches-cmd)" io500_cleanup_workdir="$(get_ini_run_param cleanup)" # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort... io500_stonewall_timer=$(get_ini_param debug stonewall-time 300) # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds io500_rules="regular" # to run this benchmark, find and edit each of these functions. Please also # also edit 'extra_description' function to help us collect the required data. function main { setup_directories setup_paths setup_ior_easy # required if you want a complete score setup_ior_hard # required if you want a complete score setup_mdt_easy # required if you want a complete score setup_mdt_hard # required if you want a complete score setup_find # required if you want a complete score setup_mdreal # optional run_benchmarks if [[ ! -s "system-information.txt" ]]; then echo "Warning: please create a system-information.txt description by" echo "copying the information from https://vi4io.org/io500-info-creator/" else cp "system-information.txt" $io500_result_dir fi create_tarball } function setup_ior_easy { local params io500_ior_easy_size=$(get_ini_param ior-easy blockSize 9920000m | tr -d m) val=$(get_ini_param ior-easy API POSIX) [ -n "$val" ] && params+=" -a $val" val="$(get_ini_param ior-easy transferSize)" [ -n "$val" ] && params+=" -t $val" val="$(get_ini_param ior-easy hintsFileName)" [ -n "$val" ] && params+=" -U $val" val="$(get_ini_param ior-easy posix.odirect)" [ "$val" = "True" ] && params+=" --posix.odirect" val="$(get_ini_param ior-easy verbosity)" if [ -n "$val" ]; then for i in $(seq $val); do params+=" -v" done fi io500_ior_easy_params="$params" echo -n "" } function setup_mdt_easy { io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves val=$(get_ini_param mdtest-easy n 1000000) [ -n "$val" ] && io500_mdtest_easy_files_per_proc="$val" val=$(get_ini_param mdtest-easy API POSIX) [ -n "$val" ] && io500_mdtest_easy_params+=" -a $val" val=$(get_ini_param mdtest-easy posix.odirect) [ "$val" = "True" ] && io500_mdtest_easy_params+=" --posix.odirect" echo -n "" } function setup_ior_hard { local params io500_ior_hard_api=$(get_ini_param ior-hard API POSIX) io500_ior_hard_writes_per_proc="$(get_ini_param ior-hard segmentCount 10000000)" val="$(get_ini_param ior-hard hintsFileName)" [ -n "$val" ] && params+=" -U $val" val="$(get_ini_param ior-hard posix.odirect)" [ "$val" = "True" ] && params+=" --posix.odirect" val="$(get_ini_param ior-easy verbosity)" if [ -n "$val" ]; then for i in $(seq $val); do params+=" -v" done fi io500_ior_hard_api_specific_options="$params" echo -n "" } function setup_mdt_hard { val=$(get_ini_param mdtest-hard n 1000000) [ -n "$val" ] && io500_mdtest_hard_files_per_proc="$val" io500_mdtest_hard_api="$(get_ini_param mdtest-hard API POSIX)" io500_mdtest_hard_api_specific_options="" echo -n "" } function setup_find { val="$(get_ini_param find external-script)" [ -z "$val" ] && io500_find_mpi="True" && io500_find_cmd="$PWD/bin/pfind" || io500_find_cmd="$val" # uses stonewalling, run pfind io500_find_cmd_args="$(get_ini_param find external-extra-args)" echo -n "" } function setup_mdreal { echo -n "" } function run_benchmarks { local app_first=$((RANDOM % 100)) local app_rc=0 # run the app and C version in random order to try and avoid bias (( app_first >= 50 )) && $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp || app_rc=$? # Important: source the io500_fixed.sh script. Do not change it. If you # discover a need to change it, please email the mailing list to discuss. source build/io500-dev/utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt (( $app_first >= 50 )) && return $app_rc echo "The io500.sh was run" echo echo "Running the C version of the benchmark now" # run the app and C version in random order to try and avoid bias $io500_mpirun $io500_mpiargs $PWD/io500 $io500_ini --timestamp $timestamp } create_tarball() { local sourcedir=$(dirname $io500_result_dir) local fname=$(basename ${io500_result_dir%-scr}) local tarball=$sourcedir/io500-$HOSTNAME-$fname.tgz cp -v $0 $io500_ini $io500_result_dir tar czf $tarball -C $sourcedir $fname-{app,scr} echo "Created result tarball $tarball" } # Information fields; these provide information about your system hardware # Use https://vi4io.org/io500-info-creator/ to generate information about # your hardware that you want to include publicly! function extra_description { # UPDATE: Please add your information into "system-information.txt" pasting the output of the info-creator # EXAMPLE: # io500_info_system_name='xxx' # DO NOT ADD IT HERE : } main - ior_easy_read
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O Began : Tue Jul 14 15:20:44 2020 Command line : /work/BMLab/Lustre/io500_isc20/io500-app/bin/ior -r -R -a POSIX -t 16m --posix.odirect -b 70000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/stonewall Machine : Linux ec01 TestID : 0 StartTime : Tue Jul 14 15:20:44 2020 Path : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy FS : 94.9 TiB Used FS: 33.1% Inodes: 926.5 Mi Used Inodes: 17.1% Options: api : POSIX apiVersion : test filename : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/ior_file_easy access : file-per-process type : independent segments : 1 ordering in a file : sequential ordering inter file : constant task offset task offset : 1 nodes : 40 tasks : 1280 clients per node : 32 repetitions : 1 xfersize : 16 MiB blocksize : 68.36 GiB aggregate filesize : 85.45 TiB Results: access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 93952409600000. WARNING: Stat() of aggregate file size = 31159987732480. WARNING: Using actual aggregate bytes moved = 31159987732480. read 91027 5690 0.074611 71680000 16384 0.026875 326.42 0.010509 326.46 0 Max Read: 91026.62 MiB/sec (95448.33 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum read 91026.62 91026.62 91026.62 0.00 5689.16 5689.16 5689.16 0.00 326.45923 NA NA 0 1280 32 1 1 1 1 0 0 1 73400320000 16777216 29716480.0 POSIX 0 Finished : Tue Jul 14 15:26:11 2020
- ior_easy_write
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O Began : Tue Jul 14 14:55:48 2020 Command line : /work/BMLab/Lustre/io500_isc20/io500-app/bin/ior -w -a POSIX -t 16m --posix.odirect -b 70000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/ior_file_easy -O stoneWallingStatusFile=/ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300 Machine : Linux ec01 TestID : 0 StartTime : Tue Jul 14 14:55:48 2020 Path : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy FS : 94.9 TiB Used FS: 0.0% Inodes: 768.0 Mi Used Inodes: 0.0% Options: api : POSIX apiVersion : test filename : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_easy/ior_file_easy access : file-per-process type : independent segments : 1 ordering in a file : sequential ordering inter file : constant task offset task offset : 1 nodes : 40 tasks : 1280 clients per node : 32 repetitions : 1 xfersize : 16 MiB blocksize : 68.36 GiB aggregate filesize : 85.45 TiB stonewallingTime : 300 stoneWallingWearOut : 1 Results: access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ---- stonewalling pairs accessed min: 884 max: 1451 -- min data: 13.8 GiB mean data: 17.5 GiB time: 300.2s WARNING: Expected aggregate file size = 93952409600000. WARNING: Stat() of aggregate file size = 31159987732480. WARNING: Using actual aggregate bytes moved = 31159987732480. WARNING: maybe caused by deadlineForStonewalling write 75334 4710 0.088906 71680000 16384 0.093032 394.35 0.020854 394.46 0 Max Write: 75333.76 MiB/sec (78993.18 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum write 75333.76 75333.76 75333.76 0.00 4708.36 4708.36 4708.36 0.00 394.46430 300.23 76421.16 0 1280 32 1 1 1 1 0 0 1 73400320000 16777216 29716480.0 POSIX 0 Finished : Tue Jul 14 15:02:22 2020
- ior_hard_read
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O Began : Tue Jul 14 15:28:12 2020 Command line : /work/BMLab/Lustre/io500_isc20/io500-app/bin/ior -r -R -s 55000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/stonewall Machine : Linux ec01 TestID : 0 StartTime : Tue Jul 14 15:28:12 2020 Path : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard FS : 94.9 TiB Used FS: 33.1% Inodes: 926.5 Mi Used Inodes: 17.1% Options: api : POSIX apiVersion : test filename : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/IOR_file access : single-shared-file type : independent segments : 55000 ordering in a file : sequential ordering inter file : constant task offset task offset : 1 nodes : 40 tasks : 1280 clients per node : 32 repetitions : 1 xfersize : 47008 bytes blocksize : 47008 bytes aggregate filesize : 3.01 TiB Results: access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ---- read 73969 1650594 42.65 45.91 45.91 0.007877 42.65 0.008210 42.67 0 Max Read: 73968.78 MiB/sec (77561.89 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum read 73968.78 73968.78 73968.78 0.00 1649972.06 1649972.06 1649972.06 0.00 42.66739 NA NA 0 1280 32 1 0 1 1 0 0 55000 47008 47008 3156054.8 POSIX 0 Finished : Tue Jul 14 15:28:55 2020
- ior_hard_write
-
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O Began : Tue Jul 14 15:07:51 2020 Command line : /work/BMLab/Lustre/io500_isc20/io500-app/bin/ior -w -s 55000 -a POSIX -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/IOR_file -O stoneWallingStatusFile=/ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300 Machine : Linux ec01 TestID : 0 StartTime : Tue Jul 14 15:07:51 2020 Path : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard FS : 94.9 TiB Used FS: 29.9% Inodes: 884.0 Mi Used Inodes: 13.1% Options: api : POSIX apiVersion : test filename : /ai400/datafiles/2020.07.14-14.55.35-scr/ior_hard/IOR_file access : single-shared-file type : independent segments : 55000 ordering in a file : sequential ordering inter file : constant task offset task offset : 1 nodes : 40 tasks : 1280 clients per node : 32 repetitions : 1 xfersize : 47008 bytes blocksize : 47008 bytes aggregate filesize : 3.01 TiB stonewallingTime : 300 stoneWallingWearOut : 1 Results: access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ---- stonewalling pairs accessed min: 54600 max: 55000 -- min data: 2.4 GiB mean data: 2.4 GiB time: 300.1s write 10438 232842 300.31 45.91 45.91 0.011040 302.35 0.011695 302.37 0 Max Write: 10437.58 MiB/sec (10944.60 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum write 10437.58 10437.58 10437.58 0.00 232824.14 232824.14 232824.14 0.00 302.37415 300.06 10517.97 0 1280 32 1 0 1 1 0 0 55000 47008 47008 3156054.8 POSIX 0 Finished : Tue Jul 14 15:12:55 2020
- mdtest_easy_delete
-
-- started at 07/14/2020 15:29:43 -- mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s) Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-r' '-F' '-P' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy' '-n' '95000' '-u' '-L' '-a' 'POSIX' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy-stonewall' '-N' '1' Path: /ai400/datafiles/2020.07.14-14.55.35-scr FS: 94.9 TiB Used FS: 33.1% Inodes: 926.5 Mi Used Inodes: 17.1% Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 V-0: Rank 0 Line 2166 Shifting ranks by 32 for each phase. 1280 tasks, 121600000 files SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 237276.720 237276.674 237276.700 0.010 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.694 0.694 0.694 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 512.482 512.482 512.482 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 1.442 1.442 1.442 0.000 -- finished at 07/14/2020 15:38:22 --
- mdtest_easy_stat
-
-- started at 07/14/2020 15:26:15 -- mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s) Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-T' '-F' '-P' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy' '-n' '95000' '-u' '-L' '-a' 'POSIX' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy-stonewall' '-N' '1' Path: /ai400/datafiles/2020.07.14-14.55.35-scr FS: 94.9 TiB Used FS: 33.1% Inodes: 926.5 Mi Used Inodes: 17.1% Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 V-0: Rank 0 Line 2166 Shifting ranks by 32 for each phase. 1280 tasks, 121600000 files SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 1069149.529 1069148.888 1069149.207 0.179 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 113.735 113.735 113.735 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 -- finished at 07/14/2020 15:28:08 --
- mdtest_easy_write
-
-- started at 07/14/2020 15:02:26 -- mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s) Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-Y' '-C' '-F' '-P' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy' '-n' '95000' '-u' '-L' '-a' 'POSIX' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_easy-stonewall' '-N' '1' '-W' '300' Path: /ai400/datafiles/2020.07.14-14.55.35-scr FS: 94.9 TiB Used FS: 29.9% Inodes: 768.0 Mi Used Inodes: 0.0% Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 V-0: Rank 0 Line 2166 Shifting ranks by 32 for each phase. 1280 tasks, 121600000 files [1594706700.740772] [ec31:23151:0] tcp_cm.c:604 UCX WARN tcp_iface 0x100e2e0: connection establishment for socket fd 58 from
to 10.128.11.31:44321 was unsuccessful [1594706731.932133] [ec31:23151:1] sock.c:344 UCX ERROR recv(fd=55) failed: Connection reset by peer [1594706731.932170] [ec31:23151:1] sock.c:344 UCX ERROR recv(fd=56) failed: Connection reset by peer [1594706731.932180] [ec31:23151:1] sock.c:344 UCX ERROR recv(fd=57) failed: Connection reset by peer [1594706731.932188] [ec31:23151:1] sock.c:344 UCX ERROR recv(fd=54) failed: Connection reset by peer [1594706731.932196] [ec31:23151:1] sock.c:344 UCX ERROR recv(fd=58) failed: Connection reset by peer [1594706820.116146] [ec31:23151:1] sock.c:344 UCX ERROR recv(fd=54) failed: Connection reset by peer [1594706830.770108] [ec31:23151:1] sock.c:344 UCX ERROR recv(fd=54) failed: Connection reset by peer Continue stonewall hit min: 85758 max: 95000 avg: 94184.3 SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 378835.691 378835.480 378835.575 0.038 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 File create (stonewall) : NA NA 402457.892 NA Tree creation : 2.333 2.333 2.333 0.000 Tree removal : 0.000 0.000 0.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 320.984 320.983 320.984 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 File create (stonewall) : NA NA 299.549 NA Tree creation : 0.429 0.429 0.429 0.000 Tree removal : 0.000 0.000 0.000 0.000 -- finished at 07/14/2020 15:07:48 -- - mdtest_hard_delete
-
-- started at 07/14/2020 15:40:39 -- mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s) Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard' '-n' '200000' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' Path: /ai400/datafiles/2020.07.14-14.55.35-scr FS: 94.9 TiB Used FS: 33.1% Inodes: 810.5 Mi Used Inodes: 5.2% Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 V-0: Rank 0 Line 2166 Shifting ranks by 32 for each phase. 1280 tasks, 256000000 files SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 99585.845 99585.815 99585.828 0.007 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.218 0.218 0.218 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 447.408 447.408 447.408 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 4.588 4.588 4.588 0.000 -- finished at 07/14/2020 15:48:11 --
- mdtest_hard_read
-
-- started at 07/14/2020 15:38:28 -- mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s) Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard' '-n' '200000' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' Path: /ai400/datafiles/2020.07.14-14.55.35-scr FS: 94.9 TiB Used FS: 33.1% Inodes: 810.5 Mi Used Inodes: 5.2% Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 V-0: Rank 0 Line 2166 Shifting ranks by 32 for each phase. 1280 tasks, 256000000 files SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 350849.430 350828.488 350849.251 0.587 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 127.001 126.993 126.993 0.000 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 -- finished at 07/14/2020 15:40:35 --
- mdtest_hard_stat
-
-- started at 07/14/2020 15:28:59 -- mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s) Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard' '-n' '200000' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' Path: /ai400/datafiles/2020.07.14-14.55.35-scr FS: 94.9 TiB Used FS: 33.1% Inodes: 926.5 Mi Used Inodes: 17.1% Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 V-0: Rank 0 Line 2166 Shifting ranks by 32 for each phase. 1280 tasks, 256000000 files SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 1108623.408 1108622.020 1108622.769 0.336 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 0.000 0.000 0.000 0.000 File stat : 40.190 40.190 40.190 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 Tree creation : 0.000 0.000 0.000 0.000 Tree removal : 0.000 0.000 0.000 0.000 -- finished at 07/14/2020 15:29:39 --
- mdtest_hard_write
-
-- started at 07/14/2020 15:12:59 -- mdtest-3.3.0+dev was launched with 1280 total task(s) on 40 node(s) Command line used: /work/BMLab/Lustre/io500_isc20/io500-app/bin/mdtest '-Y' '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard' '-n' '200000' '-x' '/ai400/datafiles/2020.07.14-14.55.35-scr/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300' Path: /ai400/datafiles/2020.07.14-14.55.35-scr FS: 94.9 TiB Used FS: 33.1% Inodes: 884.0 Mi Used Inodes: 13.1% Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 V-0: Rank 0 Line 2166 Shifting ranks by 32 for each phase. 1280 tasks, 256000000 files Continue stonewall hit min: 30291 max: 34809 avg: 32711.1 SUMMARY rate: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 131451.559 131451.518 131451.542 0.009 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 File create (stonewall) : NA NA 139575.822 NA Tree creation : 9.931 9.931 9.931 0.000 Tree removal : 0.000 0.000 0.000 0.000 SUMMARY time: (of 1 iterations) Operation Max Min Mean Std Dev --------- --- --- ---- ------- File creation : 338.950 338.950 338.950 0.000 File stat : 0.000 0.000 0.000 0.000 File read : 0.000 0.000 0.000 0.000 File removal : 0.000 0.000 0.000 0.000 File create (stonewall) : NA NA 299.982 NA Tree creation : 0.101 0.101 0.101 0.000 Tree removal : 0.000 0.000 0.000 0.000 -- finished at 07/14/2020 15:18:38 --
- result_summary
-
[RESULT] BW phase 1 ior_easy_write 73.568 GiB/s : time 394.35 seconds [RESULT] IOPS phase 1 mdtest_easy_write 378.836 kiops : time 320.98 seconds [RESULT] BW phase 2 ior_hard_write 10.193 GiB/s : time 302.35 seconds [RESULT] IOPS phase 2 mdtest_hard_write 131.452 kiops : time 338.95 seconds [RESULT] IOPS phase 3 find 1361.850 kiops : time 122.01 seconds [RESULT] BW phase 3 ior_easy_read 88.894 GiB/s : time 326.42 seconds [RESULT] IOPS phase 4 mdtest_easy_stat 1069.150 kiops : time 113.74 seconds [RESULT] BW phase 4 ior_hard_read 72.235 GiB/s : time 42.65 seconds [RESULT] IOPS phase 5 mdtest_hard_stat 1108.620 kiops : time 40.19 seconds [RESULT] IOPS phase 6 mdtest_easy_delete 237.277 kiops : time 512.48 seconds [RESULT] IOPS phase 7 mdtest_hard_read 350.849 kiops : time 127.00 seconds [RESULT] IOPS phase 8 mdtest_hard_delete 99.586 kiops : time 455.94 seconds [SCORE] Bandwidth 46.8444 GiB/s : IOPS 400.837 kiops : TOTAL 137.029