++ set -euo pipefail
++ export LC_NUMERIC=C
++ LC_NUMERIC=C
++ main
++ check_variables
++ local 'important_vars=io500_workdir io500_ior_easy_params io500_ior_easy_size io500_mdtest_hard_files_per_proc io500_ior_hard_writes_per_proc io500_find_cmd io500_ior_cmd io500_mdtest_cmd io500_mpirun'
++ for V in '$important_vars'
++ '[' -z /dac/fs1 -o /dac/fs1 = xxx ']'
++ for V in '$important_vars'
++ '[' -z '-t -B -a POSIX -t 1m -b 20g -F' -o '-t -B -a POSIX -t 1m -b 20g -F' = xxx ']'
++ for V in '$important_vars'
++ '[' -z 20 -o 20 = xxx ']'
++ for V in '$important_vars'
++ '[' -z 6000 -o 6000 = xxx ']'
++ for V in '$important_vars'
++ '[' -z 18000 -o 18000 = xxx ']'
++ for V in '$important_vars'
++ '[' -z /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/pfind -o /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/pfind = xxx ']'
++ for V in '$important_vars'
++ '[' -z /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/ior -o /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/ior = xxx ']'
++ for V in '$important_vars'
++ '[' -z /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/mdtest -o /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/mdtest = xxx ']'
++ for V in '$important_vars'
++ '[' -z mpirun -o mpirun = xxx ']'
++ return 0
++ output_description
++ extra_description
++ io500_info_system_name=Peta4
++ io500_info_institute_name=UoC-RCS
++ io500_info_storage_age_in_months=xxx
++ io500_info_storage_install_date=xxx
++ io500_info_filesystem=xxx
++ io500_info_filesystem_version=xxx
++ io500_info_filesystem_vendor=xxx
++ io500_info_num_client_nodes=xxx
++ io500_info_procs_per_node=xxx
++ io500_info_num_metadata_server_nodes=xxx
++ io500_info_num_data_server_nodes=xxx
++ io500_info_num_data_storage_devices=xxx
++ io500_info_num_metadata_storage_devices=xxx
++ io500_info_data_storage_type=xxx
++ io500_info_metadata_storage_type=xxx
++ io500_info_storage_network=xxx
++ io500_info_storage_interface=xxx
++ io500_info_whatever=WhateverElseYouThinkRelevant
+++ uname -n
++ echo 'System: ' cpu-e-5
System:  cpu-e-5
+++ tail -1
+++ df /dac/fs1
++ echo 'filesystem_utilization=10.47.18.1@o2ib1:/fs1 297036940032 14347772 282016434436   1% /dac/fs1'
filesystem_utilization=10.47.18.1@o2ib1:/fs1 297036940032 14347772 282016434436   1% /dac/fs1
++ echo date=2018.11.10-19.12.58
date=2018.11.10-19.12.58
++ set -o posix
++ sort
++ set
++ grep '^io500'
io500_cleanup_workdir=/dac/fs1
io500_find_cmd_args='-s 0 -r /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv/pfind_results'
io500_find_cmd=/rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/pfind
io500_find_mpi=True
io500_info_data_storage_type=xxx
io500_info_filesystem_vendor=xxx
io500_info_filesystem_version=xxx
io500_info_filesystem=xxx
io500_info_institute_name=UoC-RCS
io500_info_metadata_storage_type=xxx
io500_info_num_client_nodes=xxx
io500_info_num_data_server_nodes=xxx
io500_info_num_data_storage_devices=xxx
io500_info_num_metadata_server_nodes=xxx
io500_info_num_metadata_storage_devices=xxx
io500_info_procs_per_node=xxx
io500_info_storage_age_in_months=xxx
io500_info_storage_install_date=xxx
io500_info_storage_interface=xxx
io500_info_storage_network=xxx
io500_info_system_name=Peta4
io500_info_whatever=WhateverElseYouThinkRelevant
io500_ior_cmd=/rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/ior
io500_ior_easy_params='-t -B -a POSIX -t 1m -b 20g -F'
io500_ior_easy_size=20
io500_ior_hard_other_options='-a POSIX'
io500_ior_hard_writes_per_proc=18000
io500_mdreal_cmd=/rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/md-real-io
io500_mdreal_params='-P=5000 -I=1000'
io500_mdtest_cmd=/rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/mdtest
io500_mdtest_easy_files_per_proc=6000
io500_mdtest_easy_params='-u -L'
io500_mdtest_hard_files_per_proc=6000
io500_mdtest_hard_other_options=
io500_mpiargs=
io500_mpirun=mpirun
io500_result_dir=/rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv
io500_run_find=False
io500_run_ior_easy=False
io500_run_ior_easy_read=False
io500_run_ior_hard=False
io500_run_ior_hard_read=False
io500_run_md_easy_delete=False
io500_run_md_easy=False
io500_run_md_easy_stat=False
io500_run_md_hard_delete=True
io500_run_md_hard_read=True
io500_run_md_hard_stat=True
io500_run_md_hard=True
io500_run_mdreal=False
io500_stonewall_timer=0
io500_workdir=/dac/fs1
++ core_setup
++ echo 'Running the IO500 Benchmark now'
Running the IO500 Benchmark now
++ echo '[Creating] directories'
[Creating] directories
++ pushd .
++ cd /dac/fs1
++ mkdir -p ior_easy mdt_easy mdt_hard ior_hard mdreal /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv
++ popd
++ timestamp_file=/dac/fs1/timestampfile
++ summary_file=/rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv/result_summary.txt
++ iops1=0
++ iops2=0
++ iops3=0
++ iops4=0
++ iops5=0
++ iops6=0
++ iops7=0
++ iops8=0
++ bw1=0
++ bw2=0
++ bw3=0
++ bw4=0
++ mdt_hard_fsize=3901
++ io500_invalid=
++ ior_easy write
++ phase=ior_easy_write
++ '[' False '!=' True ']'
++ printf '\n[Skipping] ior_easy_write\n'

[Skipping] ior_easy_write
++ return 0
++ mdt_easy write
++ phase=mdtest_easy_write
++ '[' False '!=' True ']'
++ printf '\n[Skipping] mdtest_easy_write\n'

[Skipping] mdtest_easy_write
++ return 0
++ touch /dac/fs1/timestampfile
++ ior_hard write
++ phase=ior_hard_write
++ '[' False '!=' True ']'
++ printf '\n[Skipping] ior_hard_write\n'

[Skipping] ior_hard_write
++ return 0
++ mdt_hard write
++ phase=mdtest_hard_write
++ '[' True '!=' True ']'
++ params_md_hard='-t -F -w 3901 -e 3901 -d /dac/fs1/mdt_hard -n 6000 -x /dac/fs1/mdt_hard-stonewall '
++ result_file=/rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv/mdtest_hard_write.txt
++ [[ write == \w\r\i\t\e ]]
++ startphase mdtest_hard_write
++ echo ''

++ echo '[Starting] mdtest_hard_write'
[Starting] mdtest_hard_write
+++ date +%s.%N
++ start=1541877178.190112041
++ myrun '/rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/mdtest -C -t -F -w 3901 -e 3901 -d /dac/fs1/mdt_hard -n 6000 -x /dac/fs1/mdt_hard-stonewall  -W 0' /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv/mdtest_hard_write.txt
++ command='mpirun  /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/mdtest -C -t -F -w 3901 -e 3901 -d /dac/fs1/mdt_hard -n 6000 -x /dac/fs1/mdt_hard-stonewall  -W 0'
++ echo '[Exec] mpirun  /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/mdtest -C -t -F -w 3901 -e 3901 -d /dac/fs1/mdt_hard -n 6000 -x /dac/fs1/mdt_hard-stonewall  -W 0'
[Exec] mpirun  /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/mdtest -C -t -F -w 3901 -e 3901 -d /dac/fs1/mdt_hard -n 6000 -x /dac/fs1/mdt_hard-stonewall  -W 0
++ mpirun /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/bin/mdtest -C -t -F -w 3901 -e 3901 -d /dac/fs1/mdt_hard -n 6000 -x /dac/fs1/mdt_hard-stonewall -W 0
++ echo '[Results] in /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv/mdtest_hard_write.txt.'
[Results] in /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv/mdtest_hard_write.txt.
++ endphase_check write io500_mdtest_hard_files_per_proc
++ r=0
++ local op=write
++ [[ 0 != \0 ]]
+++ date +%s.%N
++ end=1541877195.367582825
++++ echo '1541877195.367582825 - 1541877178.190112041'
++++ bc
+++ printf %.4f 17.177470784
++ duration=17.1775
++ [[ write == \w\r\i\t\e ]]
+++ printf %.0f 17.1775
++ [[ 17 -lt 300 ]]
++ local var=io500_mdtest_hard_files_per_proc
++ echo '[Warning] This cannot be an official IO-500 score. The phase runtime of 17.1775s is below 300s.'
[Warning] This cannot be an official IO-500 score. The phase runtime of 17.1775s is below 300s.
+++ bc
+++ echo '6000 * 320 / 17.1775'
++ echo '[Warning] Suggest io500_mdtest_hard_files_per_proc=111774'
[Warning] Suggest io500_mdtest_hard_files_per_proc=111774
++ io500_invalid=-invalid
++ invalid=-invalid
+++ get_mdt_iops /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv/mdtest_hard_write.txt creation
+++ file=/rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv/mdtest_hard_write.txt
+++ op=creation
+++ grep '^ *File creation' /rds/project/hpc/rds-hpc-support/ajk203/io-500-dev-2/results.24n-8nv/mdtest_hard_write.txt
+++ awk '{print $4/1000}'
++ iops2=