System:  c0065.ofp
filesystem_utilization=172.30.140.1@o2ib:172.30.140.2@o2ib:/work 25337068386520 5326489272332 19754336180000  22% /work
date=2018.10.22-14.52.58
io500_cleanup_workdir=False
io500_find_cmd_args='-s 0 -r /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/pfind_results'
io500_find_cmd=/work/xg17i000/x10007/io-500-dev/bin/pfind
io500_find_mpi=True
io500_info_data_storage_type=HDD
io500_info_filesystem=Lustre
io500_info_filesystem_vendor=DDN
io500_info_filesystem_version=2.7.21.3.ddn21
io500_info_institute_name=JCAHPC
io500_info_metadata_storage_type=SSD
io500_info_num_client_nodes=256
io500_info_num_data_server_nodes=40
io500_info_num_data_storage_devices=4100
io500_info_num_metadata_server_nodes=10
io500_info_num_metadata_storage_devices=44
io500_info_procs_per_node=32
io500_info_storage_age_in_months=22
io500_info_storage_install_date=12/16
io500_info_storage_interface=SAS
io500_info_storage_network=OmniPath
io500_info_system_name=Oakforest-PACS
io500_info_whatever=WhateverElseYouThinkRelevant
io500_ior_cmd=/work/xg17i000/x10007/io-500-dev/bin/ior
io500_ior_easy_params='-t 2048k -b 4000m -F'
io500_ior_easy_size=4000
io500_ior_hard_other_options=
io500_ior_hard_writes_per_proc=2300
io500_mdreal_cmd=/work/xg17i000/x10007/io-500-dev/bin/md-real-io
io500_mdreal_params='-P=5000 -I=1000'
io500_mdtest_cmd=/work/xg17i000/x10007/io-500-dev/bin/mdtest
io500_mdtest_easy_files_per_proc=2500
io500_mdtest_easy_params='-u -L'
io500_mdtest_hard_files_per_proc=1600
io500_mdtest_hard_other_options='-I 800'
io500_mpiargs='-n 8192'
io500_mpirun=mpiexec.hydra
io500_result_dir=/work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58
io500_run_find=True
io500_run_ior_easy_read=True
io500_run_ior_easy=True
io500_run_ior_hard_read=True
io500_run_ior_hard=True
io500_run_md_easy_delete=True
io500_run_md_easy_stat=True
io500_run_md_easy=True
io500_run_md_hard_delete=True
io500_run_md_hard_read=True
io500_run_md_hard_stat=True
io500_run_md_hard=True
io500_run_mdreal=False
io500_stonewall_timer=0
io500_workdir=/work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58
Running the IO500 Benchmark now
[Creating] directories

[Starting] ior_easy_write
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 2048k -b 4000m -F -o /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/ior_easy/ior_file_easy -O stoneWallingStatusFile=/work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/ior_easy/stonewall -O stoneWallingWearOut=1 -D 0 
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/ior_easy_write.txt.
[RESULT] BW   phase 1            ior_easy_write               97.078 GB/s : time 329.63 seconds

[Starting] mdtest_easy_write
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/mdtest -C -F -d /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_easy -n 2500 -u -L -x /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_easy-stonewall -W 0
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/mdtest_easy_write.txt.
[RESULT] IOPS phase 1         mdtest_easy_write               58.458 kiops : time 360.00 seconds

[Starting] ior_hard_write
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 2300  -o /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/ior_hard/IOR_file -O stoneWallingStatusFile=/work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/ior_hard/stonewall -O stoneWallingWearOut=1 -D 0
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/ior_hard_write.txt.
[RESULT] BW   phase 2            ior_hard_write                2.623 GB/s : time 314.44 seconds

[Starting] mdtest_hard_write
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/mdtest -C -t -F -w 3901 -e 3901 -d /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_hard -n 1600 -x /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_hard-stonewall -I 800 -W 0
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/mdtest_hard_write.txt.
[RESULT] IOPS phase 2         mdtest_hard_write               44.604 kiops : time 303.16 seconds

[Starting] find
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/pfind /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58 -newer /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/timestampfile -size 3901c -name *01* -s 0 -r /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/pfind_results
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/find.txt.
[FIND] MATCHED 0/33595396 in 39.5621 seconds
[RESULT] IOPS phase 3                      find              849.180 kiops : time  39.56 seconds

[Starting] ior_easy_read
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 2048k -b 4000m -F -o /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/ior_easy/ior_file_easy -O stoneWallingStatusFile=/work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/ior_easy/stonewall
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/ior_easy_read.txt.
[RESULT] BW   phase 3             ior_easy_read              121.771 GB/s : time 262.79 seconds

[Starting] mdtest_easy_stat
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/mdtest -T -F -d /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_easy -n 2500 -u -L -x /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_easy-stonewall
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/mdtest_easy_stat.txt.
[RESULT] IOPS phase 4          mdtest_easy_stat              152.553 kiops : time 143.41 seconds

[Starting] ior_hard_read
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 2300  -o /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/ior_hard/IOR_file -O stoneWallingStatusFile=/work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/ior_hard/stonewall
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/ior_hard_read.txt.
[RESULT] BW   phase 4             ior_hard_read                6.786 GB/s : time 121.55 seconds

[Starting] mdtest_hard_stat
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/mdtest -T -t -F -w 3901 -e 3901 -d /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_hard -n 1600 -x /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_hard-stonewall -I 800
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/mdtest_hard_stat.txt.
[RESULT] IOPS phase 5          mdtest_hard_stat              107.715 kiops : time 130.98 seconds

[Starting] mdtest_easy_delete
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/mdtest -r -F -d /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_easy -n 2500 -u -L -x /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_easy-stonewall
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/mdtest_easy_delete.txt.
[RESULT] IOPS phase 6        mdtest_easy_delete               37.232 kiops : time 566.68 seconds

[Starting] mdtest_hard_read
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/mdtest -E -t -F -w 3901 -e 3901 -d /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_hard -n 1600 -x /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_hard-stonewall -I 800
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/mdtest_hard_read.txt.
[RESULT] IOPS phase 7          mdtest_hard_read               81.699 kiops : time 169.91 seconds

[Starting] mdtest_hard_delete
[Exec] mpiexec.hydra -n 8192 /work/xg17i000/x10007/io-500-dev/bin/mdtest -r -t -F -w 3901 -e 3901 -d /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_hard -n 1600 -x /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58/mdt_hard-stonewall -I 800
[Results] in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58/mdtest_hard_delete.txt.
[RESULT] IOPS phase 8        mdtest_hard_delete               43.988 kiops : time 307.85 seconds

[Skipping] mdreal

[Leaving] datafiles in /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58
[Summary] Results files in /work/xg17i000/x10007/io-500-dev/results/2018.10.22-14.52.58
[Summary] Data files in /work/xg17i000/x10007/io-500-dev/datafiles/io500.2018.10.22-14.52.58
[RESULT] BW   phase 1            ior_easy_write               97.078 GB/s : time 329.63 seconds
[RESULT] BW   phase 2            ior_hard_write                2.623 GB/s : time 314.44 seconds
[RESULT] BW   phase 3             ior_easy_read              121.771 GB/s : time 262.79 seconds
[RESULT] BW   phase 4             ior_hard_read                6.786 GB/s : time 121.55 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               58.458 kiops : time 360.00 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               44.604 kiops : time 303.16 seconds
[RESULT] IOPS phase 3                      find              849.180 kiops : time  39.56 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat              152.553 kiops : time 143.41 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat              107.715 kiops : time 130.98 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               37.232 kiops : time 566.68 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               81.699 kiops : time 169.91 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               43.988 kiops : time 307.85 seconds
[SCORE] Bandwidth 21.4184 GB/s : IOPS 91.3951 kiops : TOTAL 44.2441