- io500
-
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.
set -euo pipefail # better error handling
# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True" # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True" # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False" # this one is optional
io500_cleanup_workdir="False" # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_stonewall_timer=300 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...
# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
setup_directories
setup_paths
setup_ior_easy # required if you want a complete score
setup_ior_hard # required if you want a complete score
setup_mdt_easy # required if you want a complete score
setup_mdt_hard # required if you want a complete score
setup_find # required if you want a complete score
# setup_mdreal # optional
run_benchmarks
}
function setup_directories {
# set directories for where the benchmark files are created and where the results will go.
# If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
timestamp=`date +%Y.%m.%d-%H.%M.%S` # create a uniquifier
io500_workdir=/mnt/cephfs/datafiles/io500.$timestamp # directory where the data will be stored
io500_result_dir=/mnt/cephfs/io-500-dev/results/$timestamp # the directory where the output results will be kept
mkdir -p $io500_workdir $io500_result_dir
}
function setup_paths {
# Set the paths to the binaries. If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
io500_ior_cmd=/mnt/cephfs/io-500-dev/bin/ior
io500_mdtest_cmd=/mnt/cephfs/io-500-dev/bin/mdtest
io500_mdreal_cmd=/mnt/cephfs/io-500-dev/bin/md-real-io
io500_mpirun="/usr/mpi/gcc/openmpi-4.0.0rc5/bin/mpirun"
io500_mpiargs="-np 70 --hostfile /mnt/cephfs/io-500-dev/myhosts"
}
function setup_ior_easy {
# io500_ior_easy_size is the amount of data written per rank in MiB units,
# but it can be any number as long as it is somehow used to scale the IOR
# runtime as part of io500_ior_easy_params
# io500_ior_easy_size=513096
io500_ior_easy_size=12000
# 2M writes, 2 GB per proc, file per proc
# io500_ior_easy_params="-t 2048k -b ${io500_ior_easy_size}m -F"
io500_ior_easy_params="-t 4M -b ${io500_ior_easy_size}m -s 4 -F"
}
function setup_mdt_easy {
io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
io500_mdtest_easy_files_per_proc=60000
}
function setup_ior_hard {
io500_ior_hard_writes_per_proc=185192
io500_ior_hard_other_options="" #e.g., -E to keep precreated files using lfs setstripe, or -a MPIIO
}
function setup_mdt_hard {
io500_mdtest_hard_files_per_proc=51614
io500_mdtest_hard_other_options=""
}
function setup_find {
#
# setup the find command. This is an area where innovation is allowed.
# There are three default options provided. One is a serial find, one is python
# parallel version, one is C parallel version. Current default is to use serial.
# But it is very slow. We recommend to either customize or use the C parallel version.
# For GPFS, we recommend to use the provided mmfind wrapper described below.
# Instructions below.
# If a custom approach is used, please provide enough info so others can reproduce.
# the serial version that should run (SLOWLY) without modification
#io500_find_mpi="False"
#io500_find_cmd=$PWD/bin/sfind.sh
#io500_find_cmd_args=""
# a parallel version in C, the -s adds a stonewall
# for a real run, turn -s (stonewall) off or set it at 300 or more
# to prepare this (assuming you've run ./utilities/prepare.sh already):
# > cd build/pfind
# > ./prepare.sh
# > ./compile.sh
# > cp pfind ../../bin/
# If you use io500_find_mpi="True", then this will run with the same
# number of MPI nodes and ranks as the other phases.
# If you prefer another number, and fewer might be better here,
# Then you can set io500_find_mpi to be "False" and write a wrapper
# script for this which sets up MPI as you would like. Then change
# io500_find_cmd to point to your wrapper script.
io500_find_mpi="True"
io500_find_cmd="$PWD/bin/pfind"
# uses stonewalling, run pfind
io500_find_cmd_args="-s $io500_stonewall_timer -r $io500_result_dir/pfind_results"
# for GPFS systems, you should probably use the provided mmfind wrapper
# if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
#io500_find_mpi="False"
#io500_find_cmd="$PWD/bin/mmfind.sh"
#io500_find_cmd_args=""
}
function setup_mdreal {
io500_mdreal_params="-P=5000 -I=1000"
}
function run_benchmarks {
# Important: source the io500_fixed.sh script. Do not change it. If you discover
# a need to change it, please email the mailing list to discuss
source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}
# Add key/value pairs defining your system
# Feel free to add extra ones if you'd like
function extra_description {
# top level info
io500_info_system_name='TigerShark' # e.g. Oakforest-PACS
io500_info_institute_name='SUSE' # e.g. JCAHPC
io500_info_storage_age_in_months='1' # not install date but age since last refresh
io500_info_storage_install_date='11/2018' # MM/YY
io500_info_filesystem='SUSE Enterprise Storage (CephFS)' # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
io500_info_filesystem_version='5.5 (12.2.11)'
io500_info_filesystem_vendor='SUSE'
# client side info
io500_info_num_client_nodes='10'
io500_info_procs_per_node='2'
# server side info
io500_info_num_metadata_server_nodes='3'
io500_info_num_data_server_nodes='10'
io500_info_num_data_storage_devices='120' # if you have 5 data servers, and each has 5 drives, then this number is 25
io500_info_num_metadata_storage_devices='10' # if you have 2 metadata servers, and each has 5 drives, then this number is 10
io500_info_data_storage_type='Intel® SSD D3-S4510' # HDD, SSD, persistent memory, etc, feel free to put specific models
io500_info_metadata_storage_type='Intel® Optane SSD DC P4800X' # HDD, SSD, persistent memory, etc, feel free to put specific models
io500_info_storage_network='ethernet' # infiniband, omnipath, ethernet, etc
io500_info_storage_interface='SATA' # SAS, SATA, NVMe, etc
# miscellaneous
io500_info_whatever='10x Lenovo ThinkSystem SR650 for storage nodes w/1x Intel Xeon Gold 6142, 96GB RAM, 100Gb Ethernet'
}
main
- ior_easy_read
-
[loadgen1:4350][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4348][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4346][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4353][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4347][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4345][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4349][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:04348] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04346] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04350] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04347] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04345] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04353] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04349] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:8892][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10009][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10012][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8890][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10007][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8889][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9113][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8735][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10005][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8894][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9115][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8731][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9945][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8970][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10010][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8887][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9114][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8733][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9947][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8968][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8914][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10006][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8888][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9116][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8738][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9949][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8966][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8916][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10222][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10008][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8891][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9118][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8732][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9946][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8973][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8918][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10218][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8637][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10007] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08890] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:9117][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8736][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9944][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8971][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8917][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10220][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8639][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10012] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08892] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:9120][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8734][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9942][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8969][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8920][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10224][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8641][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10009] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08888] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09113] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08735] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:9943][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8967][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8915][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10217][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8640][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10005] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08887] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09115] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08738] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09947] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08968] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:8913][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10219][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8643][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10010] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08889] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09114] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08731] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09945] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08970] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08914] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10221][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8636][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10006] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08894] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09116] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08732] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09944] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08966] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08916] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10222] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:8638][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10008] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08891] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09118] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08736] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09946] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08973] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08918] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10220] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08639] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09117] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08733] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09942] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08969] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08915] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10218] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08637] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09120] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08734] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09943] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08971] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08920] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10219] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08641] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09949] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08967] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08917] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10221] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08643] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08913] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10217] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08636] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10224] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08640] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08638] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Wed May 29 07:13:42 2019
Command line : /mnt/cephfs/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 4M -b 12000m -s 4 -F -o /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_easy/stonewall
Machine : Linux loadgen1
TestID : 0
StartTime : Wed May 29 07:13:42 2019
Path : /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_easy
FS : 48.3 TiB Used FS: 6.6% Inodes: 8.0 Mi Used Inodes: 100.0%
Options:
api : POSIX
apiVersion :
test filename : /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 4
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 70
clients per node : 7
repetitions : 1
xfersize : 4 MiB
blocksize : 11.72 GiB
aggregate filesize : 3.20 TiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
WARNING: Expected aggregate file size = 3523215360000.
WARNING: Stat() of aggregate file size = 3057270128640.
WARNING: Using actual aggregate bytes moved = 3057270128640.
read 1364.00 12288000 4096 6.75 2130.81 0.156091 2137.56 0
Max Read: 1364.00 MiB/sec (1430.26 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 1364.00 1364.00 1364.00 0.00 341.00 341.00 341.00 0.00 2137.56194 0 70 7 1 1 1 1 0 0 4 12582912000 4194304 2915640.0 POSIX 0
Finished : Wed May 29 07:49:19 2019
- ior_easy_write
-
[loadgen1:3887][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3885][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3889][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3890][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3884][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3888][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3886][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:03885] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03887] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03889] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03888] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03884] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03890] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03886] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:8332][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8336][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9441][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9437][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9376][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8555][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8334][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9443][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9378][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8557][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8337][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9439][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9381][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8559][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8333][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8403][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9438][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9379][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8558][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8331][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8405][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8187][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9681][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8079][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8387][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9442][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9377][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8554][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8335][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8401][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8193][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9677][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8075][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8385][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9440][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9380][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8556][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:08336] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:8407][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8191][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9679][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8077][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8382][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:09443] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:9382][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8560][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:08332] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:8404][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8189][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9682][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8080][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8384][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:09439] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09378] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08559] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08334] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:8402][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8188][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9676][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8078][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8388][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:09441] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09376] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08557] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08333] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:8406][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8192][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9678][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8076][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8386][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:09438] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09381] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08555] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08337] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08407] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:8190][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9680][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8074][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8383][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:09437] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09379] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08558] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08335] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08403] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08187] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09682] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08077] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08382] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09442] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09377] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08554] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08331] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08405] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08193] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09680] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08079] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08384] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09440] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09380] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08556] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08401] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08191] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09676] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08075] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08388] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09382] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08560] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08404] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08189] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09677] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08076] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08387] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08402] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08192] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09678] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08078] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08385] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08406] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08188] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09681] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08080] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08386] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08190] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09679] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08074] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08383] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Wed May 29 06:49:38 2019
Command line : /mnt/cephfs/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 4M -b 12000m -s 4 -F -o /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux loadgen1
TestID : 0
StartTime : Wed May 29 06:49:38 2019
Path : /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_easy
FS : 49.7 TiB Used FS: 0.0% Inodes: 0.0 Mi Used Inodes: 100.1%
Options:
api : POSIX
apiVersion :
test filename : /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 4
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 70
clients per node : 7
repetitions : 1
xfersize : 4 MiB
blocksize : 11.72 GiB
aggregate filesize : 3.20 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 8688 max: 10413 -- min data: 33.9 GiB mean data: 38.4 GiB time: 300.0s
WARNING: Expected aggregate file size = 3523215360000.
WARNING: Stat() of aggregate file size = 3057270128640.
WARNING: Using actual aggregate bytes moved = 3057270128640.
WARNING: maybe caused by deadlineForStonewalling
write 8373 12288000 4096 0.004926 348.21 0.025612 348.24 0
Max Write: 8372.58 MiB/sec (8779.29 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 8372.58 8372.58 8372.58 0.00 2093.15 2093.15 2093.15 0.00 348.23654 0 70 7 1 1 1 1 0 0 4 12582912000 4194304 2915640.0 POSIX 0
Finished : Wed May 29 06:55:26 2019
- ior_hard_read
-
[loadgen1:4598][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4596][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4594][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4597][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4593][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4595][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4601][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:04598] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04594] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04596] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04595] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04593] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04597] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04601] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10334][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10332][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9299][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10265][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10336][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9295][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10263][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9057][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9207][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9457][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10339][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9297][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10267][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9064][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9205][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9459][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10333][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9296][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10262][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9059][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9203][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9455][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9248][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8959][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10337][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9294][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10266][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9061][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9209][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9461][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9250][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8958][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10543][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10335][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9301][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10264][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9060][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9206][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9458][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9246][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8963][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10541][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10339] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:9298][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10269][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9058][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9202][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9454][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9252][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8957][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10545][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10336] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09295] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10265] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:9062][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9204][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9456][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9249][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8956][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10544][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10332] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09297] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10267] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09059] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09207] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09459] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:9247][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8960][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10546][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10334] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09299] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10266] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09057] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09205] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09455] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:9245][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8961][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10542][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10335] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09296] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10264] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09061] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09203] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09457] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09246] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08957] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10548][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10337] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09294] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10269] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09064] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09209] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09461] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09248] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08960] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10543] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10333] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09298] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10262] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09060] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09202] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09458] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09250] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08956] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10545] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09301] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10263] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09058] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09206] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09454] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09247] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08961] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10546] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09062] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09204] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09456] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09249] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08963] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10544] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09252] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08959] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10541] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09245] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08958] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10542] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10548] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Wed May 29 07:51:16 2019
Command line : /mnt/cephfs/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 185192 -o /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_hard/stonewall
Machine : Linux loadgen1
TestID : 0
StartTime : Wed May 29 07:51:16 2019
Path : /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_hard
FS : 48.3 TiB Used FS: 6.6% Inodes: 8.0 Mi Used Inodes: 100.0%
Options:
api : POSIX
apiVersion :
test filename : /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 185192
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 70
clients per node : 7
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 567.53 GiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
WARNING: Expected aggregate file size = 609385387520.
WARNING: Stat() of aggregate file size = 439635268800.
WARNING: Using actual aggregate bytes moved = 439635268800.
read 3204.01 45.91 45.91 0.011031 130.85 0.009910 130.86 0
Max Read: 3204.01 MiB/sec (3359.65 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 3204.01 3204.01 3204.01 0.00 71469.74 71469.74 71469.74 0.00 130.85749 0 70 7 1 0 1 1 0 0 185192 47008 47008 419268.9 POSIX 0
Finished : Wed May 29 07:53:27 2019
- ior_hard_write
-
[loadgen1:4089][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4087][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4086][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4090][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4088][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4094][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4091][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:04090] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04086] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04088] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04089] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04094] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04087] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04091] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:8624][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8408][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8411][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8404][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8563][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9610][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8406][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8565][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9617][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8626][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8776][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8561][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9612][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8622][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8783][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9674][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8310][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8560][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9614][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8625][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8780][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9676][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8312][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8407][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9909][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8603][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8567][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9613][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8628][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8778][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9672][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8308][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8409][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9911][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8601][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8564][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9611][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8623][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8777][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9673][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8309][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8405][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9907][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8605][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8562][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9615][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8621][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8779][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9679][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8307][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:08411] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:9913][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8607][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:08565] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09610] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08624] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:8781][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9671][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8314][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:08404] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:9908][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8600][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:08563] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09612] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08622] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08783] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:9675][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8311][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:08408] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:9910][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8602][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:08560] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09613] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08626] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08776] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09676] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08312] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08406] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:9906][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8604][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:08561] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09611] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08628] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08778] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09672] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08310] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08407] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09911] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08605] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08567] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09617] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08623] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08780] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09674] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08308] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08409] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09909] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08601] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08564] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09615] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08625] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08779] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09679] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08309] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08405] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09907] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08603] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08562] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09614] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08621] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08781] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09671] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08307] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09913] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08607] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08777] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09673] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08314] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09908] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08600] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09675] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08311] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09906] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08604] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09910] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08602] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Wed May 29 07:01:14 2019
Command line : /mnt/cephfs/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 185192 -o /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux loadgen1
TestID : 0
StartTime : Wed May 29 07:01:14 2019
Path : /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_hard
FS : 48.5 TiB Used FS: 5.7% Inodes: 4.6 Mi Used Inodes: 100.0%
Options:
api : POSIX
apiVersion :
test filename : /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 185192
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 70
clients per node : 7
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 567.53 GiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 102637 max: 133605 -- min data: 4.5 GiB mean data: 5.2 GiB time: 300.0s
WARNING: Expected aggregate file size = 609385387520.
WARNING: Stat() of aggregate file size = 439635268800.
WARNING: Using actual aggregate bytes moved = 439635268800.
WARNING: maybe caused by deadlineForStonewalling
write 1133.42 45.91 45.91 0.005141 369.89 0.027087 369.91 0
Max Write: 1133.42 MiB/sec (1188.48 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 1133.42 1133.42 1133.42 0.00 25282.51 25282.51 25282.51 0.00 369.91378 0 70 7 1 0 1 1 0 0 185192 47008 47008 419268.9 POSIX 0
Finished : Wed May 29 07:07:24 2019
- mdtest_easy_delete
-
[loadgen1:4716][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4714][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4718][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4717][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4713][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4715][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4720][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:04714] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04716] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04718] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04717] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04713] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04715] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04720] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10490][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10420][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10494][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10423][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9456][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10493][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10418][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9455][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9606][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10491][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10416][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9453][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9608][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9216][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9360][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9401][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10495][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10421][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9452][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9609][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9214][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9364][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9406][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10695][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10492][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10417][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9454][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9605][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9219][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9362][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9403][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10697][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9118][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10497][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10419][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9451][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9612][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9212][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9367][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9407][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10700][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9116][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10494] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10423] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:9458][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9607][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9215][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9365][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9402][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10693][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9114][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10493] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10418] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09456] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:9610][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9213][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9363][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9400][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10696][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9121][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10495] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10420] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09453] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09606] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:9217][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9361][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9404][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10698][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9113][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10492] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10416] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09455] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09608] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09214] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09367] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09401] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10694][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9115][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10491] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10417] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09452] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09605] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09219] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09364] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09406] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10695] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:9117][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10490] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10419] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09454] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09612] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09212] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09365] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09403] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10700] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09116] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10497] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10421] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09451] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09607] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09216] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09361] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09407] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10697] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09114] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09458] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09610] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09217] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09360] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09400] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10693] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09118] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09609] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09215] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09363] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09402] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10698] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09115] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09213] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09362] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09404] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10696] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09121] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10694] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09117] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09113] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/29/2019 07:54:36 --
mdtest-1.9.3 was launched with 70 total task(s) on 10 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-r" "-F" "-d" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_easy" "-n" "60000" "-u" "-L" "-x" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_easy-stonewall"
Path: /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36
FS: 48.3 TiB Used FS: 6.6% Inodes: 8.0 Mi Used Inodes: 100.0%
70 tasks, 4200000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 10685.724 10684.945 10685.422 0.205
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 1.384 1.384 1.384 0.000
-- finished at 05/29/2019 08:01:10 --
- mdtest_easy_stat
-
[loadgen1:4531][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4533][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4535][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4532][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4534][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4536][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4530][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:04531] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04533] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04535] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04534] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04532] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04536] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04530] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:9125][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9121][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10182][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9366][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9123][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10180][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9362][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10248][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10179][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9363][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10246][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8977][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9214][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10183][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9361][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10244][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8975][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9213][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9122][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10460][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9159][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8876][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10178][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9367][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10250][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8979][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9211][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9124][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10456][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9163][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8878][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10181][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9365][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10247][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8974][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9217][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9120][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10462][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9162][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8874][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10177][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9364][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10245][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8980][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9216][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9126][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10458][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9164][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8877][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10179] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09366] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10249][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8978][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9215][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09125] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10459][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9160][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8875][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10177] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09363] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10244] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:8976][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9212][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09121] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10457][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9161][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8879][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10182] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09361] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10246] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08977] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09216] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09123] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10461][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9158][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8873][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10178] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09362] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10248] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08975] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09212] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09120] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10462] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09159] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08874] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10180] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09364] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10250] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08979] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09211] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09122] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10458] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09164] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08876] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10183] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09367] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10245] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08974] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09215] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09124] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10459] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09160] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08879] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10181] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09365] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10247] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08978] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09213] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09126] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10457] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09162] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08875] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10249] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08980] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09217] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10460] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09163] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08877] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08976] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09214] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10461] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09158] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08873] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10456] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09161] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08878] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/29/2019 07:49:21 --
mdtest-1.9.3 was launched with 70 total task(s) on 10 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-T" "-F" "-d" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_easy" "-n" "60000" "-u" "-L" "-x" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_easy-stonewall"
Path: /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36
FS: 48.3 TiB Used FS: 6.6% Inodes: 8.0 Mi Used Inodes: 100.0%
70 tasks, 4200000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 36685.023 36682.322 36683.988 0.695
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/29/2019 07:51:16 --
- mdtest_easy_write
-
[loadgen1:3959][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3955][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3957][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3956][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3958][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3954][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:3960][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:03959] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03957] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03955] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03954] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03956] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03958] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:03960] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:8417][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8419][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8643][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8641][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8421][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8647][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8274][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8494][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9529][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8646][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8275][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8492][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9533][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8422][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8642][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8280][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8495][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9528][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8420][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9471][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9763][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8161][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8644][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8279][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8491][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9531][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8416][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9465][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9765][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8163][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8475][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8645][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8277][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8497][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9532][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8418][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9469][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9767][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8167][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8473][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:08643] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:8276][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8496][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9530][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:08417] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:9467][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9762][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8165][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8471][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:08641] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:8278][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8493][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9534][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:08419] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:9470][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9768][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8166][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8474][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:08647] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08275] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08492] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09529] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08421] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:9466][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9764][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8164][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8476][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:08646] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08279] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08494] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09533] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08420] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:9468][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9766][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8162][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8470][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:08642] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08280] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08495] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09531] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08422] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09467] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09765] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08163] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:8472][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:08644] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08274] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08497] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09528] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08418] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09471] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09763] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08161] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08471] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08645] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08277] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08491] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09530] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08416] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09469] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09767] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08167] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08475] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08276] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08496] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09534] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09465] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09768] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08165] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08473] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08278] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08493] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09532] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09470] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09762] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08164] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08474] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09466] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09764] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08166] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08476] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09468] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09766] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08162] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08472] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08470] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/29/2019 06:55:28 --
mdtest-1.9.3 was launched with 70 total task(s) on 10 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-C" "-F" "-d" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_easy" "-n" "60000" "-u" "-L" "-x" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_easy-stonewall" "-W" "300"
Path: /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36
FS: 48.5 TiB Used FS: 5.7% Inodes: 0.7 Mi Used Inodes: 100.0%
70 tasks, 4200000 files
Continue stonewall hit min: 39774 max: 60000 avg: 54890.1
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 12156.913 12156.021 12156.576 0.233
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 161.384 161.384 161.384 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/29/2019 07:01:14 --
- mdtest_hard_delete
-
[loadgen1:4889][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4890][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4891][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4887][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4894][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4886][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4888][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:04891] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04889] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04894] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04890] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04887] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04886] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04888] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10720][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10718][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10722][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9831][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9674][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10647][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9434][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9597][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9824][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9673][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10643][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9432][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9592][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10721][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9618][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9828][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9670][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10650][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9435][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9590][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10724][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9622][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10924][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9826][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9672][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10645][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9431][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9594][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10719][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9620][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10925][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9341][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9827][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9669][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10646][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9433][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9591][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10717][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9624][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10923][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9337][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9825][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9676][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10648][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9438][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9593][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10722] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:9621][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10928][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9339][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9829][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9671][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10644][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9430][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9595][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10718] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:9617][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10926][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9344][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:09831] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09670] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10643] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09434] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09597] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10720] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:9619][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10927][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9340][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:09824] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09674] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10650] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09432] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09590] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10724] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09620] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10930][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9338][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:09828] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09672] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10646] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09431] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09592] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10719] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09622] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10924] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:9342][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:09826] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09673] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10648] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09435] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09594] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10721] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09618] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10925] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09341] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09827] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09669] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10644] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09433] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09593] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10717] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09621] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10923] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09344] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09829] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09676] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10645] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09438] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09591] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09624] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10928] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09337] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09825] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09671] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10647] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09430] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09595] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09617] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10926] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09339] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09619] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10927] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09342] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10930] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09338] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09340] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/29/2019 08:04:16 --
mdtest-1.9.3 was launched with 70 total task(s) on 10 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-r" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_hard" "-n" "51614" "-x" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_hard-stonewall"
Path: /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36
FS: 48.3 TiB Used FS: 6.6% Inodes: 4.0 Mi Used Inodes: 100.0%
70 tasks, 3612980 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 8171.164 8170.564 8170.936 0.156
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 63.017 63.017 63.017 0.000
-- finished at 05/29/2019 08:11:12 --
- mdtest_hard_read
-
[loadgen1:4827][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4825][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4833][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4831][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4828][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4829][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4826][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:04825] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04827] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04833] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04829] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04828] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04831] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04826] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:9508][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9504][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10633][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9505][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10629][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9748][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10561][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9507][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10631][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9745][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10563][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9344][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9588][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9506][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10632][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9743][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10559][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9351][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9584][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10844][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9485][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9503][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10628][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9741][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10560][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9348][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9583][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10839][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9489][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9256][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9510][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10630][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9746][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10562][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9346][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9586][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10837][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9492][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9258][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09508] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10635][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9744][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10558][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9349][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9587][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10841][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9487][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9259][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09506] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10629] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:9742][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10565][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9345][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9585][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10842][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9486][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9261][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09507] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10633] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09745] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10559] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:9347][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9591][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10840][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9488][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9260][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09505] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10631] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09743] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10561] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09351] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09588] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10838][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9491][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9257][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09504] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10632] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09748] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10563] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09346] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09584] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10844] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09489] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:9264][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09503] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10628] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09741] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10560] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09348] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09586] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10839] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09487] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09261] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09510] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10635] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09746] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10562] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09344] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09587] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10837] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09485] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09257] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10630] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09744] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10558] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09345] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09585] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10840] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09492] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09256] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09742] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10565] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09349] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09583] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10842] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09488] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09260] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09347] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09591] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10841] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09486] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09259] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10838] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09491] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09264] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09258] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/29/2019 08:01:10 --
mdtest-1.9.3 was launched with 70 total task(s) on 10 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-E" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_hard" "-n" "51614" "-x" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_hard-stonewall"
Path: /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36
FS: 48.3 TiB Used FS: 6.6% Inodes: 4.1 Mi Used Inodes: 100.0%
70 tasks, 3612980 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 18256.096 18254.750 18255.582 0.349
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/29/2019 08:04:16 --
- mdtest_hard_stat
-
[loadgen1:4660][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4662][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4658][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4665][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4661][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4659][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4657][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:04660] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04658] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04662] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04659] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04661] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04665] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04657] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:9286][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9284][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9533][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9283][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9536][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10342][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9142][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9379][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9290][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9531][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10339][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9135][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9377][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9285][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9535][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10343][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9139][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9375][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10624][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9287][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9529][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10340][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9137][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9374][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10617][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10414][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9039][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9325][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:9288][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.[loadgen7:9532][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10341][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9138][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9381][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10619][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10416][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9043][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9330][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:9530][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:10345][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9136][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9376][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10621][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10413][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9045][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9324][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09286] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09533] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10338][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:9140][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:9378][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:10620][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10412][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9038][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9328][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09284] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09536] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10343] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09135] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09379] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10622][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10411][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9042][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9331][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09283] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09531] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10339] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09139] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09377] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10618][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:10418][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9040][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9327][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09285] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09535] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10341] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09137] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09375] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10617] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10415][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:9041][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:9326][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:09290] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09529] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10340] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09142] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09376] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10619] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10413] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09043] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09330] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09287] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09532] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10345] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09138] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09381] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10621] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10414] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09045] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09325] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:09288] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:09530] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10342] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09136] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09374] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10624] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10416] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09042] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09328] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:10338] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:09140] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:09378] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10618] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10412] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09040] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09324] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10622] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10411] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09041] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09327] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:10620] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10418] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09038] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09331] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:10415] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:09039] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:09326] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/29/2019 07:53:28 --
mdtest-1.9.3 was launched with 70 total task(s) on 10 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-T" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_hard" "-n" "51614" "-x" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_hard-stonewall"
Path: /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36
FS: 48.3 TiB Used FS: 6.6% Inodes: 8.0 Mi Used Inodes: 100.0%
70 tasks, 3612980 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 50099.406 50095.699 50097.986 0.954
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/29/2019 07:54:35 --
- mdtest_hard_write
-
[loadgen1:4158][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4156][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4154][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4157][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4155][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4161][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:4153][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:04158] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04156] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04154] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04157] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04161] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04153] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:04155] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:9696][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9698][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9756][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9754][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9695][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8704][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8859][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8493][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9761][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9693][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8706][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8861][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8496][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8641][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9758][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9697][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8711][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8858][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8491][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8645][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8688][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9989][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9757][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9694][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8708][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8862][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8489][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8643][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8683][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9991][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8389][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9759][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:9700][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:8705][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8863][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8490][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8648][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8681][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9987][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8391][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:9755][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:09696] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:8707][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8860][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8492][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8642][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8685][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9988][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8388][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:09756] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09698] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:8709][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:8865][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:8494][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:8644][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8682][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9990][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8393][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:09758] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09695] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08706] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08859] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08493] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:8646][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:8686][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9986][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8392][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:09759] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09693] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08711] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08861] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08489] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08643] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:8684][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:9993][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:8395][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:09755] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09694] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08704] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08862] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08496] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08645] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08683] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09987] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:8390][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:09757] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09697] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08708] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08863] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08491] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08648] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08681] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09991] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08389] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09761] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:09700] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08707] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08860] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08490] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08642] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08688] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09989] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08393] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:09754] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08705] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08865] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08494] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08644] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08686] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09988] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08391] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:08709] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:08858] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:08492] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08646] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08682] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09990] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08388] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:08641] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08684] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09993] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08395] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:08685] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:09986] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08392] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:08390] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/29/2019 07:07:25 --
mdtest-1.9.3 was launched with 70 total task(s) on 10 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-C" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_hard" "-n" "51614" "-x" "/mnt/cephfs/datafiles/io500.2019.05.29-06.49.36/mdt_hard-stonewall" "-W" "300"
Path: /mnt/cephfs/datafiles/io500.2019.05.29-06.49.36
FS: 48.4 TiB Used FS: 6.6% Inodes: 4.8 Mi Used Inodes: 100.0%
70 tasks, 3612980 files
Continue stonewall hit min: 44942 max: 48350 avg: 46264.9
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 10742.683 10741.893 10742.386 0.206
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 3333.900 3333.900 3333.900 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/29/2019 07:12:40 --
- result_summary
-
[RESULT] BW phase 1 ior_easy_write 8.177 GB/s : time 348.24 seconds
[RESULT] IOPS phase 1 mdtest_easy_write 12.157 kiops : time 347.70 seconds
[RESULT] BW phase 2 ior_hard_write 1.107 GB/s : time 369.91 seconds
[RESULT] IOPS phase 2 mdtest_hard_write 10.743 kiops : time 315.60 seconds
[RESULT] IOPS phase 3 find 123.940 kiops : time 61.19 seconds
[RESULT] BW phase 3 ior_easy_read 1.332 GB/s : time 2137.56 seconds
[RESULT] IOPS phase 4 mdtest_easy_stat 36.685 kiops : time 116.82 seconds
[RESULT] BW phase 4 ior_hard_read 3.129 GB/s : time 130.86 seconds
[RESULT] IOPS phase 5 mdtest_hard_stat 50.099 kiops : time 68.09 seconds
[RESULT] IOPS phase 6 mdtest_easy_delete 10.686 kiops : time 394.38 seconds
[RESULT] IOPS phase 7 mdtest_hard_read 18.256 kiops : time 185.92 seconds
[RESULT] IOPS phase 8 mdtest_hard_delete 8.171 kiops : time 416.03 seconds
[SCORE] Bandwidth 2.47825 GB/s : IOPS 21.6021 kiops : TOTAL 7.31679