- io500
-
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.
set -euo pipefail # better error handling
# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True" # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True" # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False" # this one is optional
io500_cleanup_workdir="False" # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_stonewall_timer=300 # Stonewalling timer, stop with wearout after 300s with default test, set to 0, if you never want to abort...
# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
setup_directories
setup_paths
setup_ior_easy # required if you want a complete score
setup_ior_hard # required if you want a complete score
setup_mdt_easy # required if you want a complete score
setup_mdt_hard # required if you want a complete score
setup_find # required if you want a complete score
# setup_mdreal # optional
run_benchmarks
}
function setup_directories {
# set directories for where the benchmark files are created and where the results will go.
# If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
timestamp=`date +%Y.%m.%d-%H.%M.%S` # create a uniquifier
io500_workdir=/mnt/cephfs/datafiles/io500.$timestamp # directory where the data will be stored
io500_result_dir=/mnt/cephfs/io-500-dev/results/$timestamp # the directory where the output results will be kept
mkdir -p $io500_workdir $io500_result_dir
}
function setup_paths {
# Set the paths to the binaries. If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
io500_ior_cmd=/mnt/cephfs/io-500-dev/bin/ior
io500_mdtest_cmd=/mnt/cephfs/io-500-dev/bin/mdtest
io500_mdreal_cmd=/mnt/cephfs/io-500-dev/bin/md-real-io
io500_mpirun="/usr/mpi/gcc/openmpi-4.0.0rc5/bin/mpirun"
io500_mpiargs="-np 98 --hostfile /mnt/cephfs/io-500-dev/myhosts"
}
function setup_ior_easy {
# io500_ior_easy_size is the amount of data written per rank in MiB units,
# but it can be any number as long as it is somehow used to scale the IOR
# runtime as part of io500_ior_easy_params
# io500_ior_easy_size=513096
io500_ior_easy_size=12000
# 2M writes, 2 GB per proc, file per proc
# io500_ior_easy_params="-t 2048k -b ${io500_ior_easy_size}m -F"
io500_ior_easy_params="-t 4M -b ${io500_ior_easy_size}m -s 4 -F"
}
function setup_mdt_easy {
io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
io500_mdtest_easy_files_per_proc=60000
}
function setup_ior_hard {
io500_ior_hard_writes_per_proc=185192
io500_ior_hard_other_options="" #e.g., -E to keep precreated files using lfs setstripe, or -a MPIIO
}
function setup_mdt_hard {
io500_mdtest_hard_files_per_proc=51614
io500_mdtest_hard_other_options=""
}
function setup_find {
#
# setup the find command. This is an area where innovation is allowed.
# There are three default options provided. One is a serial find, one is python
# parallel version, one is C parallel version. Current default is to use serial.
# But it is very slow. We recommend to either customize or use the C parallel version.
# For GPFS, we recommend to use the provided mmfind wrapper described below.
# Instructions below.
# If a custom approach is used, please provide enough info so others can reproduce.
# the serial version that should run (SLOWLY) without modification
#io500_find_mpi="False"
#io500_find_cmd=$PWD/bin/sfind.sh
#io500_find_cmd_args=""
# a parallel version in C, the -s adds a stonewall
# for a real run, turn -s (stonewall) off or set it at 300 or more
# to prepare this (assuming you've run ./utilities/prepare.sh already):
# > cd build/pfind
# > ./prepare.sh
# > ./compile.sh
# > cp pfind ../../bin/
# If you use io500_find_mpi="True", then this will run with the same
# number of MPI nodes and ranks as the other phases.
# If you prefer another number, and fewer might be better here,
# Then you can set io500_find_mpi to be "False" and write a wrapper
# script for this which sets up MPI as you would like. Then change
# io500_find_cmd to point to your wrapper script.
io500_find_mpi="True"
io500_find_cmd="$PWD/bin/pfind"
# uses stonewalling, run pfind
io500_find_cmd_args="-s $io500_stonewall_timer -r $io500_result_dir/pfind_results"
# for GPFS systems, you should probably use the provided mmfind wrapper
# if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
#io500_find_mpi="False"
#io500_find_cmd="$PWD/bin/mmfind.sh"
#io500_find_cmd_args=""
}
function setup_mdreal {
io500_mdreal_params="-P=5000 -I=1000"
}
function run_benchmarks {
# Important: source the io500_fixed.sh script. Do not change it. If you discover
# a need to change it, please email the mailing list to discuss
source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}
# Add key/value pairs defining your system
# Feel free to add extra ones if you'd like
function extra_description {
# top level info
io500_info_system_name='TigerShark' # e.g. Oakforest-PACS
io500_info_institute_name='SUSE' # e.g. JCAHPC
io500_info_storage_age_in_months='1' # not install date but age since last refresh
io500_info_storage_install_date='11/2018' # MM/YY
io500_info_filesystem='SUSE Enterprise Storage (CephFS)' # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
io500_info_filesystem_version='5.5 (12.2.11)'
io500_info_filesystem_vendor='SUSE'
# client side info
io500_info_num_client_nodes='10'
io500_info_procs_per_node='2'
# server side info
io500_info_num_metadata_server_nodes='3'
io500_info_num_data_server_nodes='10'
io500_info_num_data_storage_devices='120' # if you have 5 data servers, and each has 5 drives, then this number is 25
io500_info_num_metadata_storage_devices='10' # if you have 2 metadata servers, and each has 5 drives, then this number is 10
io500_info_data_storage_type='Intel® SSD D3-S4510' # HDD, SSD, persistent memory, etc, feel free to put specific models
io500_info_metadata_storage_type='Intel® Optane SSD DC P4800X' # HDD, SSD, persistent memory, etc, feel free to put specific models
io500_info_storage_network='ethernet' # infiniband, omnipath, ethernet, etc
io500_info_storage_interface='SATA' # SAS, SATA, NVMe, etc
# miscellaneous
io500_info_whatever='10x Lenovo ThinkSystem SR650 for storage nodes w/1x Intel Xeon Gold 6142, 96GB RAM, 100Gb Ethernet'
}
main
- ior_easy_read
-
[loadgen1:13441][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13439][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13443][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13447][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13440][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13445][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13442][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13441] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13439] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13447] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13443] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13445] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13442] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13440] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20143][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18261][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18259][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20141][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18257][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20144][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17991][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20120][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18258][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20148][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17986][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20118][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18471][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18317][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28923][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18260][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20145][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17988][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20122][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18477][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18315][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28925][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2918][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3829][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20398][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18226][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18256][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20140][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17984][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20121][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18472][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18319][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28927][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2925][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3833][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20399][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18222][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17973][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6151][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18263][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20142][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17989][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20124][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18473][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18318][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28924][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2922][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3831][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20403][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18224][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17971][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6153][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18259] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20143] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17987][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20117][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18475][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18321][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28929][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2923][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3828][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20401][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18223][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17969][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6149][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18261] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20141] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17985][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20119][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18474][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18314][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28922][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2921][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3830][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20400][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18225][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17975][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6155][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18257] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20144] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17986] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20122] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18470][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18316][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28926][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2919][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3832][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20405][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18221][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17972][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6152][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18258] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20148] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17991] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20118] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18471] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18319] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28923] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:2920][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3835][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20402][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18228][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17968][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6148][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18260] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20145] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17984] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20120] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18477] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18315] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28925] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02918] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03833] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20398] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18224] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17970][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6150][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18256] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20140] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17988] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20121] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18475] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18317] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28927] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02925] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03831] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20403] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18226] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17969] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06153] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18263] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20142] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17989] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20124] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18473] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18318] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28929] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02922] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03829] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20401] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18222] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17973] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06151] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17987] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20117] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18472] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18314] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28922] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02919] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03828] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20399] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18223] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17971] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06149] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17985] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20119] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18474] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18321] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28924] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02921] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03830] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20400] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18221] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17975] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06155] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18470] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18316] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28926] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02923] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03835] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20405] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18225] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17972] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06150] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02920] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03832] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20402] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18228] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17970] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06148] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17968] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06152] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Thu May 30 08:54:38 2019
Command line : /mnt/cephfs/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 4M -b 12000m -s 4 -F -o /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_easy/stonewall
Machine : Linux loadgen1
TestID : 0
StartTime : Thu May 30 08:54:38 2019
Path : /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_easy
FS : 93.2 TiB Used FS: 14.2% Inodes: 11.8 Mi Used Inodes: 100.0%
Options:
api : POSIX
apiVersion :
test filename : /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 4
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 98
clients per node : 7
repetitions : 1
xfersize : 4 MiB
blocksize : 11.72 GiB
aggregate filesize : 4.49 TiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
WARNING: Expected aggregate file size = 4932501504000.
WARNING: Stat() of aggregate file size = 3130083246080.
WARNING: Using actual aggregate bytes moved = 3130083246080.
read 2253.82 12288000 4096 6.45 1318.00 0.096765 1324.45 0
Max Read: 2253.82 MiB/sec (2363.30 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 2253.82 2253.82 2253.82 0.00 563.45 563.45 563.45 0.00 1324.45365 0 98 7 1 1 1 1 0 0 4 12582912000 4194304 2985080.0 POSIX 0
Finished : Thu May 30 09:16:43 2019
- ior_easy_write
-
[loadgen1:12953][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:12955][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:12954][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:12959][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:12952][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:12956][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:12957][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:12953] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:12952] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:12954] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:12955] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:12956] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:12957] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:12959] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17402][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17690][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17406][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17404][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17688][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28330][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:17901][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17409][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17686][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28332][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:17897][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19562][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17720][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19529][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17405][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17685][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28334][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:17899][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19560][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17723][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19524][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17670][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3253][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:19821][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17403][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17687][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28336][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:17903][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19564][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17716][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19522][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17668][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3251][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:19822][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2317][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5569][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17403][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17407][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17692][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28333][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:17898][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19566][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17719][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19526][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17666][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3255][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:19825][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2315][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5567][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17401][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17402] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17689][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28329][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:17900][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19563][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17717][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19523][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17667][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3250][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:19823][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2321][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5571][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17399][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17404] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17688] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28331][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:17896][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19561][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17718][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19525][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17669][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3254][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:19827][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2323][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5568][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17400][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17409] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17690] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28332] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:17899] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19559][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17721][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19527][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17672][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3252][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:19824][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2316][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5572][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17398][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17406] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17686] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28330] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:17901] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19560] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17716] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19529] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17665][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3256][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:19820][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2322][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5570][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17402][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17405] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17685] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28334] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:17897] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19562] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17718] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19524] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17666] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03255] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19820] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:2320][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5566][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17405][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17407] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17687] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28336] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:17903] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19564] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17723] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19527] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17670] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03251] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19822] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02315] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05567] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17403] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17403] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17692] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28333] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:17896] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19566] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17717] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19523] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17668] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03253] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19824] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02323] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05571] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17399] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17689] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28329] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:17900] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19561] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17719] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19525] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17672] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03250] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19825] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02321] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05569] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17401] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28331] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:17898] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19559] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17720] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19526] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17667] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03254] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19827] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02317] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05572] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17398] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19563] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17721] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19522] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17665] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03252] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19823] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02322] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05570] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17400] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17669] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03256] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19821] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02320] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05568] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17405] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02316] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05566] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17402] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Thu May 30 08:24:52 2019
Command line : /mnt/cephfs/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 4M -b 12000m -s 4 -F -o /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_easy/ior_file_easy -O stoneWallingStatusFile=/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux loadgen1
TestID : 0
StartTime : Thu May 30 08:24:52 2019
Path : /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_easy
FS : 95.0 TiB Used FS: 10.1% Inodes: 2.4 Mi Used Inodes: 100.0%
Options:
api : POSIX
apiVersion :
test filename : /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_easy/ior_file_easy
access : file-per-process
type : independent
segments : 4
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 98
clients per node : 7
repetitions : 1
xfersize : 4 MiB
blocksize : 11.72 GiB
aggregate filesize : 4.49 TiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 6027 max: 7615 -- min data: 23.5 GiB mean data: 28.0 GiB time: 300.1s
WARNING: Expected aggregate file size = 4932501504000.
WARNING: Stat() of aggregate file size = 3130083246080.
WARNING: Using actual aggregate bytes moved = 3130083246080.
WARNING: maybe caused by deadlineForStonewalling
write 8461 12288000 4096 0.019827 352.76 0.029809 352.79 0
Max Write: 8461.43 MiB/sec (8872.46 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 8461.43 8461.43 8461.43 0.00 2115.36 2115.36 2115.36 0.00 352.78659 0 98 7 1 1 1 1 0 0 4 12582912000 4194304 2985080.0 POSIX 0
Finished : Thu May 30 08:30:45 2019
- ior_hard_read
-
[loadgen1:13666][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13672][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13668][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13664][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13670][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13667][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13665][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13668] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13672] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13664] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13666] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13667] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13670] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13665] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20422][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20418][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18746][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20417][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18741][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18264][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18581][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20421][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18739][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18260][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18588][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20398][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18539][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4097][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29181][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20420][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18743][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18267][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18583][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20396][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18537][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4099][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29177][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3189][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18255][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18496][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20419][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18740][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18262][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18585][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20395][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18541][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4094][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29179][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3194][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18253][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18494][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6430][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20680][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20424][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18745][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18261][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18584][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20400][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18544][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4096][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29184][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3191][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18251][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18498][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6437][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20682][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20422] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18742][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18265][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18586][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20397][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18538][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4098][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29182][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3187][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18254][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18493][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6432][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20684][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20418] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18741] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18263][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18582][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20393][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18542][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4101][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29180][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3188][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18252][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18497][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6434][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20679][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20417] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18746] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18264] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18581] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20394][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18540][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4095][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29178][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3192][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18250][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18495][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6431][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20681][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20419] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18739] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18260] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18583] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20398] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18541] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04099] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29177] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:3190][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18257][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18500][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6433][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20686][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20424] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18743] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18267] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18588] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20396] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18539] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04097] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29181] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03189] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18255] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18496] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:6435][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20683][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20420] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18740] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18262] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18585] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20400] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18544] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04094] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29184] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03194] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18253] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18494] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06430] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20682] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20421] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18742] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18261] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18584] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20395] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18542] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04098] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29179] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03191] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18251] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18498] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06437] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20684] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18745] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18265] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18586] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20393] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18538] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04096] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29180] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03187] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18254] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18497] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06432] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20680] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18263] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18582] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20394] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18540] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04101] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29182] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03192] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18250] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18495] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06434] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20681] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20397] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18537] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04095] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29178] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03188] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18252] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18500] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06433] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20686] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03190] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18257] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18493] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06431] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20679] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06435] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20683] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Thu May 30 09:19:23 2019
Command line : /mnt/cephfs/io-500-dev/bin/ior -r -R -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 185192 -o /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_hard/stonewall
Machine : Linux loadgen1
TestID : 0
StartTime : Thu May 30 09:19:23 2019
Path : /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_hard
FS : 93.2 TiB Used FS: 14.2% Inodes: 11.8 Mi Used Inodes: 100.0%
Options:
api : POSIX
apiVersion :
test filename : /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 185192
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 98
clients per node : 7
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 794.55 GiB
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
WARNING: Expected aggregate file size = 853139542528.
WARNING: Stat() of aggregate file size = 840047062400.
WARNING: Using actual aggregate bytes moved = 840047062400.
read 4535 45.91 45.91 0.012125 176.65 0.023690 176.65 0
Max Read: 4535.02 MiB/sec (4755.32 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
read 4535.02 4535.02 4535.02 0.00 101159.71 101159.71 101159.71 0.00 176.65432 0 98 7 1 0 1 1 0 0 185192 47008 47008 801131.3 POSIX 0
Finished : Thu May 30 09:22:19 2019
- ior_hard_write
-
[loadgen1:13135][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13129][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13127][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13131][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13130][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13132][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13128][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13135] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13131] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13129] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13130] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13132] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13128] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13127] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19745][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17617][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19753][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17619][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19781][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28540][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3478][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18134][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17905][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19749][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17624][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19788][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28536][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3480][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18129][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17901][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2537][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17951][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17884][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20043][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19751][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17618][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19785][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28538][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3482][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18127][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17903][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2539][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17953][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17886][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20041][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17624][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5789][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19748][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17622][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19786][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28539][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3481][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18130][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17907][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2541][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17950][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17888][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20039][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17622][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5791][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19746][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17620][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19782][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28537][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3484][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18128][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17902][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2538][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17957][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17887][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20038][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17620][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5786][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19747][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17621][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19784][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28535][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3477][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18131][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17904][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2540][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17954][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17890][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20045][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17623][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5793][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19746] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17624] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19783][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28542][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3479][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18132][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17900][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2543][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17952][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17883][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20040][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17619][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5788][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19751] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17617] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19785] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28540] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03482] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18134] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17905] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:2536][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17955][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17885][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20042][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.[loadgen11:17621][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5790][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19748] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17620] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19781] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28536] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03480] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18129] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17901] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02537] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17951] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17886] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17626][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5787][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19745] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17618] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19786] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28538] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03478] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18127] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17903] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02541] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17953] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17888] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20043] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17624] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05790] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19753] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17622] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19788] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28539] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03484] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18130] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17902] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02539] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17950] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17884] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20038] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17622] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05786] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19747] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17619] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19783] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28537] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03481] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18128] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17907] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02540] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17954] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17887] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20039] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17621] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05791] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19749] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17621] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19784] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28542] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03477] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18132] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17904] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02538] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17952] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17890] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20045] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17623] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05789] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19782] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28535] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03479] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18131] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17900] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02543] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17955] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17885] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20040] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17619] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05788] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02536] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17957] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17883] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20041] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17620] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05787] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20042] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17626] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05793] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
IOR-3.2.0: MPI Coordinated Test of Parallel I/O
Began : Thu May 30 08:42:03 2019
Command line : /mnt/cephfs/io-500-dev/bin/ior -w -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -s 185192 -o /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_hard/IOR_file -O stoneWallingStatusFile=/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine : Linux loadgen1
TestID : 0
StartTime : Thu May 30 08:42:03 2019
Path : /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_hard
FS : 93.6 TiB Used FS: 13.3% Inodes: 8.7 Mi Used Inodes: 100.0%
Options:
api : POSIX
apiVersion :
test filename : /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/ior_hard/IOR_file
access : single-shared-file
type : independent
segments : 185192
ordering in a file : sequential
ordering inter file : constant task offset
task offset : 1
tasks : 98
clients per node : 7
repetitions : 1
xfersize : 47008 bytes
blocksize : 47008 bytes
aggregate filesize : 794.55 GiB
stonewallingTime : 300
stoneWallingWearOut : 1
Results:
access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter
------ --------- ---------- --------- -------- -------- -------- -------- ----
stonewalling pairs accessed min: 132935 max: 182350 -- min data: 5.8 GiB mean data: 7.0 GiB time: 300.1s
WARNING: Expected aggregate file size = 853139542528.
WARNING: Stat() of aggregate file size = 840047062400.
WARNING: Using actual aggregate bytes moved = 840047062400.
WARNING: maybe caused by deadlineForStonewalling
write 2088.99 45.91 45.91 0.002681 383.48 0.028331 383.50 0
Max Write: 2088.99 MiB/sec (2190.46 MB/sec)
Summary of all tests:
Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Max(OPs) Min(OPs) Mean(OPs) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggs(MiB) API RefNum
write 2088.99 2088.99 2088.99 0.00 46597.69 46597.69 46597.69 0.00 383.50186 0 98 7 1 0 1 1 0 0 185192 47008 47008 801131.3 POSIX 0
Finished : Thu May 30 08:48:27 2019
- mdtest_easy_delete
-
[loadgen1:13793][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13797][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13801][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13795][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13796][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13799][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13794][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13797] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13793] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13801] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13795] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13796] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13799] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13794] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20582][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20578][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20556][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18697][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20558][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18700][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18900][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20585][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20553][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18695][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.[loadgen7:18907][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20580][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3346][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18417][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4254][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29340][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20843][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20554][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18905][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20583][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3350][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18425][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4258][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29337][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20841][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18743][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18660][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20555][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18694][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18903][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20581][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3353][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18421][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4256][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29333][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20839][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18739][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18658][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6596][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18407][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20551][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18701][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18901][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20579][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3351][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18419][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4260][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29335][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20840][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18741][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18659][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6593][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18414][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20552][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18698][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18902][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20582] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:3347][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18420][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4255][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29334][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20838][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18738][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18661][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6591][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18409][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20556] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18696][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18904][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20580] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:3349][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18418][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4257][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29338][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20845][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18745][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18665][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6592][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18411][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20554] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18697] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18907] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20585] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:3348][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18422][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4253][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29336][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20842][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18742][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18662][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6590][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18413][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20555] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18700] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18900] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20578] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03350] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18421] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04258] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29333] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20839] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18740][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18657][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6594][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18408][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20558] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18695] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18903] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20581] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03346] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18417] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04254] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29340] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20843] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18743] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18660] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:6589][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18410][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20552] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18701] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18901] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20583] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03353] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18419] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04256] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29337] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20841] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18739] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18658] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06591] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18409] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20551] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18696] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18905] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20579] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03348] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18420] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04255] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29338] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20845] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18741] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18659] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06593] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18414] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20553] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18694] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18902] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03349] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18418] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04253] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29334] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20838] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18738] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18662] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06590] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18411] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18698] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18904] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03347] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18422] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04257] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29336] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20840] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18745] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18665] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06594] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18413] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03351] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18425] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04260] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29335] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20842] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18742] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18661] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06592] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18408] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18740] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18657] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06596] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18410] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06589] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18407] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/30/2019 09:23:18 --
mdtest-1.9.3 was launched with 98 total task(s) on 14 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-r" "-F" "-d" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_easy" "-n" "60000" "-u" "-L" "-x" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_easy-stonewall"
Path: /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51
FS: 93.2 TiB Used FS: 14.2% Inodes: 11.8 Mi Used Inodes: 100.0%
98 tasks, 5880000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 9356.307 9355.620 9356.041 0.163
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 1.025 1.025 1.025 0.000
-- finished at 05/30/2019 09:33:48 --
- mdtest_easy_stat
-
[loadgen1:13596][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13598][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13594][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13595][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13599][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13593][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13597][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13598] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13594] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13596] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13595] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13599] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13593] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13597] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18656][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18658][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18654][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18655][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18180][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20332][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20312][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18657][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18182][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20336][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20314][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18500][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18450][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29090][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3108][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18653][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18178][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20334][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20311][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18498][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18454][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29092][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3106][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18412][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4013][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6346][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20592][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18167][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18659][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18181][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20330][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20315][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18495][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18452][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29094][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3110][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18414][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4015][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6348][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20594][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18165][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18658] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18177][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20333][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20313][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18497][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18451][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29089][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3111][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18410][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4011][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6344][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20596][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18163][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18656] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18179][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20331][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20309][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18499][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18455][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29091][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3109][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18415][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4016][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6349][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20593][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18168][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18654] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18183][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20335][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20310][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18494][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18453][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29095][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3107][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18411][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4012][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6347][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20595][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18164][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18655] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18180] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20334] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20312] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18496][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18449][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29093][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3112][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18409][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4010][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6343][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20597][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18166][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18659] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18182] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20332] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20314] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18495] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18454] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29091] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03110] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18413][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4014][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6345][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20591][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18162][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18653] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18178] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20330] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20313] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18497] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18450] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29095] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03106] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18412] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04015] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06346] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20594] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18163] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18657] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18179] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20336] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20311] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18499] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18452] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29089] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03108] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18411] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04013] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06348] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20596] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18167] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18183] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20335] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20315] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18496] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18451] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29093] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03111] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18409] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04011] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06347] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20592] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18165] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18181] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20331] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20309] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18494] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18453] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29092] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03107] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18414] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04012] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06349] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20593] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18164] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18177] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20333] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20310] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18500] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18455] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29090] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03109] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18413] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04014] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06343] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20595] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18166] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18498] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18449] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29094] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03112] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18415] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04010] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06344] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20597] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18162] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18410] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04016] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06345] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20591] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18168] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/30/2019 09:16:45 --
mdtest-1.9.3 was launched with 98 total task(s) on 14 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-T" "-F" "-d" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_easy" "-n" "60000" "-u" "-L" "-x" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_easy-stonewall"
Path: /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51
FS: 93.2 TiB Used FS: 14.2% Inodes: 11.8 Mi Used Inodes: 100.0%
98 tasks, 5880000 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 37465.241 37462.495 37464.192 0.649
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/30/2019 09:19:22 --
- mdtest_easy_write
-
[loadgen1:13060][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13064][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13065][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13062][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13059][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13061][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13063][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13064] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13060] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13065] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13062] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13059] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13061] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13063] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19695][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19697][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19693][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19691][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19694][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19696][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17819][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18042][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19692][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17817][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18047][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19693] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17821][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18046][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19695] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17823][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18044][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28457][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17538][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3399][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19663][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17862][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19697] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17820][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18045][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28455][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17534][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3401][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19661][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17860][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17488][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:19956][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17749][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19691] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17818][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18043][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28459][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17536][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3397][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19659][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17863][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17493][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:19952][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17751][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5702][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2454][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19694] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17822][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18041][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28458][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17539][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3400][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19658][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17865][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17490][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:19954][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17752][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5704][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2458][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19696] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17817] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18042] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28454][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17535][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3396][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19660][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17859][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17492][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17492] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19957][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17754][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5706][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2456][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19692] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17823] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18044] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28456][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17537][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3398][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19662][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17864][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17491][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17491] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19955][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17750][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5708][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2453][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17819] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18046] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28460][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17533][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3395][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19657][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:17861][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17489][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17489] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19951][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17755][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5707][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2459][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17821] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18043] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28455] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17539] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03399] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19663] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17862] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17487][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17487] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19953][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:17753][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5705][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2457][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17822] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18045] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28457] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17538] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03397] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19659] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17860] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17488] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19954] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17749] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:5703][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2455][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:17820] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18047] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28456] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17535] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03401] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19662] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17863] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17493] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19956] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17754] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05704] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02456] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:17818] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18041] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28454] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17537] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03395] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19660] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17865] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17490] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19952] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17752] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05706] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02454] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28459] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17536] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03396] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19661] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17859] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19957] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17750] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05708] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02458] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28460] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17533] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03398] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19657] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17864] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19951] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17753] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05702] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02453] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28458] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17534] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03400] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19658] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:17861] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19953] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17755] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05707] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02459] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:19955] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:17751] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05705] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02457] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05703] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02455] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/30/2019 08:30:47 --
mdtest-1.9.3 was launched with 98 total task(s) on 14 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-C" "-F" "-d" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_easy" "-n" "60000" "-u" "-L" "-x" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_easy-stonewall" "-W" "300"
Path: /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51
FS: 93.6 TiB Used FS: 13.3% Inodes: 3.1 Mi Used Inodes: 100.0%
98 tasks, 5880000 files
Continue stonewall hit min: 22132 max: 60000 avg: 39444.5
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 8709.292 8708.657 8709.044 0.151
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 117.753 117.753 117.753 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/30/2019 08:42:03 --
- mdtest_hard_delete
-
[loadgen1:13972][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13974][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13979][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13971][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13975][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13973][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13977][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13972] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13974] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13979] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13973] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13975] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13971] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13977] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20808][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20810][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18918][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18914][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20783][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20809][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20785][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20805][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18916][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3570][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18969][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20788][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18920][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3572][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18976][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19136][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20784][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18917][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3569][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18971][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19134][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4490][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29544][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18636][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18880][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20782][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18915][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3567][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18973][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19132][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4492][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29546][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18638][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18882][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6814][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20806][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:21057][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18631][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20786][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18913][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3574][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18972][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19130][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4488][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29543][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18640][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18884][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6816][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20807][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:21059][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18627][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20781][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18914] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:3568][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18970][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19133][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4494][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29539][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18643][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18881][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6818][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20812][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:21055][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18633][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20785] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18918] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:3571][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18974][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19129][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4487][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29540][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18641][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18886][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6813][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20808] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21061][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18626][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20783] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18916] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03570] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18971] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19131][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4491][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29541][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18639][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18883][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6815][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20810] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21054][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18629][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20788] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18917] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03572] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18976] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19136] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:4489][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29542][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18637][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18879][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6817][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20809] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21056][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18628][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20784] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18913] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03567] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18973] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19134] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04488] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29542] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18640] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18880] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:6820][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20805] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21058][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18630][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20782] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18920] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03569] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18974] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19133] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04492] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29544] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18638] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18884] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06816] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20806] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21059] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18627] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20786] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18915] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03568] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18972] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19130] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04490] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29546] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18636] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18881] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06814] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20807] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21055] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18633] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20781] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03574] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18969] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19132] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04487] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29539] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18643] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18886] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06818] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20812] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21061] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18629] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03571] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18970] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19129] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04491] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29543] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18641] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18882] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06815] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21054] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18626] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19131] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04494] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29541] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18639] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18883] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06813] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21057] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18631] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04489] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29540] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18637] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18879] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06817] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21056] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18628] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06820] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:21058] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18630] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/30/2019 09:37:47 --
mdtest-1.9.3 was launched with 98 total task(s) on 14 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-r" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_hard" "-n" "51614" "-x" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_hard-stonewall"
Path: /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51
FS: 93.2 TiB Used FS: 14.2% Inodes: 6.2 Mi Used Inodes: 100.0%
98 tasks, 5058172 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 7367.279 7366.739 7367.070 0.128
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 16.251 16.251 16.251 0.000
-- finished at 05/30/2019 09:44:36 --
- mdtest_hard_read
-
[loadgen1:13897][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13903][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13895][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13899][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13898][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13896][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13901][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13896] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13898] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13897] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13903] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13895] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13899] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13901] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20717][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20715][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20694][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18831][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19044][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20713][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20687][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18833][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19042][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18881][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18547][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3483][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4397][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20716][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20688][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18835][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19043][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18879][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18551][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3479][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4399][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29459][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20976][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18796][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6727][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20714][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20689][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18830][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19045][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18886][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18549][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3481][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4395][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29452][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20972][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18794][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6732][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18539][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20719][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20691][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18832][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19041][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18883][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18554][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3482][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4398][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29454][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20973][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18792][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6725][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18542][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20712][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20693][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18837][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19040][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18884][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18548][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3480][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4401][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29455][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20980][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18795][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6726][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18545][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20717] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20690][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18834][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:19047][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18882][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18550][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3484][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4394][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29453][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20975][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18798][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6730][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18540][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20715] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20691] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18830] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19042] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18880][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18552][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3486][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4396][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29457][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20974][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18793][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6728][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18543][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20716] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20689] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18833] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19043] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18879] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18551] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03483] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04397] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29456][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20977][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18791][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6729][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18538][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20714] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20694] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18835] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19045] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18886] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18549] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03481] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04399] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29454] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20980] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18796] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06725] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18541][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20713] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20693] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18831] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19044] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18881] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18554] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03479] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04395] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29455] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20975] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18795] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06727] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18540] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20719] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20688] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18832] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19041] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18883] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18547] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03482] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04394] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29453] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20973] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18791] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06728] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18543] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20712] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20687] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18837] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19040] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18882] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18548] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03484] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04398] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29457] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20977] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18793] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06726] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18545] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20690] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18834] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:19047] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18884] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18550] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03480] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04401] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29459] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20972] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18798] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06730] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18539] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18880] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18552] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03486] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04396] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29456] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20976] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18794] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06732] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18538] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29452] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20974] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18792] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06729] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18541] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18542] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/30/2019 09:33:48 --
mdtest-1.9.3 was launched with 98 total task(s) on 14 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-E" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_hard" "-n" "51614" "-x" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_hard-stonewall"
Path: /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51
FS: 93.2 TiB Used FS: 14.2% Inodes: 6.2 Mi Used Inodes: 100.0%
98 tasks, 5058172 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 0.000 0.000 0.000 0.000
File read : 12629.528 12628.604 12629.155 0.222
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/30/2019 09:37:46 --
- mdtest_hard_stat
-
[loadgen1:13734][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13740][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13733][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13736][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13738][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13735][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13732][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13732] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13740] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13738] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13733] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13736] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13735] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13734] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20500][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20480][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20499][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20476][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18822][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20501][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20477][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18824][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18666][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20763][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18342][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3268][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18617][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20503][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20479][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18825][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18659][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20765][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18340][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3270][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18615][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29259][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4176][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18578][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18330][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6514][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20502][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20481][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18823][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18663][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20761][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18343][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3272][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18618][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29258][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4178][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18582][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18332][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6512][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20504][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20478][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18828][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18660][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20764][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18346][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3271][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18616][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29261][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4180][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18585][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18334][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6510][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20506][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:20483][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18826][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18662][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20760][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18339][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3273][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18622][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29256][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4175][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18580][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18336][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6513][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20501] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20476] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18821][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18664][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20767][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18344][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3269][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18619][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29257][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4182][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18583][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18329][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6511][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20499] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20477] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18824] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18661][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20762][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:18341][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:3275][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18614][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:29260][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4177][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18581][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18331][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6509][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20500] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20479] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18822] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18663] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20763] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18346] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03272] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18615] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29263][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:4179][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18579][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:18333][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:6516][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:20502] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20478] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18825] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18666] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20765] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18342] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03268] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18617] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29259] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04178] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18585] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18330] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06510] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20504] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20481] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18823] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18659] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20767] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18344] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03270] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18618] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29261] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04180] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18582] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18334] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06514] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20503] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20480] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18826] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18662] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20760] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18340] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03269] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18622] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29260] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04175] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18578] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18336] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06512] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:20506] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:20483] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18828] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18664] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20764] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18339] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03273] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18619] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29257] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04182] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18580] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18331] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06513] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18821] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18661] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20762] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18343] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03271] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18616] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29256] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04176] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18583] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18329] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06511] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18660] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20761] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:18341] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:03275] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18614] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29263] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04179] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18581] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18332] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06509] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:29258] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:04177] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18579] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:18333] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:06516] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/30/2019 09:22:20 --
mdtest-1.9.3 was launched with 98 total task(s) on 14 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-T" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_hard" "-n" "51614" "-x" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_hard-stonewall"
Path: /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51
FS: 93.2 TiB Used FS: 14.2% Inodes: 11.8 Mi Used Inodes: 100.0%
98 tasks, 5058172 files
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 0.000 0.000 0.000 0.000
File stat : 51759.424 51755.600 51757.853 0.915
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 0.000 0.000 0.000 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/30/2019 09:23:18 --
- mdtest_hard_write
-
[loadgen1:13239][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13236][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13234][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13241][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13237][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13235][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13233][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen1:13239] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13234] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13236] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13241] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13233] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13237] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen1:13235] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17751][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17749][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19913][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18034][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19917][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18032][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18257][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19876][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19914][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18036][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18264][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19883][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18084][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20174][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3614][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19915][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18035][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18261][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19877][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18086][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20172][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3610][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28660][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2670][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17750][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5918][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19916][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18038][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18259][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19879][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18082][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20170][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3612][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28658][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2668][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17746][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5920][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18028][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17751][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19912][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18033][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18263][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19881][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18085][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20176][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3611][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28664][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2672][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17747][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5922][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18026][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17749][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19919][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen10:18031][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen7:18258][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19878][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18081][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20169][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3609][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28657][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2667][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17748][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5919][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18027][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17757][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19914] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18032] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18260][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen5:19880][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen9:18088][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20171][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3613][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28661][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2671][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17754][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen14:5921][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18025][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17753][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19917] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18036] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18257] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19876] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18083][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen4:20173][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen13:3616][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen12:28659][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2674][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17751] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:5917][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18032][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17750][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19915] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18034] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18264] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19881] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18084] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20172] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03612] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28662][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen16:2669][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen8:17749] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:5924][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen6:18030][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17752][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19913] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18035] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18261] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19877] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18086] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20174] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03610] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28658] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02670] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17746] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05920] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18029][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen11:17754][coll_ml_mca.c:239:set_hcoll_device] COLL-ML You must specify a valid HCA device by setting:
-x HCOLL_MAIN_IB= or -x MXM_RDMA_PORTS=.
If no device was specified for HCOLL (or the calling library), automatic device detection will be run.
In case of unfounded HCA device please contact your system administrator.
[loadgen2:19916] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18038] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18259] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19879] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18082] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20170] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03614] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28660] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02668] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17750] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05918] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18026] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17751] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19912] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18033] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18263] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19883] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18081] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20169] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03609] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28664] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02672] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17754] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05922] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18028] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17753] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen2:19919] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen10:18031] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18258] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19878] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18088] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20171] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03611] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28661] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02667] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17747] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05919] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18025] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17750] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen7:18260] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen5:19880] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18085] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20176] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03616] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28659] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02674] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen8:17748] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05917] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18030] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17752] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen9:18083] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen4:20173] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen13:03613] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28662] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02671] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05921] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18027] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17754] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen12:28657] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen16:02669] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen14:05924] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18032] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17749] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen6:18029] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
[loadgen11:17757] Error: coll_hcoll_module.c:301 - mca_coll_hcoll_comm_query() Hcol library init failed
-- started at 05/30/2019 08:48:28 --
mdtest-1.9.3 was launched with 98 total task(s) on 14 node(s)
Command line used: /mnt/cephfs/io-500-dev/bin/mdtest "-C" "-t" "-F" "-w" "3901" "-e" "3901" "-d" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_hard" "-n" "51614" "-x" "/mnt/cephfs/datafiles/io500.2019.05.30-08.24.51/mdt_hard-stonewall" "-W" "300"
Path: /mnt/cephfs/datafiles/io500.2019.05.30-08.24.51
FS: 93.3 TiB Used FS: 14.1% Inodes: 8.9 Mi Used Inodes: 100.0%
98 tasks, 5058172 files
Continue stonewall hit min: 28349 max: 30667 avg: 29518.2
SUMMARY rate: (of 1 iterations)
Operation Max Min Mean Std Dev
--------- --- --- ---- -------
File creation : 9612.049 9611.348 9611.776 0.167
File stat : 0.000 0.000 0.000 0.000
File read : 0.000 0.000 0.000 0.000
File removal : 0.000 0.000 0.000 0.000
Tree creation : 2699.143 2699.143 2699.143 0.000
Tree removal : 0.000 0.000 0.000 0.000
-- finished at 05/30/2019 08:53:40 --
- result_summary
-
[RESULT] BW phase 1 ior_easy_write 8.263 GB/s : time 352.79 seconds
[RESULT] IOPS phase 1 mdtest_easy_write 8.709 kiops : time 677.44 seconds
[RESULT] BW phase 2 ior_hard_write 2.040 GB/s : time 383.50 seconds
[RESULT] IOPS phase 2 mdtest_hard_write 9.612 kiops : time 313.22 seconds
[RESULT] IOPS phase 3 find 154.200 kiops : time 57.62 seconds
[RESULT] BW phase 3 ior_easy_read 2.201 GB/s : time 1324.45 seconds
[RESULT] IOPS phase 4 mdtest_easy_stat 37.465 kiops : time 158.96 seconds
[RESULT] BW phase 4 ior_hard_read 4.429 GB/s : time 176.65 seconds
[RESULT] IOPS phase 5 mdtest_hard_stat 51.759 kiops : time 58.63 seconds
[RESULT] IOPS phase 6 mdtest_easy_delete 9.356 kiops : time 630.02 seconds
[RESULT] IOPS phase 7 mdtest_hard_read 12.629 kiops : time 238.50 seconds
[RESULT] IOPS phase 8 mdtest_hard_delete 7.367 kiops : time 409.11 seconds
[SCORE] Bandwidth 3.58025 GB/s : IOPS 19.6014 kiops : TOTAL 8.37723