Data Accelerator

Institution University of Cambridge
Client Procs Per Node
Client Operating System CentOS
Client Operating System Version 7.7
Client Kernel Version 3.10.0-957.10.1.el7.x86_64

DATA SERVER

Storage Type NVMe
Volatile Memory 192GiB
Storage Interface NVMe
Network OPA (2x OPA HFIs per server)
Software Version 2.13 (tip of master branch as of running)
OS Version CentoS 7.7

INFORMATION

Client Nodes 10
Client Total Procs 320
Metadata Nodes 24
Metadata Storage Devices 1
Data Nodes 24
Data Storage Devices 288

METADATA

Easy Write 435.70 kIOP/s
Easy Stat 1,184.27 kIOP/s
Easy Delete 129.62 kIOP/s
Hard Write 29.81 kIOP/s
Hard Read 59.01 kIOP/s
Hard Stat 65.40 kIOP/s
Hard Delete 10.74 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

PROGRAM="${0##*/}"

function usage () {
  echo
  cat <<-_EOF
  Usage:
    $PROGRAM work_dir results_dir [ io500_dir ]
      where 'work_dir' is the place where the benchmark IO will take place
      and 'results_dir' is where the output results will be kept. The
      optional 'io500_dir' is where the io500 directory is (containing
      the pre-built IOR and mdtest binaries

_EOF
}

if [[ $# -ne 3 ]]; then
  usage
  exit 0
fi

JOB_WORK_DIR="$1"
JOB_RESULTS_DIR="$2"
IO500_DIR="${3:-$HOME/projects/benchmarking/io-500-src}"

if [ ! -d "$JOB_RESULTS_DIR" ] ; then
  echo "JOB_RESULTS_DIR: $JOB_RESULTS_DIR does not exist"
  exit 1
fi

if [ ! -d "$JOB_WORK_DIR" ] ; then
  echo "JOB_WORK_DIR: $JOB_WORK_DIR does not exist"
  mkdir -pv $JOB_WORK_DIR
fi
# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  io500_workdir=$JOB_WORK_DIR                       # directory where the data will be stored
  io500_result_dir=$JOB_RESULTS_DIR                 # the directory where the output results will be kept
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier

  io500_ior_easy=$io500_workdir/ior_easy
  io500_ior_hard=$io500_workdir/ior_hard
  io500_mdt_easy=$io500_workdir/mdt_easy
  io500_mdt_hard=$io500_workdir/mdt_hard

  mkdir -p $io500_workdir $io500_result_dir $io500_ior_easy $io500_ior_hard $io500_mdt_easy $io500_mdt_hard

  # Use DNE2 striped directory for mdt_easy
  lfs setdirstripe -c -1 -D $io500_mdt_easy

  # Use DoM for mdt_easy
  lfs setstripe -L mdt -E 1M "$io500_mdt_easy"

  # 1 stripes, 16M stripe-size
  lfs setstripe -c 8 -S 16M $io500_ior_easy

  # Use Lustre OST overstriping - 1 Stripe per client thread
  lfs setstripe -C 320 -S 16M $io500_ior_hard

  # Use DNE2 striped directory for mdt_hard
  lfs setdirstripe -c -1 -D $io500_mdt_hard
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$IO500_DIR/bin/ior
  io500_mdtest_cmd=$IO500_DIR/bin/mdtest
  io500_mdreal_cmd=$IO500_DIR/bin/md-real-io
  io500_mpirun="srun"
  io500_mpiargs="--mpi=pmi2"
}

function setup_ior_easy {
  io500_ior_easy_params="-a=POSIX --posix.odirect -t 16m"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
}

function setup_ior_hard {
  io500_ior_hard_api="MPIIO"
  io500_ior_hard_api_specific_options=""
}

function setup_mdt_hard {
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$IO500_DIR/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  #io500_find_mpi="False"
  io500_find_cmd="$IO500_DIR/bin/pfind"
  #io500_find_cmd="mpirun -ppn 12 $IO500_DIR/bin/pfind"
  # uses stonewalling, run pfind
  #io500_find_cmd_args=""
  io500_find_cmd_args="-N -P -s $io500_stonewall_timer -r $io500_result_dir/pfind_results"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source $IO500_DIR/utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
  io500_info_system_name='Cambridge-DAC'      # e.g. Oakforest-PACS
  io500_info_institute_name='University of Cambridge RCS'   # e.g. JCAHPC
  io500_info_storage_age_in_months='18' # not install date but age since last refresh
  io500_info_storage_install_date='05/18'  # MM/YY
  io500_info_filesystem='Lustre'     # e.g. BeeGFS, DataWarp, GPFS, IME, Lustre
  io500_info_filesystem_version='2.13 (tip of master branch as of running)'
  io500_info_filesystem_vendor='Dell EMC'
  # client side info
  io500_info_num_client_nodes='32'
  io500_info_procs_per_node='32'
  # server side info
  io500_info_num_metadata_server_nodes='24'
  io500_info_num_data_server_nodes='24'
  io500_info_num_data_storage_devices='288'  # if you have 5 data servers, and each has 5 drives, then this number is 25
  io500_info_num_metadata_storage_devices='24'  # if you have 2 metadata servers, and each has 5 drives, then this number is 10
  io500_info_data_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_metadata_storage_type='SSD' # HDD, SSD, persistent memory, etc, feel free to put specific models
  io500_info_storage_network='omnipath' # infiniband, omnipath, ethernet, etc
  io500_info_storage_interface='NVMe' # SAS, SATA, NVMe, etc
  # miscellaneous
  io500_info_whatever="More details on the architecture were presented at LAD'19: https://www.eofs.eu/_media/events/lad19/03_matt_raso-barnett-io500-cambridge.pdf"
}

main
ior_easy_read
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       35669    cpu-e-829   0
[0] MPI startup(): 1       35670    cpu-e-829   1
[0] MPI startup(): 2       35671    cpu-e-829   2
[0] MPI startup(): 3       35672    cpu-e-829   3
[0] MPI startup(): 4       35673    cpu-e-829   4
[0] MPI startup(): 5       35674    cpu-e-829   5
[0] MPI startup(): 6       35675    cpu-e-829   6
[0] MPI startup(): 7       35676    cpu-e-829   7
[0] MPI startup(): 8       35677    cpu-e-829   8
[0] MPI startup(): 9       35678    cpu-e-829   9
[0] MPI startup(): 10      35679    cpu-e-829   10
[0] MPI startup(): 11      35680    cpu-e-829   11
[0] MPI startup(): 12      35681    cpu-e-829   12
[0] MPI startup(): 13      35682    cpu-e-829   13
[0] MPI startup(): 14      35683    cpu-e-829   14
[0] MPI startup(): 15      35684    cpu-e-829   15
[0] MPI startup(): 16      35685    cpu-e-829   16
[0] MPI startup(): 17      35686    cpu-e-829   17
[0] MPI startup(): 18      35687    cpu-e-829   18
[0] MPI startup(): 19      35688    cpu-e-829   19
[0] MPI startup(): 20      35689    cpu-e-829   20
[0] MPI startup(): 21      35690    cpu-e-829   21
[0] MPI startup(): 22      35691    cpu-e-829   22
[0] MPI startup(): 23      35692    cpu-e-829   23
[0] MPI startup(): 24      35693    cpu-e-829   24
[0] MPI startup(): 25      35694    cpu-e-829   25
[0] MPI startup(): 26      35695    cpu-e-829   26
[0] MPI startup(): 27      35696    cpu-e-829   27
[0] MPI startup(): 28      35697    cpu-e-829   28
[0] MPI startup(): 29      35698    cpu-e-829   29
[0] MPI startup(): 30      35699    cpu-e-829   30
[0] MPI startup(): 31      35700    cpu-e-829   31
[0] MPI startup(): 32      34969    cpu-e-830   0
[0] MPI startup(): 33      34970    cpu-e-830   1
[0] MPI startup(): 34      34971    cpu-e-830   2
[0] MPI startup(): 35      34972    cpu-e-830   3
[0] MPI startup(): 36      34973    cpu-e-830   4
[0] MPI startup(): 37      34974    cpu-e-830   5
[0] MPI startup(): 38      34975    cpu-e-830   6
[0] MPI startup(): 39      34976    cpu-e-830   7
[0] MPI startup(): 40      34977    cpu-e-830   8
[0] MPI startup(): 41      34978    cpu-e-830   9
[0] MPI startup(): 42      34979    cpu-e-830   10
[0] MPI startup(): 43      34980    cpu-e-830   11
[0] MPI startup(): 44      34981    cpu-e-830   12
[0] MPI startup(): 45      34982    cpu-e-830   13
[0] MPI startup(): 46      34983    cpu-e-830   14
[0] MPI startup(): 47      34984    cpu-e-830   15
[0] MPI startup(): 48      34985    cpu-e-830   16
[0] MPI startup(): 49      34986    cpu-e-830   17
[0] MPI startup(): 50      34987    cpu-e-830   18
[0] MPI startup(): 51      34988    cpu-e-830   19
[0] MPI startup(): 52      34989    cpu-e-830   20
[0] MPI startup(): 53      34990    cpu-e-830   21
[0] MPI startup(): 54      34991    cpu-e-830   22
[0] MPI startup(): 55      34992    cpu-e-830   23
[0] MPI startup(): 56      34993    cpu-e-830   24
[0] MPI startup(): 57      34994    cpu-e-830   25
[0] MPI startup(): 58      34995    cpu-e-830   26
[0] MPI startup(): 59      34996    cpu-e-830   27
[0] MPI startup(): 60      34997    cpu-e-830   28
[0] MPI startup(): 61      34998    cpu-e-830   29
[0] MPI startup(): 62      34999    cpu-e-830   30
[0] MPI startup(): 63      35000    cpu-e-830   31
[0] MPI startup(): 64      34721    cpu-e-893   0
[0] MPI startup(): 65      34722    cpu-e-893   1
[0] MPI startup(): 66      34723    cpu-e-893   2
[0] MPI startup(): 67      34724    cpu-e-893   3
[0] MPI startup(): 68      34725    cpu-e-893   4
[0] MPI startup(): 69      34726    cpu-e-893   5
[0] MPI startup(): 70      34727    cpu-e-893   6
[0] MPI startup(): 71      34728    cpu-e-893   7
[0] MPI startup(): 72      34729    cpu-e-893   8
[0] MPI startup(): 73      34730    cpu-e-893   9
[0] MPI startup(): 74      34731    cpu-e-893   10
[0] MPI startup(): 75      34732    cpu-e-893   11
[0] MPI startup(): 76      34733    cpu-e-893   12
[0] MPI startup(): 77      34734    cpu-e-893   13
[0] MPI startup(): 78      34735    cpu-e-893   14
[0] MPI startup(): 79      34736    cpu-e-893   15
[0] MPI startup(): 80      34737    cpu-e-893   16
[0] MPI startup(): 81      34738    cpu-e-893   17
[0] MPI startup(): 82      34739    cpu-e-893   18
[0] MPI startup(): 83      34740    cpu-e-893   19
[0] MPI startup(): 84      34741    cpu-e-893   20
[0] MPI startup(): 85      34742    cpu-e-893   21
[0] MPI startup(): 86      34743    cpu-e-893   22
[0] MPI startup(): 87      34744    cpu-e-893   23
[0] MPI startup(): 88      34745    cpu-e-893   24
[0] MPI startup(): 89      34746    cpu-e-893   25
[0] MPI startup(): 90      34747    cpu-e-893   26
[0] MPI startup(): 91      34748    cpu-e-893   27
[0] MPI startup(): 92      34749    cpu-e-893   28
[0] MPI startup(): 93      34750    cpu-e-893   29
[0] MPI startup(): 94      34751    cpu-e-893   30
[0] MPI startup(): 95      34752    cpu-e-893   31
[0] MPI startup(): 96      34704    cpu-e-894   0
[0] MPI startup(): 97      34705    cpu-e-894   1
[0] MPI startup(): 98      34706    cpu-e-894   2
[0] MPI startup(): 99      34707    cpu-e-894   3
[0] MPI startup(): 100     34708    cpu-e-894   4
[0] MPI startup(): 101     34709    cpu-e-894   5
[0] MPI startup(): 102     34710    cpu-e-894   6
[0] MPI startup(): 103     34711    cpu-e-894   7
[0] MPI startup(): 104     34712    cpu-e-894   8
[0] MPI startup(): 105     34713    cpu-e-894   9
[0] MPI startup(): 106     34714    cpu-e-894   10
[0] MPI startup(): 107     34715    cpu-e-894   11
[0] MPI startup(): 108     34716    cpu-e-894   12
[0] MPI startup(): 109     34717    cpu-e-894   13
[0] MPI startup(): 110     34718    cpu-e-894   14
[0] MPI startup(): 111     34719    cpu-e-894   15
[0] MPI startup(): 112     34720    cpu-e-894   16
[0] MPI startup(): 113     34721    cpu-e-894   17
[0] MPI startup(): 114     34722    cpu-e-894   18
[0] MPI startup(): 115     34723    cpu-e-894   19
[0] MPI startup(): 116     34724    cpu-e-894   20
[0] MPI startup(): 117     34725    cpu-e-894   21
[0] MPI startup(): 118     34726    cpu-e-894   22
[0] MPI startup(): 119     34727    cpu-e-894   23
[0] MPI startup(): 120     34728    cpu-e-894   24
[0] MPI startup(): 121     34729    cpu-e-894   25
[0] MPI startup(): 122     34730    cpu-e-894   26
[0] MPI startup(): 123     34731    cpu-e-894   27
[0] MPI startup(): 124     34732    cpu-e-894   28
[0] MPI startup(): 125     34733    cpu-e-894   29
[0] MPI startup(): 126     34734    cpu-e-894   30
[0] MPI startup(): 127     34735    cpu-e-894   31
[0] MPI startup(): 128     34848    cpu-e-957   0
[0] MPI startup(): 129     34849    cpu-e-957   1
[0] MPI startup(): 130     34850    cpu-e-957   2
[0] MPI startup(): 131     34851    cpu-e-957   3
[0] MPI startup(): 132     34852    cpu-e-957   4
[0] MPI startup(): 133     34853    cpu-e-957   5
[0] MPI startup(): 134     34854    cpu-e-957   6
[0] MPI startup(): 135     34855    cpu-e-957   7
[0] MPI startup(): 136     34856    cpu-e-957   8
[0] MPI startup(): 137     34857    cpu-e-957   9
[0] MPI startup(): 138     34858    cpu-e-957   10
[0] MPI startup(): 139     34859    cpu-e-957   11
[0] MPI startup(): 140     34860    cpu-e-957   12
[0] MPI startup(): 141     34861    cpu-e-957   13
[0] MPI startup(): 142     34862    cpu-e-957   14
[0] MPI startup(): 143     34863    cpu-e-957   15
[0] MPI startup(): 144     34864    cpu-e-957   16
[0] MPI startup(): 145     34865    cpu-e-957   17
[0] MPI startup(): 146     34866    cpu-e-957   18
[0] MPI startup(): 147     34867    cpu-e-957   19
[0] MPI startup(): 148     34868    cpu-e-957   20
[0] MPI startup(): 149     34869    cpu-e-957   21
[0] MPI startup(): 150     34870    cpu-e-957   22
[0] MPI startup(): 151     34871    cpu-e-957   23
[0] MPI startup(): 152     34872    cpu-e-957   24
[0] MPI startup(): 153     34873    cpu-e-957   25
[0] MPI startup(): 154     34874    cpu-e-957   26
[0] MPI startup(): 155     34875    cpu-e-957   27
[0] MPI startup(): 156     34876    cpu-e-957   28
[0] MPI startup(): 157     34877    cpu-e-957   29
[0] MPI startup(): 158     34878    cpu-e-957   30
[0] MPI startup(): 159     34879    cpu-e-957   31
[0] MPI startup(): 160     34794    cpu-e-958   0
[0] MPI startup(): 161     34795    cpu-e-958   1
[0] MPI startup(): 162     34796    cpu-e-958   2
[0] MPI startup(): 163     34797    cpu-e-958   3
[0] MPI startup(): 164     34798    cpu-e-958   4
[0] MPI startup(): 165     34799    cpu-e-958   5
[0] MPI startup(): 166     34800    cpu-e-958   6
[0] MPI startup(): 167     34801    cpu-e-958   7
[0] MPI startup(): 168     34802    cpu-e-958   8
[0] MPI startup(): 169     34803    cpu-e-958   9
[0] MPI startup(): 170     34804    cpu-e-958   10
[0] MPI startup(): 171     34805    cpu-e-958   11
[0] MPI startup(): 172     34806    cpu-e-958   12
[0] MPI startup(): 173     34807    cpu-e-958   13
[0] MPI startup(): 174     34808    cpu-e-958   14
[0] MPI startup(): 175     34809    cpu-e-958   15
[0] MPI startup(): 176     34810    cpu-e-958   16
[0] MPI startup(): 177     34811    cpu-e-958   17
[0] MPI startup(): 178     34812    cpu-e-958   18
[0] MPI startup(): 179     34813    cpu-e-958   19
[0] MPI startup(): 180     34814    cpu-e-958   20
[0] MPI startup(): 181     34815    cpu-e-958   21
[0] MPI startup(): 182     34816    cpu-e-958   22
[0] MPI startup(): 183     34817    cpu-e-958   23
[0] MPI startup(): 184     34818    cpu-e-958   24
[0] MPI startup(): 185     34819    cpu-e-958   25
[0] MPI startup(): 186     34820    cpu-e-958   26
[0] MPI startup(): 187     34821    cpu-e-958   27
[0] MPI startup(): 188     34822    cpu-e-958   28
[0] MPI startup(): 189     34823    cpu-e-958   29
[0] MPI startup(): 190     34824    cpu-e-958   30
[0] MPI startup(): 191     34825    cpu-e-958   31
[0] MPI startup(): 192     35047    cpu-e-1021  0
[0] MPI startup(): 193     35048    cpu-e-1021  1
[0] MPI startup(): 194     35049    cpu-e-1021  2
[0] MPI startup(): 195     35050    cpu-e-1021  3
[0] MPI startup(): 196     35051    cpu-e-1021  4
[0] MPI startup(): 197     35052    cpu-e-1021  5
[0] MPI startup(): 198     35053    cpu-e-1021  6
[0] MPI startup(): 199     35054    cpu-e-1021  7
[0] MPI startup(): 200     35055    cpu-e-1021  8
[0] MPI startup(): 201     35056    cpu-e-1021  9
[0] MPI startup(): 202     35057    cpu-e-1021  10
[0] MPI startup(): 203     35058    cpu-e-1021  11
[0] MPI startup(): 204     35059    cpu-e-1021  12
[0] MPI startup(): 205     35060    cpu-e-1021  13
[0] MPI startup(): 206     35061    cpu-e-1021  14
[0] MPI startup(): 207     35062    cpu-e-1021  15
[0] MPI startup(): 208     35063    cpu-e-1021  16
[0] MPI startup(): 209     35064    cpu-e-1021  17
[0] MPI startup(): 210     35065    cpu-e-1021  18
[0] MPI startup(): 211     35066    cpu-e-1021  19
[0] MPI startup(): 212     35067    cpu-e-1021  20
[0] MPI startup(): 213     35068    cpu-e-1021  21
[0] MPI startup(): 214     35069    cpu-e-1021  22
[0] MPI startup(): 215     35070    cpu-e-1021  23
[0] MPI startup(): 216     35071    cpu-e-1021  24
[0] MPI startup(): 217     35072    cpu-e-1021  25
[0] MPI startup(): 218     35073    cpu-e-1021  26
[0] MPI startup(): 219     35074    cpu-e-1021  27
[0] MPI startup(): 220     35075    cpu-e-1021  28
[0] MPI startup(): 221     35076    cpu-e-1021  29
[0] MPI startup(): 222     35077    cpu-e-1021  30
[0] MPI startup(): 223     35078    cpu-e-1021  31
[0] MPI startup(): 224     34808    cpu-e-1022  0
[0] MPI startup(): 225     34809    cpu-e-1022  1
[0] MPI startup(): 226     34810    cpu-e-1022  2
[0] MPI startup(): 227     34811    cpu-e-1022  3
[0] MPI startup(): 228     34812    cpu-e-1022  4
[0] MPI startup(): 229     34813    cpu-e-1022  5
[0] MPI startup(): 230     34814    cpu-e-1022  6
[0] MPI startup(): 231     34815    cpu-e-1022  7
[0] MPI startup(): 232     34816    cpu-e-1022  8
[0] MPI startup(): 233     34817    cpu-e-1022  9
[0] MPI startup(): 234     34818    cpu-e-1022  10
[0] MPI startup(): 235     34819    cpu-e-1022  11
[0] MPI startup(): 236     34820    cpu-e-1022  12
[0] MPI startup(): 237     34821    cpu-e-1022  13
[0] MPI startup(): 238     34822    cpu-e-1022  14
[0] MPI startup(): 239     34823    cpu-e-1022  15
[0] MPI startup(): 240     34824    cpu-e-1022  16
[0] MPI startup(): 241     34825    cpu-e-1022  17
[0] MPI startup(): 242     34826    cpu-e-1022  18
[0] MPI startup(): 243     34827    cpu-e-1022  19
[0] MPI startup(): 244     34828    cpu-e-1022  20
[0] MPI startup(): 245     34829    cpu-e-1022  21
[0] MPI startup(): 246     34830    cpu-e-1022  22
[0] MPI startup(): 247     34831    cpu-e-1022  23
[0] MPI startup(): 248     34832    cpu-e-1022  24
[0] MPI startup(): 249     34833    cpu-e-1022  25
[0] MPI startup(): 250     34834    cpu-e-1022  26
[0] MPI startup(): 251     34835    cpu-e-1022  27
[0] MPI startup(): 252     34836    cpu-e-1022  28
[0] MPI startup(): 253     34837    cpu-e-1022  29
[0] MPI startup(): 254     34838    cpu-e-1022  30
[0] MPI startup(): 255     34839    cpu-e-1022  31
[0] MPI startup(): 256     34834    cpu-e-1149  0
[0] MPI startup(): 257     34835    cpu-e-1149  1
[0] MPI startup(): 258     34836    cpu-e-1149  2
[0] MPI startup(): 259     34837    cpu-e-1149  3
[0] MPI startup(): 260     34838    cpu-e-1149  4
[0] MPI startup(): 261     34839    cpu-e-1149  5
[0] MPI startup(): 262     34840    cpu-e-1149  6
[0] MPI startup(): 263     34841    cpu-e-1149  7
[0] MPI startup(): 264     34842    cpu-e-1149  8
[0] MPI startup(): 265     34843    cpu-e-1149  9
[0] MPI startup(): 266     34844    cpu-e-1149  10
[0] MPI startup(): 267     34845    cpu-e-1149  11
[0] MPI startup(): 268     34846    cpu-e-1149  12
[0] MPI startup(): 269     34847    cpu-e-1149  13
[0] MPI startup(): 270     34848    cpu-e-1149  14
[0] MPI startup(): 271     34849    cpu-e-1149  15
[0] MPI startup(): 272     34850    cpu-e-1149  16
[0] MPI startup(): 273     34851    cpu-e-1149  17
[0] MPI startup(): 274     34852    cpu-e-1149  18
[0] MPI startup(): 275     34853    cpu-e-1149  19
[0] MPI startup(): 276     34854    cpu-e-1149  20
[0] MPI startup(): 277     34855    cpu-e-1149  21
[0] MPI startup(): 278     34856    cpu-e-1149  22
[0] MPI startup(): 279     34857    cpu-e-1149  23
[0] MPI startup(): 280     34858    cpu-e-1149  24
[0] MPI startup(): 281     34859    cpu-e-1149  25
[0] MPI startup(): 282     34860    cpu-e-1149  26
[0] MPI startup(): 283     34861    cpu-e-1149  27
[0] MPI startup(): 284     34862    cpu-e-1149  28
[0] MPI startup(): 285     34863    cpu-e-1149  29
[0] MPI startup(): 286     34864    cpu-e-1149  30
[0] MPI startup(): 287     34865    cpu-e-1149  31
[0] MPI startup(): 288     34858    cpu-e-1150  0
[0] MPI startup(): 289     34859    cpu-e-1150  1
[0] MPI startup(): 290     34860    cpu-e-1150  2
[0] MPI startup(): 291     34861    cpu-e-1150  3
[0] MPI startup(): 292     34862    cpu-e-1150  4
[0] MPI startup(): 293     34863    cpu-e-1150  5
[0] MPI startup(): 294     34864    cpu-e-1150  6
[0] MPI startup(): 295     34865    cpu-e-1150  7
[0] MPI startup(): 296     34866    cpu-e-1150  8
[0] MPI startup(): 297     34867    cpu-e-1150  9
[0] MPI startup(): 298     34868    cpu-e-1150  10
[0] MPI startup(): 299     34869    cpu-e-1150  11
[0] MPI startup(): 300     34870    cpu-e-1150  12
[0] MPI startup(): 301     34871    cpu-e-1150  13
[0] MPI startup(): 302     34872    cpu-e-1150  14
[0] MPI startup(): 303     34873    cpu-e-1150  15
[0] MPI startup(): 304     34874    cpu-e-1150  16
[0] MPI startup(): 305     34875    cpu-e-1150  17
[0] MPI startup(): 306     34876    cpu-e-1150  18
[0] MPI startup(): 307     34877    cpu-e-1150  19
[0] MPI startup(): 308     34878    cpu-e-1150  20
[0] MPI startup(): 309     34879    cpu-e-1150  21
[0] MPI startup(): 310     34880    cpu-e-1150  22
[0] MPI startup(): 311     34881    cpu-e-1150  23
[0] MPI startup(): 312     34882    cpu-e-1150  24
[0] MPI startup(): 313     34883    cpu-e-1150  25
[0] MPI startup(): 314     34884    cpu-e-1150  26
[0] MPI startup(): 315     34885    cpu-e-1150  27
[0] MPI startup(): 316     34886    cpu-e-1150  28
[0] MPI startup(): 317     34887    cpu-e-1150  29
[0] MPI startup(): 318     34888    cpu-e-1150  30
[0] MPI startup(): 319     34889    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Nov  8 00:25:21 2019
Command line        : /home/wjt27/io-500-sc19/bin/ior -r -R -a=POSIX --posix.odirect -t 16m -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_easy/ior_file_easy -O stoneWallingStatusFile=/dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_easy/stonewall
Machine             : Linux cpu-e-829
TestID              : 0
StartTime           : Fri Nov  8 00:25:21 2019
Path                : /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_easy
FS                  : 405.5 TiB   Used FS: 26.5%   Inodes: 1820.8 Mi   Used Inodes: 10.5%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 320
clients per node    : 32
repetitions         : 1
xfersize            : 16 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 3027.34 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 3328599654400000.
WARNING: Stat() of aggregate file size      = 53252225761280.
WARNING: Using actual aggregate bytes moved = 53252225761280.
read      108887     6806       0.000752    10158080000 16384      0.021328   466.38     0.005932   466.41     0   
Max Read:  108886.50 MiB/sec (114175.77 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read       108886.50  108886.50  108886.50       0.00    6805.41    6805.41    6805.41       0.00  466.40565         NA            NA     0    320  32    1   1     1        1         0    0      1 10401873920000 16777216 50785280.0 POSIX      0
Finished            : Fri Nov  8 00:33:08 2019
ior_easy_write
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       33707    cpu-e-829   0
[0] MPI startup(): 1       33708    cpu-e-829   1
[0] MPI startup(): 2       33709    cpu-e-829   2
[0] MPI startup(): 3       33710    cpu-e-829   3
[0] MPI startup(): 4       33711    cpu-e-829   4
[0] MPI startup(): 5       33712    cpu-e-829   5
[0] MPI startup(): 6       33713    cpu-e-829   6
[0] MPI startup(): 7       33714    cpu-e-829   7
[0] MPI startup(): 8       33715    cpu-e-829   8
[0] MPI startup(): 9       33716    cpu-e-829   9
[0] MPI startup(): 10      33717    cpu-e-829   10
[0] MPI startup(): 11      33718    cpu-e-829   11
[0] MPI startup(): 12      33719    cpu-e-829   12
[0] MPI startup(): 13      33720    cpu-e-829   13
[0] MPI startup(): 14      33721    cpu-e-829   14
[0] MPI startup(): 15      33722    cpu-e-829   15
[0] MPI startup(): 16      33723    cpu-e-829   16
[0] MPI startup(): 17      33724    cpu-e-829   17
[0] MPI startup(): 18      33725    cpu-e-829   18
[0] MPI startup(): 19      33726    cpu-e-829   19
[0] MPI startup(): 20      33727    cpu-e-829   20
[0] MPI startup(): 21      33728    cpu-e-829   21
[0] MPI startup(): 22      33729    cpu-e-829   22
[0] MPI startup(): 23      33730    cpu-e-829   23
[0] MPI startup(): 24      33731    cpu-e-829   24
[0] MPI startup(): 25      33732    cpu-e-829   25
[0] MPI startup(): 26      33733    cpu-e-829   26
[0] MPI startup(): 27      33734    cpu-e-829   27
[0] MPI startup(): 28      33735    cpu-e-829   28
[0] MPI startup(): 29      33736    cpu-e-829   29
[0] MPI startup(): 30      33737    cpu-e-829   30
[0] MPI startup(): 31      33738    cpu-e-829   31
[0] MPI startup(): 32      33361    cpu-e-830   0
[0] MPI startup(): 33      33362    cpu-e-830   1
[0] MPI startup(): 34      33363    cpu-e-830   2
[0] MPI startup(): 35      33364    cpu-e-830   3
[0] MPI startup(): 36      33365    cpu-e-830   4
[0] MPI startup(): 37      33366    cpu-e-830   5
[0] MPI startup(): 38      33367    cpu-e-830   6
[0] MPI startup(): 39      33368    cpu-e-830   7
[0] MPI startup(): 40      33369    cpu-e-830   8
[0] MPI startup(): 41      33370    cpu-e-830   9
[0] MPI startup(): 42      33371    cpu-e-830   10
[0] MPI startup(): 43      33372    cpu-e-830   11
[0] MPI startup(): 44      33373    cpu-e-830   12
[0] MPI startup(): 45      33374    cpu-e-830   13
[0] MPI startup(): 46      33375    cpu-e-830   14
[0] MPI startup(): 47      33376    cpu-e-830   15
[0] MPI startup(): 48      33377    cpu-e-830   16
[0] MPI startup(): 49      33378    cpu-e-830   17
[0] MPI startup(): 50      33379    cpu-e-830   18
[0] MPI startup(): 51      33380    cpu-e-830   19
[0] MPI startup(): 52      33381    cpu-e-830   20
[0] MPI startup(): 53      33382    cpu-e-830   21
[0] MPI startup(): 54      33383    cpu-e-830   22
[0] MPI startup(): 55      33384    cpu-e-830   23
[0] MPI startup(): 56      33385    cpu-e-830   24
[0] MPI startup(): 57      33386    cpu-e-830   25
[0] MPI startup(): 58      33387    cpu-e-830   26
[0] MPI startup(): 59      33388    cpu-e-830   27
[0] MPI startup(): 60      33389    cpu-e-830   28
[0] MPI startup(): 61      33390    cpu-e-830   29
[0] MPI startup(): 62      33391    cpu-e-830   30
[0] MPI startup(): 63      33392    cpu-e-830   31
[0] MPI startup(): 64      33198    cpu-e-893   0
[0] MPI startup(): 65      33199    cpu-e-893   1
[0] MPI startup(): 66      33200    cpu-e-893   2
[0] MPI startup(): 67      33201    cpu-e-893   3
[0] MPI startup(): 68      33202    cpu-e-893   4
[0] MPI startup(): 69      33203    cpu-e-893   5
[0] MPI startup(): 70      33204    cpu-e-893   6
[0] MPI startup(): 71      33205    cpu-e-893   7
[0] MPI startup(): 72      33206    cpu-e-893   8
[0] MPI startup(): 73      33207    cpu-e-893   9
[0] MPI startup(): 74      33208    cpu-e-893   10
[0] MPI startup(): 75      33209    cpu-e-893   11
[0] MPI startup(): 76      33210    cpu-e-893   12
[0] MPI startup(): 77      33211    cpu-e-893   13
[0] MPI startup(): 78      33212    cpu-e-893   14
[0] MPI startup(): 79      33213    cpu-e-893   15
[0] MPI startup(): 80      33214    cpu-e-893   16
[0] MPI startup(): 81      33215    cpu-e-893   17
[0] MPI startup(): 82      33216    cpu-e-893   18
[0] MPI startup(): 83      33217    cpu-e-893   19
[0] MPI startup(): 84      33218    cpu-e-893   20
[0] MPI startup(): 85      33219    cpu-e-893   21
[0] MPI startup(): 86      33220    cpu-e-893   22
[0] MPI startup(): 87      33221    cpu-e-893   23
[0] MPI startup(): 88      33222    cpu-e-893   24
[0] MPI startup(): 89      33223    cpu-e-893   25
[0] MPI startup(): 90      33224    cpu-e-893   26
[0] MPI startup(): 91      33225    cpu-e-893   27
[0] MPI startup(): 92      33226    cpu-e-893   28
[0] MPI startup(): 93      33227    cpu-e-893   29
[0] MPI startup(): 94      33228    cpu-e-893   30
[0] MPI startup(): 95      33229    cpu-e-893   31
[0] MPI startup(): 96      33192    cpu-e-894   0
[0] MPI startup(): 97      33193    cpu-e-894   1
[0] MPI startup(): 98      33194    cpu-e-894   2
[0] MPI startup(): 99      33195    cpu-e-894   3
[0] MPI startup(): 100     33196    cpu-e-894   4
[0] MPI startup(): 101     33197    cpu-e-894   5
[0] MPI startup(): 102     33198    cpu-e-894   6
[0] MPI startup(): 103     33199    cpu-e-894   7
[0] MPI startup(): 104     33200    cpu-e-894   8
[0] MPI startup(): 105     33201    cpu-e-894   9
[0] MPI startup(): 106     33202    cpu-e-894   10
[0] MPI startup(): 107     33203    cpu-e-894   11
[0] MPI startup(): 108     33204    cpu-e-894   12
[0] MPI startup(): 109     33205    cpu-e-894   13
[0] MPI startup(): 110     33206    cpu-e-894   14
[0] MPI startup(): 111     33207    cpu-e-894   15
[0] MPI startup(): 112     33208    cpu-e-894   16
[0] MPI startup(): 113     33209    cpu-e-894   17
[0] MPI startup(): 114     33210    cpu-e-894   18
[0] MPI startup(): 115     33211    cpu-e-894   19
[0] MPI startup(): 116     33212    cpu-e-894   20
[0] MPI startup(): 117     33213    cpu-e-894   21
[0] MPI startup(): 118     33214    cpu-e-894   22
[0] MPI startup(): 119     33215    cpu-e-894   23
[0] MPI startup(): 120     33216    cpu-e-894   24
[0] MPI startup(): 121     33217    cpu-e-894   25
[0] MPI startup(): 122     33218    cpu-e-894   26
[0] MPI startup(): 123     33219    cpu-e-894   27
[0] MPI startup(): 124     33220    cpu-e-894   28
[0] MPI startup(): 125     33221    cpu-e-894   29
[0] MPI startup(): 126     33222    cpu-e-894   30
[0] MPI startup(): 127     33223    cpu-e-894   31
[0] MPI startup(): 128     33324    cpu-e-957   0
[0] MPI startup(): 129     33325    cpu-e-957   1
[0] MPI startup(): 130     33326    cpu-e-957   2
[0] MPI startup(): 131     33327    cpu-e-957   3
[0] MPI startup(): 132     33328    cpu-e-957   4
[0] MPI startup(): 133     33329    cpu-e-957   5
[0] MPI startup(): 134     33330    cpu-e-957   6
[0] MPI startup(): 135     33331    cpu-e-957   7
[0] MPI startup(): 136     33332    cpu-e-957   8
[0] MPI startup(): 137     33333    cpu-e-957   9
[0] MPI startup(): 138     33334    cpu-e-957   10
[0] MPI startup(): 139     33335    cpu-e-957   11
[0] MPI startup(): 140     33336    cpu-e-957   12
[0] MPI startup(): 141     33337    cpu-e-957   13
[0] MPI startup(): 142     33338    cpu-e-957   14
[0] MPI startup(): 143     33339    cpu-e-957   15
[0] MPI startup(): 144     33340    cpu-e-957   16
[0] MPI startup(): 145     33341    cpu-e-957   17
[0] MPI startup(): 146     33342    cpu-e-957   18
[0] MPI startup(): 147     33343    cpu-e-957   19
[0] MPI startup(): 148     33344    cpu-e-957   20
[0] MPI startup(): 149     33345    cpu-e-957   21
[0] MPI startup(): 150     33346    cpu-e-957   22
[0] MPI startup(): 151     33347    cpu-e-957   23
[0] MPI startup(): 152     33348    cpu-e-957   24
[0] MPI startup(): 153     33349    cpu-e-957   25
[0] MPI startup(): 154     33350    cpu-e-957   26
[0] MPI startup(): 155     33351    cpu-e-957   27
[0] MPI startup(): 156     33352    cpu-e-957   28
[0] MPI startup(): 157     33353    cpu-e-957   29
[0] MPI startup(): 158     33354    cpu-e-957   30
[0] MPI startup(): 159     33355    cpu-e-957   31
[0] MPI startup(): 160     33281    cpu-e-958   0
[0] MPI startup(): 161     33282    cpu-e-958   1
[0] MPI startup(): 162     33283    cpu-e-958   2
[0] MPI startup(): 163     33284    cpu-e-958   3
[0] MPI startup(): 164     33285    cpu-e-958   4
[0] MPI startup(): 165     33286    cpu-e-958   5
[0] MPI startup(): 166     33287    cpu-e-958   6
[0] MPI startup(): 167     33288    cpu-e-958   7
[0] MPI startup(): 168     33289    cpu-e-958   8
[0] MPI startup(): 169     33290    cpu-e-958   9
[0] MPI startup(): 170     33291    cpu-e-958   10
[0] MPI startup(): 171     33292    cpu-e-958   11
[0] MPI startup(): 172     33293    cpu-e-958   12
[0] MPI startup(): 173     33294    cpu-e-958   13
[0] MPI startup(): 174     33295    cpu-e-958   14
[0] MPI startup(): 175     33296    cpu-e-958   15
[0] MPI startup(): 176     33297    cpu-e-958   16
[0] MPI startup(): 177     33298    cpu-e-958   17
[0] MPI startup(): 178     33299    cpu-e-958   18
[0] MPI startup(): 179     33300    cpu-e-958   19
[0] MPI startup(): 180     33301    cpu-e-958   20
[0] MPI startup(): 181     33302    cpu-e-958   21
[0] MPI startup(): 182     33303    cpu-e-958   22
[0] MPI startup(): 183     33304    cpu-e-958   23
[0] MPI startup(): 184     33305    cpu-e-958   24
[0] MPI startup(): 185     33306    cpu-e-958   25
[0] MPI startup(): 186     33307    cpu-e-958   26
[0] MPI startup(): 187     33308    cpu-e-958   27
[0] MPI startup(): 188     33309    cpu-e-958   28
[0] MPI startup(): 189     33310    cpu-e-958   29
[0] MPI startup(): 190     33311    cpu-e-958   30
[0] MPI startup(): 191     33312    cpu-e-958   31
[0] MPI startup(): 192     33508    cpu-e-1021  0
[0] MPI startup(): 193     33509    cpu-e-1021  1
[0] MPI startup(): 194     33510    cpu-e-1021  2
[0] MPI startup(): 195     33511    cpu-e-1021  3
[0] MPI startup(): 196     33512    cpu-e-1021  4
[0] MPI startup(): 197     33513    cpu-e-1021  5
[0] MPI startup(): 198     33514    cpu-e-1021  6
[0] MPI startup(): 199     33515    cpu-e-1021  7
[0] MPI startup(): 200     33516    cpu-e-1021  8
[0] MPI startup(): 201     33517    cpu-e-1021  9
[0] MPI startup(): 202     33518    cpu-e-1021  10
[0] MPI startup(): 203     33519    cpu-e-1021  11
[0] MPI startup(): 204     33520    cpu-e-1021  12
[0] MPI startup(): 205     33521    cpu-e-1021  13
[0] MPI startup(): 206     33522    cpu-e-1021  14
[0] MPI startup(): 207     33523    cpu-e-1021  15
[0] MPI startup(): 208     33524    cpu-e-1021  16
[0] MPI startup(): 209     33525    cpu-e-1021  17
[0] MPI startup(): 210     33526    cpu-e-1021  18
[0] MPI startup(): 211     33527    cpu-e-1021  19
[0] MPI startup(): 212     33528    cpu-e-1021  20
[0] MPI startup(): 213     33529    cpu-e-1021  21
[0] MPI startup(): 214     33530    cpu-e-1021  22
[0] MPI startup(): 215     33531    cpu-e-1021  23
[0] MPI startup(): 216     33532    cpu-e-1021  24
[0] MPI startup(): 217     33533    cpu-e-1021  25
[0] MPI startup(): 218     33534    cpu-e-1021  26
[0] MPI startup(): 219     33535    cpu-e-1021  27
[0] MPI startup(): 220     33536    cpu-e-1021  28
[0] MPI startup(): 221     33537    cpu-e-1021  29
[0] MPI startup(): 222     33538    cpu-e-1021  30
[0] MPI startup(): 223     33539    cpu-e-1021  31
[0] MPI startup(): 224     33281    cpu-e-1022  0
[0] MPI startup(): 225     33282    cpu-e-1022  1
[0] MPI startup(): 226     33283    cpu-e-1022  2
[0] MPI startup(): 227     33284    cpu-e-1022  3
[0] MPI startup(): 228     33285    cpu-e-1022  4
[0] MPI startup(): 229     33286    cpu-e-1022  5
[0] MPI startup(): 230     33287    cpu-e-1022  6
[0] MPI startup(): 231     33288    cpu-e-1022  7
[0] MPI startup(): 232     33289    cpu-e-1022  8
[0] MPI startup(): 233     33290    cpu-e-1022  9
[0] MPI startup(): 234     33291    cpu-e-1022  10
[0] MPI startup(): 235     33292    cpu-e-1022  11
[0] MPI startup(): 236     33293    cpu-e-1022  12
[0] MPI startup(): 237     33294    cpu-e-1022  13
[0] MPI startup(): 238     33295    cpu-e-1022  14
[0] MPI startup(): 239     33296    cpu-e-1022  15
[0] MPI startup(): 240     33297    cpu-e-1022  16
[0] MPI startup(): 241     33298    cpu-e-1022  17
[0] MPI startup(): 242     33299    cpu-e-1022  18
[0] MPI startup(): 243     33300    cpu-e-1022  19
[0] MPI startup(): 244     33301    cpu-e-1022  20
[0] MPI startup(): 245     33302    cpu-e-1022  21
[0] MPI startup(): 246     33303    cpu-e-1022  22
[0] MPI startup(): 247     33304    cpu-e-1022  23
[0] MPI startup(): 248     33305    cpu-e-1022  24
[0] MPI startup(): 249     33306    cpu-e-1022  25
[0] MPI startup(): 250     33307    cpu-e-1022  26
[0] MPI startup(): 251     33308    cpu-e-1022  27
[0] MPI startup(): 252     33309    cpu-e-1022  28
[0] MPI startup(): 253     33310    cpu-e-1022  29
[0] MPI startup(): 254     33311    cpu-e-1022  30
[0] MPI startup(): 255     33312    cpu-e-1022  31
[0] MPI startup(): 256     33295    cpu-e-1149  0
[0] MPI startup(): 257     33296    cpu-e-1149  1
[0] MPI startup(): 258     33297    cpu-e-1149  2
[0] MPI startup(): 259     33298    cpu-e-1149  3
[0] MPI startup(): 260     33299    cpu-e-1149  4
[0] MPI startup(): 261     33300    cpu-e-1149  5
[0] MPI startup(): 262     33301    cpu-e-1149  6
[0] MPI startup(): 263     33302    cpu-e-1149  7
[0] MPI startup(): 264     33303    cpu-e-1149  8
[0] MPI startup(): 265     33304    cpu-e-1149  9
[0] MPI startup(): 266     33305    cpu-e-1149  10
[0] MPI startup(): 267     33306    cpu-e-1149  11
[0] MPI startup(): 268     33307    cpu-e-1149  12
[0] MPI startup(): 269     33308    cpu-e-1149  13
[0] MPI startup(): 270     33309    cpu-e-1149  14
[0] MPI startup(): 271     33310    cpu-e-1149  15
[0] MPI startup(): 272     33311    cpu-e-1149  16
[0] MPI startup(): 273     33312    cpu-e-1149  17
[0] MPI startup(): 274     33313    cpu-e-1149  18
[0] MPI startup(): 275     33314    cpu-e-1149  19
[0] MPI startup(): 276     33315    cpu-e-1149  20
[0] MPI startup(): 277     33316    cpu-e-1149  21
[0] MPI startup(): 278     33317    cpu-e-1149  22
[0] MPI startup(): 279     33318    cpu-e-1149  23
[0] MPI startup(): 280     33319    cpu-e-1149  24
[0] MPI startup(): 281     33320    cpu-e-1149  25
[0] MPI startup(): 282     33321    cpu-e-1149  26
[0] MPI startup(): 283     33322    cpu-e-1149  27
[0] MPI startup(): 284     33323    cpu-e-1149  28
[0] MPI startup(): 285     33324    cpu-e-1149  29
[0] MPI startup(): 286     33325    cpu-e-1149  30
[0] MPI startup(): 287     33326    cpu-e-1149  31
[0] MPI startup(): 288     33333    cpu-e-1150  0
[0] MPI startup(): 289     33334    cpu-e-1150  1
[0] MPI startup(): 290     33335    cpu-e-1150  2
[0] MPI startup(): 291     33336    cpu-e-1150  3
[0] MPI startup(): 292     33337    cpu-e-1150  4
[0] MPI startup(): 293     33338    cpu-e-1150  5
[0] MPI startup(): 294     33339    cpu-e-1150  6
[0] MPI startup(): 295     33340    cpu-e-1150  7
[0] MPI startup(): 296     33341    cpu-e-1150  8
[0] MPI startup(): 297     33342    cpu-e-1150  9
[0] MPI startup(): 298     33343    cpu-e-1150  10
[0] MPI startup(): 299     33344    cpu-e-1150  11
[0] MPI startup(): 300     33345    cpu-e-1150  12
[0] MPI startup(): 301     33346    cpu-e-1150  13
[0] MPI startup(): 302     33347    cpu-e-1150  14
[0] MPI startup(): 303     33348    cpu-e-1150  15
[0] MPI startup(): 304     33349    cpu-e-1150  16
[0] MPI startup(): 305     33350    cpu-e-1150  17
[0] MPI startup(): 306     33351    cpu-e-1150  18
[0] MPI startup(): 307     33352    cpu-e-1150  19
[0] MPI startup(): 308     33353    cpu-e-1150  20
[0] MPI startup(): 309     33354    cpu-e-1150  21
[0] MPI startup(): 310     33355    cpu-e-1150  22
[0] MPI startup(): 311     33356    cpu-e-1150  23
[0] MPI startup(): 312     33357    cpu-e-1150  24
[0] MPI startup(): 313     33358    cpu-e-1150  25
[0] MPI startup(): 314     33359    cpu-e-1150  26
[0] MPI startup(): 315     33360    cpu-e-1150  27
[0] MPI startup(): 316     33361    cpu-e-1150  28
[0] MPI startup(): 317     33362    cpu-e-1150  29
[0] MPI startup(): 318     33363    cpu-e-1150  30
[0] MPI startup(): 319     33364    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Thu Nov  7 23:52:51 2019
Command line        : /home/wjt27/io-500-sc19/bin/ior -w -a=POSIX --posix.odirect -t 16m -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_easy/ior_file_easy -O stoneWallingStatusFile=/dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux cpu-e-829
TestID              : 0
StartTime           : Thu Nov  7 23:52:51 2019
Path                : /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_easy
FS                  : 405.5 TiB   Used FS: 13.9%   Inodes: 1638.8 Mi   Used Inodes: 0.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 320
clients per node    : 32
repetitions         : 1
xfersize            : 16 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 3027.34 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 8125 max: 9919 -- min data: 127.0 GiB mean data: 143.8 GiB time: 300.1s
WARNING: Expected aggregate file size       = 3328599654400000.
WARNING: Stat() of aggregate file size      = 53252225761280.
WARNING: Using actual aggregate bytes moved = 53252225761280.
WARNING: maybe caused by deadlineForStonewalling
write     145819     9115       0.000517    10158080000 16384      0.035278   348.24     0.018039   348.28     0   
Max Write: 145818.62 MiB/sec (152901.90 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write      145818.62  145818.62  145818.62       0.00    9113.66    9113.66    9113.66       0.00  348.27706     300.06     156991.63     0    320  32    1   1     1        1         0    0      1 10401873920000 16777216 50785280.0 POSIX      0
Finished            : Thu Nov  7 23:58:39 2019
ior_hard_read
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       36330    cpu-e-829   0
[0] MPI startup(): 1       36331    cpu-e-829   1
[0] MPI startup(): 2       36332    cpu-e-829   2
[0] MPI startup(): 3       36333    cpu-e-829   3
[0] MPI startup(): 4       36334    cpu-e-829   4
[0] MPI startup(): 5       36335    cpu-e-829   5
[0] MPI startup(): 6       36336    cpu-e-829   6
[0] MPI startup(): 7       36337    cpu-e-829   7
[0] MPI startup(): 8       36338    cpu-e-829   8
[0] MPI startup(): 9       36339    cpu-e-829   9
[0] MPI startup(): 10      36340    cpu-e-829   10
[0] MPI startup(): 11      36341    cpu-e-829   11
[0] MPI startup(): 12      36342    cpu-e-829   12
[0] MPI startup(): 13      36343    cpu-e-829   13
[0] MPI startup(): 14      36344    cpu-e-829   14
[0] MPI startup(): 15      36345    cpu-e-829   15
[0] MPI startup(): 16      36346    cpu-e-829   16
[0] MPI startup(): 17      36347    cpu-e-829   17
[0] MPI startup(): 18      36348    cpu-e-829   18
[0] MPI startup(): 19      36349    cpu-e-829   19
[0] MPI startup(): 20      36350    cpu-e-829   20
[0] MPI startup(): 21      36351    cpu-e-829   21
[0] MPI startup(): 22      36352    cpu-e-829   22
[0] MPI startup(): 23      36353    cpu-e-829   23
[0] MPI startup(): 24      36354    cpu-e-829   24
[0] MPI startup(): 25      36355    cpu-e-829   25
[0] MPI startup(): 26      36356    cpu-e-829   26
[0] MPI startup(): 27      36357    cpu-e-829   27
[0] MPI startup(): 28      36358    cpu-e-829   28
[0] MPI startup(): 29      36359    cpu-e-829   29
[0] MPI startup(): 30      36360    cpu-e-829   30
[0] MPI startup(): 31      36361    cpu-e-829   31
[0] MPI startup(): 32      35548    cpu-e-830   0
[0] MPI startup(): 33      35549    cpu-e-830   1
[0] MPI startup(): 34      35550    cpu-e-830   2
[0] MPI startup(): 35      35551    cpu-e-830   3
[0] MPI startup(): 36      35552    cpu-e-830   4
[0] MPI startup(): 37      35553    cpu-e-830   5
[0] MPI startup(): 38      35554    cpu-e-830   6
[0] MPI startup(): 39      35555    cpu-e-830   7
[0] MPI startup(): 40      35556    cpu-e-830   8
[0] MPI startup(): 41      35557    cpu-e-830   9
[0] MPI startup(): 42      35558    cpu-e-830   10
[0] MPI startup(): 43      35559    cpu-e-830   11
[0] MPI startup(): 44      35560    cpu-e-830   12
[0] MPI startup(): 45      35561    cpu-e-830   13
[0] MPI startup(): 46      35562    cpu-e-830   14
[0] MPI startup(): 47      35563    cpu-e-830   15
[0] MPI startup(): 48      35564    cpu-e-830   16
[0] MPI startup(): 49      35565    cpu-e-830   17
[0] MPI startup(): 50      35566    cpu-e-830   18
[0] MPI startup(): 51      35567    cpu-e-830   19
[0] MPI startup(): 52      35568    cpu-e-830   20
[0] MPI startup(): 53      35569    cpu-e-830   21
[0] MPI startup(): 54      35570    cpu-e-830   22
[0] MPI startup(): 55      35571    cpu-e-830   23
[0] MPI startup(): 56      35572    cpu-e-830   24
[0] MPI startup(): 57      35573    cpu-e-830   25
[0] MPI startup(): 58      35574    cpu-e-830   26
[0] MPI startup(): 59      35575    cpu-e-830   27
[0] MPI startup(): 60      35576    cpu-e-830   28
[0] MPI startup(): 61      35577    cpu-e-830   29
[0] MPI startup(): 62      35578    cpu-e-830   30
[0] MPI startup(): 63      35579    cpu-e-830   31
[0] MPI startup(): 64      35260    cpu-e-893   0
[0] MPI startup(): 65      35261    cpu-e-893   1
[0] MPI startup(): 66      35262    cpu-e-893   2
[0] MPI startup(): 67      35263    cpu-e-893   3
[0] MPI startup(): 68      35264    cpu-e-893   4
[0] MPI startup(): 69      35265    cpu-e-893   5
[0] MPI startup(): 70      35266    cpu-e-893   6
[0] MPI startup(): 71      35267    cpu-e-893   7
[0] MPI startup(): 72      35268    cpu-e-893   8
[0] MPI startup(): 73      35269    cpu-e-893   9
[0] MPI startup(): 74      35270    cpu-e-893   10
[0] MPI startup(): 75      35271    cpu-e-893   11
[0] MPI startup(): 76      35272    cpu-e-893   12
[0] MPI startup(): 77      35273    cpu-e-893   13
[0] MPI startup(): 78      35274    cpu-e-893   14
[0] MPI startup(): 79      35275    cpu-e-893   15
[0] MPI startup(): 80      35276    cpu-e-893   16
[0] MPI startup(): 81      35277    cpu-e-893   17
[0] MPI startup(): 82      35278    cpu-e-893   18
[0] MPI startup(): 83      35279    cpu-e-893   19
[0] MPI startup(): 84      35280    cpu-e-893   20
[0] MPI startup(): 85      35281    cpu-e-893   21
[0] MPI startup(): 86      35282    cpu-e-893   22
[0] MPI startup(): 87      35283    cpu-e-893   23
[0] MPI startup(): 88      35284    cpu-e-893   24
[0] MPI startup(): 89      35285    cpu-e-893   25
[0] MPI startup(): 90      35286    cpu-e-893   26
[0] MPI startup(): 91      35287    cpu-e-893   27
[0] MPI startup(): 92      35288    cpu-e-893   28
[0] MPI startup(): 93      35289    cpu-e-893   29
[0] MPI startup(): 94      35290    cpu-e-893   30
[0] MPI startup(): 95      35291    cpu-e-893   31
[0] MPI startup(): 96      35250    cpu-e-894   0
[0] MPI startup(): 97      35251    cpu-e-894   1
[0] MPI startup(): 98      35252    cpu-e-894   2
[0] MPI startup(): 99      35253    cpu-e-894   3
[0] MPI startup(): 100     35254    cpu-e-894   4
[0] MPI startup(): 101     35255    cpu-e-894   5
[0] MPI startup(): 102     35256    cpu-e-894   6
[0] MPI startup(): 103     35257    cpu-e-894   7
[0] MPI startup(): 104     35258    cpu-e-894   8
[0] MPI startup(): 105     35259    cpu-e-894   9
[0] MPI startup(): 106     35260    cpu-e-894   10
[0] MPI startup(): 107     35261    cpu-e-894   11
[0] MPI startup(): 108     35262    cpu-e-894   12
[0] MPI startup(): 109     35263    cpu-e-894   13
[0] MPI startup(): 110     35264    cpu-e-894   14
[0] MPI startup(): 111     35265    cpu-e-894   15
[0] MPI startup(): 112     35266    cpu-e-894   16
[0] MPI startup(): 113     35267    cpu-e-894   17
[0] MPI startup(): 114     35268    cpu-e-894   18
[0] MPI startup(): 115     35269    cpu-e-894   19
[0] MPI startup(): 116     35270    cpu-e-894   20
[0] MPI startup(): 117     35271    cpu-e-894   21
[0] MPI startup(): 118     35272    cpu-e-894   22
[0] MPI startup(): 119     35273    cpu-e-894   23
[0] MPI startup(): 120     35274    cpu-e-894   24
[0] MPI startup(): 121     35275    cpu-e-894   25
[0] MPI startup(): 122     35276    cpu-e-894   26
[0] MPI startup(): 123     35277    cpu-e-894   27
[0] MPI startup(): 124     35278    cpu-e-894   28
[0] MPI startup(): 125     35279    cpu-e-894   29
[0] MPI startup(): 126     35280    cpu-e-894   30
[0] MPI startup(): 127     35281    cpu-e-894   31
[0] MPI startup(): 128     35386    cpu-e-957   0
[0] MPI startup(): 129     35387    cpu-e-957   1
[0] MPI startup(): 130     35388    cpu-e-957   2
[0] MPI startup(): 131     35389    cpu-e-957   3
[0] MPI startup(): 132     35390    cpu-e-957   4
[0] MPI startup(): 133     35391    cpu-e-957   5
[0] MPI startup(): 134     35392    cpu-e-957   6
[0] MPI startup(): 135     35393    cpu-e-957   7
[0] MPI startup(): 136     35394    cpu-e-957   8
[0] MPI startup(): 137     35395    cpu-e-957   9
[0] MPI startup(): 138     35396    cpu-e-957   10
[0] MPI startup(): 139     35397    cpu-e-957   11
[0] MPI startup(): 140     35398    cpu-e-957   12
[0] MPI startup(): 141     35399    cpu-e-957   13
[0] MPI startup(): 142     35400    cpu-e-957   14
[0] MPI startup(): 143     35401    cpu-e-957   15
[0] MPI startup(): 144     35402    cpu-e-957   16
[0] MPI startup(): 145     35403    cpu-e-957   17
[0] MPI startup(): 146     35404    cpu-e-957   18
[0] MPI startup(): 147     35405    cpu-e-957   19
[0] MPI startup(): 148     35406    cpu-e-957   20
[0] MPI startup(): 149     35407    cpu-e-957   21
[0] MPI startup(): 150     35408    cpu-e-957   22
[0] MPI startup(): 151     35409    cpu-e-957   23
[0] MPI startup(): 152     35410    cpu-e-957   24
[0] MPI startup(): 153     35411    cpu-e-957   25
[0] MPI startup(): 154     35412    cpu-e-957   26
[0] MPI startup(): 155     35413    cpu-e-957   27
[0] MPI startup(): 156     35414    cpu-e-957   28
[0] MPI startup(): 157     35415    cpu-e-957   29
[0] MPI startup(): 158     35416    cpu-e-957   30
[0] MPI startup(): 159     35417    cpu-e-957   31
[0] MPI startup(): 160     35332    cpu-e-958   0
[0] MPI startup(): 161     35333    cpu-e-958   1
[0] MPI startup(): 162     35334    cpu-e-958   2
[0] MPI startup(): 163     35335    cpu-e-958   3
[0] MPI startup(): 164     35336    cpu-e-958   4
[0] MPI startup(): 165     35337    cpu-e-958   5
[0] MPI startup(): 166     35338    cpu-e-958   6
[0] MPI startup(): 167     35339    cpu-e-958   7
[0] MPI startup(): 168     35340    cpu-e-958   8
[0] MPI startup(): 169     35341    cpu-e-958   9
[0] MPI startup(): 170     35342    cpu-e-958   10
[0] MPI startup(): 171     35343    cpu-e-958   11
[0] MPI startup(): 172     35344    cpu-e-958   12
[0] MPI startup(): 173     35345    cpu-e-958   13
[0] MPI startup(): 174     35346    cpu-e-958   14
[0] MPI startup(): 175     35347    cpu-e-958   15
[0] MPI startup(): 176     35348    cpu-e-958   16
[0] MPI startup(): 177     35349    cpu-e-958   17
[0] MPI startup(): 178     35350    cpu-e-958   18
[0] MPI startup(): 179     35351    cpu-e-958   19
[0] MPI startup(): 180     35352    cpu-e-958   20
[0] MPI startup(): 181     35353    cpu-e-958   21
[0] MPI startup(): 182     35354    cpu-e-958   22
[0] MPI startup(): 183     35355    cpu-e-958   23
[0] MPI startup(): 184     35356    cpu-e-958   24
[0] MPI startup(): 185     35357    cpu-e-958   25
[0] MPI startup(): 186     35358    cpu-e-958   26
[0] MPI startup(): 187     35359    cpu-e-958   27
[0] MPI startup(): 188     35360    cpu-e-958   28
[0] MPI startup(): 189     35361    cpu-e-958   29
[0] MPI startup(): 190     35362    cpu-e-958   30
[0] MPI startup(): 191     35363    cpu-e-958   31
[0] MPI startup(): 192     35593    cpu-e-1021  0
[0] MPI startup(): 193     35594    cpu-e-1021  1
[0] MPI startup(): 194     35595    cpu-e-1021  2
[0] MPI startup(): 195     35596    cpu-e-1021  3
[0] MPI startup(): 196     35597    cpu-e-1021  4
[0] MPI startup(): 197     35598    cpu-e-1021  5
[0] MPI startup(): 198     35599    cpu-e-1021  6
[0] MPI startup(): 199     35600    cpu-e-1021  7
[0] MPI startup(): 200     35601    cpu-e-1021  8
[0] MPI startup(): 201     35602    cpu-e-1021  9
[0] MPI startup(): 202     35603    cpu-e-1021  10
[0] MPI startup(): 203     35604    cpu-e-1021  11
[0] MPI startup(): 204     35605    cpu-e-1021  12
[0] MPI startup(): 205     35606    cpu-e-1021  13
[0] MPI startup(): 206     35607    cpu-e-1021  14
[0] MPI startup(): 207     35608    cpu-e-1021  15
[0] MPI startup(): 208     35609    cpu-e-1021  16
[0] MPI startup(): 209     35610    cpu-e-1021  17
[0] MPI startup(): 210     35611    cpu-e-1021  18
[0] MPI startup(): 211     35612    cpu-e-1021  19
[0] MPI startup(): 212     35613    cpu-e-1021  20
[0] MPI startup(): 213     35614    cpu-e-1021  21
[0] MPI startup(): 214     35615    cpu-e-1021  22
[0] MPI startup(): 215     35616    cpu-e-1021  23
[0] MPI startup(): 216     35617    cpu-e-1021  24
[0] MPI startup(): 217     35618    cpu-e-1021  25
[0] MPI startup(): 218     35619    cpu-e-1021  26
[0] MPI startup(): 219     35620    cpu-e-1021  27
[0] MPI startup(): 220     35621    cpu-e-1021  28
[0] MPI startup(): 221     35622    cpu-e-1021  29
[0] MPI startup(): 222     35623    cpu-e-1021  30
[0] MPI startup(): 223     35624    cpu-e-1021  31
[0] MPI startup(): 224     35366    cpu-e-1022  0
[0] MPI startup(): 225     35367    cpu-e-1022  1
[0] MPI startup(): 226     35368    cpu-e-1022  2
[0] MPI startup(): 227     35369    cpu-e-1022  3
[0] MPI startup(): 228     35370    cpu-e-1022  4
[0] MPI startup(): 229     35371    cpu-e-1022  5
[0] MPI startup(): 230     35372    cpu-e-1022  6
[0] MPI startup(): 231     35373    cpu-e-1022  7
[0] MPI startup(): 232     35374    cpu-e-1022  8
[0] MPI startup(): 233     35375    cpu-e-1022  9
[0] MPI startup(): 234     35376    cpu-e-1022  10
[0] MPI startup(): 235     35377    cpu-e-1022  11
[0] MPI startup(): 236     35378    cpu-e-1022  12
[0] MPI startup(): 237     35379    cpu-e-1022  13
[0] MPI startup(): 238     35380    cpu-e-1022  14
[0] MPI startup(): 239     35381    cpu-e-1022  15
[0] MPI startup(): 240     35382    cpu-e-1022  16
[0] MPI startup(): 241     35383    cpu-e-1022  17
[0] MPI startup(): 242     35384    cpu-e-1022  18
[0] MPI startup(): 243     35385    cpu-e-1022  19
[0] MPI startup(): 244     35386    cpu-e-1022  20
[0] MPI startup(): 245     35387    cpu-e-1022  21
[0] MPI startup(): 246     35388    cpu-e-1022  22
[0] MPI startup(): 247     35389    cpu-e-1022  23
[0] MPI startup(): 248     35390    cpu-e-1022  24
[0] MPI startup(): 249     35391    cpu-e-1022  25
[0] MPI startup(): 250     35392    cpu-e-1022  26
[0] MPI startup(): 251     35393    cpu-e-1022  27
[0] MPI startup(): 252     35394    cpu-e-1022  28
[0] MPI startup(): 253     35395    cpu-e-1022  29
[0] MPI startup(): 254     35396    cpu-e-1022  30
[0] MPI startup(): 255     35397    cpu-e-1022  31
[0] MPI startup(): 256     35388    cpu-e-1149  0
[0] MPI startup(): 257     35389    cpu-e-1149  1
[0] MPI startup(): 258     35390    cpu-e-1149  2
[0] MPI startup(): 259     35391    cpu-e-1149  3
[0] MPI startup(): 260     35392    cpu-e-1149  4
[0] MPI startup(): 261     35393    cpu-e-1149  5
[0] MPI startup(): 262     35394    cpu-e-1149  6
[0] MPI startup(): 263     35395    cpu-e-1149  7
[0] MPI startup(): 264     35396    cpu-e-1149  8
[0] MPI startup(): 265     35397    cpu-e-1149  9
[0] MPI startup(): 266     35398    cpu-e-1149  10
[0] MPI startup(): 267     35399    cpu-e-1149  11
[0] MPI startup(): 268     35400    cpu-e-1149  12
[0] MPI startup(): 269     35401    cpu-e-1149  13
[0] MPI startup(): 270     35402    cpu-e-1149  14
[0] MPI startup(): 271     35403    cpu-e-1149  15
[0] MPI startup(): 272     35404    cpu-e-1149  16
[0] MPI startup(): 273     35405    cpu-e-1149  17
[0] MPI startup(): 274     35406    cpu-e-1149  18
[0] MPI startup(): 275     35407    cpu-e-1149  19
[0] MPI startup(): 276     35408    cpu-e-1149  20
[0] MPI startup(): 277     35409    cpu-e-1149  21
[0] MPI startup(): 278     35410    cpu-e-1149  22
[0] MPI startup(): 279     35411    cpu-e-1149  23
[0] MPI startup(): 280     35412    cpu-e-1149  24
[0] MPI startup(): 281     35413    cpu-e-1149  25
[0] MPI startup(): 282     35414    cpu-e-1149  26
[0] MPI startup(): 283     35415    cpu-e-1149  27
[0] MPI startup(): 284     35416    cpu-e-1149  28
[0] MPI startup(): 285     35417    cpu-e-1149  29
[0] MPI startup(): 286     35418    cpu-e-1149  30
[0] MPI startup(): 287     35419    cpu-e-1149  31
[0] MPI startup(): 288     35455    cpu-e-1150  0
[0] MPI startup(): 289     35456    cpu-e-1150  1
[0] MPI startup(): 290     35457    cpu-e-1150  2
[0] MPI startup(): 291     35458    cpu-e-1150  3
[0] MPI startup(): 292     35459    cpu-e-1150  4
[0] MPI startup(): 293     35460    cpu-e-1150  5
[0] MPI startup(): 294     35461    cpu-e-1150  6
[0] MPI startup(): 295     35462    cpu-e-1150  7
[0] MPI startup(): 296     35463    cpu-e-1150  8
[0] MPI startup(): 297     35464    cpu-e-1150  9
[0] MPI startup(): 298     35465    cpu-e-1150  10
[0] MPI startup(): 299     35466    cpu-e-1150  11
[0] MPI startup(): 300     35467    cpu-e-1150  12
[0] MPI startup(): 301     35468    cpu-e-1150  13
[0] MPI startup(): 302     35469    cpu-e-1150  14
[0] MPI startup(): 303     35470    cpu-e-1150  15
[0] MPI startup(): 304     35471    cpu-e-1150  16
[0] MPI startup(): 305     35472    cpu-e-1150  17
[0] MPI startup(): 306     35473    cpu-e-1150  18
[0] MPI startup(): 307     35474    cpu-e-1150  19
[0] MPI startup(): 308     35475    cpu-e-1150  20
[0] MPI startup(): 309     35476    cpu-e-1150  21
[0] MPI startup(): 310     35477    cpu-e-1150  22
[0] MPI startup(): 311     35478    cpu-e-1150  23
[0] MPI startup(): 312     35479    cpu-e-1150  24
[0] MPI startup(): 313     35480    cpu-e-1150  25
[0] MPI startup(): 314     35481    cpu-e-1150  26
[0] MPI startup(): 315     35482    cpu-e-1150  27
[0] MPI startup(): 316     35483    cpu-e-1150  28
[0] MPI startup(): 317     35484    cpu-e-1150  29
[0] MPI startup(): 318     35485    cpu-e-1150  30
[0] MPI startup(): 319     35486    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Nov  8 00:35:53 2019
Command line        : /home/wjt27/io-500-sc19/bin/ior -r -R -s 1900000 -a MPIIO -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_hard/IOR_file -O stoneWallingStatusFile=/dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_hard/stonewall
Machine             : Linux cpu-e-829
TestID              : 0
StartTime           : Fri Nov  8 00:35:53 2019
Path                : /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_hard
FS                  : 405.5 TiB   Used FS: 26.5%   Inodes: 1820.8 Mi   Used Inodes: 10.5%

Options: 
api                 : MPIIO
apiVersion          : (3.1)
test filename       : /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 320
clients per node    : 32
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 25.99 TiB

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 28580864000000.
WARNING: Stat() of aggregate file size      = 2733579130880.
WARNING: Using actual aggregate bytes moved = 2733579130880.
read      4735       105729     549.99      45.91      45.91      0.460130   550.00     0.168720   550.59     0   
Max Read:  4734.81 MiB/sec (4964.81 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read         4734.81    4734.81    4734.81       0.00  105616.29  105616.29  105616.29       0.00  550.59081         NA            NA     0    320  32    1   0     1        1         0    0 1900000    47008    47008 2606944.2 MPIIO      0
Finished            : Fri Nov  8 00:45:10 2019
ior_hard_write
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       34611    cpu-e-829   0
[0] MPI startup(): 1       34612    cpu-e-829   1
[0] MPI startup(): 2       34613    cpu-e-829   2
[0] MPI startup(): 3       34614    cpu-e-829   3
[0] MPI startup(): 4       34615    cpu-e-829   4
[0] MPI startup(): 5       34616    cpu-e-829   5
[0] MPI startup(): 6       34617    cpu-e-829   6
[0] MPI startup(): 7       34618    cpu-e-829   7
[0] MPI startup(): 8       34619    cpu-e-829   8
[0] MPI startup(): 9       34620    cpu-e-829   9
[0] MPI startup(): 10      34621    cpu-e-829   10
[0] MPI startup(): 11      34622    cpu-e-829   11
[0] MPI startup(): 12      34623    cpu-e-829   12
[0] MPI startup(): 13      34624    cpu-e-829   13
[0] MPI startup(): 14      34625    cpu-e-829   14
[0] MPI startup(): 15      34626    cpu-e-829   15
[0] MPI startup(): 16      34627    cpu-e-829   16
[0] MPI startup(): 17      34628    cpu-e-829   17
[0] MPI startup(): 18      34629    cpu-e-829   18
[0] MPI startup(): 19      34630    cpu-e-829   19
[0] MPI startup(): 20      34631    cpu-e-829   20
[0] MPI startup(): 21      34632    cpu-e-829   21
[0] MPI startup(): 22      34633    cpu-e-829   22
[0] MPI startup(): 23      34634    cpu-e-829   23
[0] MPI startup(): 24      34635    cpu-e-829   24
[0] MPI startup(): 25      34636    cpu-e-829   25
[0] MPI startup(): 26      34637    cpu-e-829   26
[0] MPI startup(): 27      34638    cpu-e-829   27
[0] MPI startup(): 28      34639    cpu-e-829   28
[0] MPI startup(): 29      34640    cpu-e-829   29
[0] MPI startup(): 30      34641    cpu-e-829   30
[0] MPI startup(): 31      34642    cpu-e-829   31
[0] MPI startup(): 32      34056    cpu-e-830   0
[0] MPI startup(): 33      34057    cpu-e-830   1
[0] MPI startup(): 34      34058    cpu-e-830   2
[0] MPI startup(): 35      34059    cpu-e-830   3
[0] MPI startup(): 36      34060    cpu-e-830   4
[0] MPI startup(): 37      34061    cpu-e-830   5
[0] MPI startup(): 38      34062    cpu-e-830   6
[0] MPI startup(): 39      34063    cpu-e-830   7
[0] MPI startup(): 40      34064    cpu-e-830   8
[0] MPI startup(): 41      34065    cpu-e-830   9
[0] MPI startup(): 42      34066    cpu-e-830   10
[0] MPI startup(): 43      34067    cpu-e-830   11
[0] MPI startup(): 44      34068    cpu-e-830   12
[0] MPI startup(): 45      34069    cpu-e-830   13
[0] MPI startup(): 46      34070    cpu-e-830   14
[0] MPI startup(): 47      34071    cpu-e-830   15
[0] MPI startup(): 48      34072    cpu-e-830   16
[0] MPI startup(): 49      34073    cpu-e-830   17
[0] MPI startup(): 50      34074    cpu-e-830   18
[0] MPI startup(): 51      34075    cpu-e-830   19
[0] MPI startup(): 52      34076    cpu-e-830   20
[0] MPI startup(): 53      34077    cpu-e-830   21
[0] MPI startup(): 54      34078    cpu-e-830   22
[0] MPI startup(): 55      34079    cpu-e-830   23
[0] MPI startup(): 56      34080    cpu-e-830   24
[0] MPI startup(): 57      34081    cpu-e-830   25
[0] MPI startup(): 58      34082    cpu-e-830   26
[0] MPI startup(): 59      34083    cpu-e-830   27
[0] MPI startup(): 60      34084    cpu-e-830   28
[0] MPI startup(): 61      34085    cpu-e-830   29
[0] MPI startup(): 62      34086    cpu-e-830   30
[0] MPI startup(): 63      34087    cpu-e-830   31
[0] MPI startup(): 64      33866    cpu-e-893   0
[0] MPI startup(): 65      33867    cpu-e-893   1
[0] MPI startup(): 66      33868    cpu-e-893   2
[0] MPI startup(): 67      33869    cpu-e-893   3
[0] MPI startup(): 68      33870    cpu-e-893   4
[0] MPI startup(): 69      33871    cpu-e-893   5
[0] MPI startup(): 70      33872    cpu-e-893   6
[0] MPI startup(): 71      33873    cpu-e-893   7
[0] MPI startup(): 72      33874    cpu-e-893   8
[0] MPI startup(): 73      33875    cpu-e-893   9
[0] MPI startup(): 74      33876    cpu-e-893   10
[0] MPI startup(): 75      33877    cpu-e-893   11
[0] MPI startup(): 76      33878    cpu-e-893   12
[0] MPI startup(): 77      33879    cpu-e-893   13
[0] MPI startup(): 78      33880    cpu-e-893   14
[0] MPI startup(): 79      33881    cpu-e-893   15
[0] MPI startup(): 80      33882    cpu-e-893   16
[0] MPI startup(): 81      33883    cpu-e-893   17
[0] MPI startup(): 82      33884    cpu-e-893   18
[0] MPI startup(): 83      33885    cpu-e-893   19
[0] MPI startup(): 84      33886    cpu-e-893   20
[0] MPI startup(): 85      33887    cpu-e-893   21
[0] MPI startup(): 86      33888    cpu-e-893   22
[0] MPI startup(): 87      33889    cpu-e-893   23
[0] MPI startup(): 88      33890    cpu-e-893   24
[0] MPI startup(): 89      33891    cpu-e-893   25
[0] MPI startup(): 90      33892    cpu-e-893   26
[0] MPI startup(): 91      33893    cpu-e-893   27
[0] MPI startup(): 92      33894    cpu-e-893   28
[0] MPI startup(): 93      33895    cpu-e-893   29
[0] MPI startup(): 94      33896    cpu-e-893   30
[0] MPI startup(): 95      33897    cpu-e-893   31
[0] MPI startup(): 96      33849    cpu-e-894   0
[0] MPI startup(): 97      33850    cpu-e-894   1
[0] MPI startup(): 98      33851    cpu-e-894   2
[0] MPI startup(): 99      33852    cpu-e-894   3
[0] MPI startup(): 100     33853    cpu-e-894   4
[0] MPI startup(): 101     33854    cpu-e-894   5
[0] MPI startup(): 102     33855    cpu-e-894   6
[0] MPI startup(): 103     33856    cpu-e-894   7
[0] MPI startup(): 104     33857    cpu-e-894   8
[0] MPI startup(): 105     33858    cpu-e-894   9
[0] MPI startup(): 106     33859    cpu-e-894   10
[0] MPI startup(): 107     33860    cpu-e-894   11
[0] MPI startup(): 108     33861    cpu-e-894   12
[0] MPI startup(): 109     33862    cpu-e-894   13
[0] MPI startup(): 110     33863    cpu-e-894   14
[0] MPI startup(): 111     33864    cpu-e-894   15
[0] MPI startup(): 112     33865    cpu-e-894   16
[0] MPI startup(): 113     33866    cpu-e-894   17
[0] MPI startup(): 114     33867    cpu-e-894   18
[0] MPI startup(): 115     33868    cpu-e-894   19
[0] MPI startup(): 116     33869    cpu-e-894   20
[0] MPI startup(): 117     33870    cpu-e-894   21
[0] MPI startup(): 118     33871    cpu-e-894   22
[0] MPI startup(): 119     33872    cpu-e-894   23
[0] MPI startup(): 120     33873    cpu-e-894   24
[0] MPI startup(): 121     33874    cpu-e-894   25
[0] MPI startup(): 122     33875    cpu-e-894   26
[0] MPI startup(): 123     33876    cpu-e-894   27
[0] MPI startup(): 124     33877    cpu-e-894   28
[0] MPI startup(): 125     33878    cpu-e-894   29
[0] MPI startup(): 126     33879    cpu-e-894   30
[0] MPI startup(): 127     33880    cpu-e-894   31
[0] MPI startup(): 128     33979    cpu-e-957   0
[0] MPI startup(): 129     33980    cpu-e-957   1
[0] MPI startup(): 130     33981    cpu-e-957   2
[0] MPI startup(): 131     33982    cpu-e-957   3
[0] MPI startup(): 132     33983    cpu-e-957   4
[0] MPI startup(): 133     33984    cpu-e-957   5
[0] MPI startup(): 134     33985    cpu-e-957   6
[0] MPI startup(): 135     33986    cpu-e-957   7
[0] MPI startup(): 136     33987    cpu-e-957   8
[0] MPI startup(): 137     33988    cpu-e-957   9
[0] MPI startup(): 138     33989    cpu-e-957   10
[0] MPI startup(): 139     33990    cpu-e-957   11
[0] MPI startup(): 140     33991    cpu-e-957   12
[0] MPI startup(): 141     33992    cpu-e-957   13
[0] MPI startup(): 142     33993    cpu-e-957   14
[0] MPI startup(): 143     33994    cpu-e-957   15
[0] MPI startup(): 144     33995    cpu-e-957   16
[0] MPI startup(): 145     33996    cpu-e-957   17
[0] MPI startup(): 146     33997    cpu-e-957   18
[0] MPI startup(): 147     33998    cpu-e-957   19
[0] MPI startup(): 148     33999    cpu-e-957   20
[0] MPI startup(): 149     34000    cpu-e-957   21
[0] MPI startup(): 150     34001    cpu-e-957   22
[0] MPI startup(): 151     34002    cpu-e-957   23
[0] MPI startup(): 152     34003    cpu-e-957   24
[0] MPI startup(): 153     34004    cpu-e-957   25
[0] MPI startup(): 154     34005    cpu-e-957   26
[0] MPI startup(): 155     34006    cpu-e-957   27
[0] MPI startup(): 156     34007    cpu-e-957   28
[0] MPI startup(): 157     34008    cpu-e-957   29
[0] MPI startup(): 158     34009    cpu-e-957   30
[0] MPI startup(): 159     34010    cpu-e-957   31
[0] MPI startup(): 160     33936    cpu-e-958   0
[0] MPI startup(): 161     33937    cpu-e-958   1
[0] MPI startup(): 162     33938    cpu-e-958   2
[0] MPI startup(): 163     33939    cpu-e-958   3
[0] MPI startup(): 164     33940    cpu-e-958   4
[0] MPI startup(): 165     33941    cpu-e-958   5
[0] MPI startup(): 166     33942    cpu-e-958   6
[0] MPI startup(): 167     33943    cpu-e-958   7
[0] MPI startup(): 168     33944    cpu-e-958   8
[0] MPI startup(): 169     33945    cpu-e-958   9
[0] MPI startup(): 170     33946    cpu-e-958   10
[0] MPI startup(): 171     33947    cpu-e-958   11
[0] MPI startup(): 172     33948    cpu-e-958   12
[0] MPI startup(): 173     33949    cpu-e-958   13
[0] MPI startup(): 174     33950    cpu-e-958   14
[0] MPI startup(): 175     33951    cpu-e-958   15
[0] MPI startup(): 176     33952    cpu-e-958   16
[0] MPI startup(): 177     33953    cpu-e-958   17
[0] MPI startup(): 178     33954    cpu-e-958   18
[0] MPI startup(): 179     33955    cpu-e-958   19
[0] MPI startup(): 180     33956    cpu-e-958   20
[0] MPI startup(): 181     33957    cpu-e-958   21
[0] MPI startup(): 182     33958    cpu-e-958   22
[0] MPI startup(): 183     33959    cpu-e-958   23
[0] MPI startup(): 184     33960    cpu-e-958   24
[0] MPI startup(): 185     33961    cpu-e-958   25
[0] MPI startup(): 186     33962    cpu-e-958   26
[0] MPI startup(): 187     33963    cpu-e-958   27
[0] MPI startup(): 188     33964    cpu-e-958   28
[0] MPI startup(): 189     33965    cpu-e-958   29
[0] MPI startup(): 190     33966    cpu-e-958   30
[0] MPI startup(): 191     33967    cpu-e-958   31
[0] MPI startup(): 192     34162    cpu-e-1021  0
[0] MPI startup(): 193     34163    cpu-e-1021  1
[0] MPI startup(): 194     34164    cpu-e-1021  2
[0] MPI startup(): 195     34165    cpu-e-1021  3
[0] MPI startup(): 196     34166    cpu-e-1021  4
[0] MPI startup(): 197     34167    cpu-e-1021  5
[0] MPI startup(): 198     34168    cpu-e-1021  6
[0] MPI startup(): 199     34169    cpu-e-1021  7
[0] MPI startup(): 200     34170    cpu-e-1021  8
[0] MPI startup(): 201     34171    cpu-e-1021  9
[0] MPI startup(): 202     34172    cpu-e-1021  10
[0] MPI startup(): 203     34173    cpu-e-1021  11
[0] MPI startup(): 204     34174    cpu-e-1021  12
[0] MPI startup(): 205     34175    cpu-e-1021  13
[0] MPI startup(): 206     34176    cpu-e-1021  14
[0] MPI startup(): 207     34177    cpu-e-1021  15
[0] MPI startup(): 208     34178    cpu-e-1021  16
[0] MPI startup(): 209     34179    cpu-e-1021  17
[0] MPI startup(): 210     34180    cpu-e-1021  18
[0] MPI startup(): 211     34181    cpu-e-1021  19
[0] MPI startup(): 212     34182    cpu-e-1021  20
[0] MPI startup(): 213     34183    cpu-e-1021  21
[0] MPI startup(): 214     34184    cpu-e-1021  22
[0] MPI startup(): 215     34185    cpu-e-1021  23
[0] MPI startup(): 216     34186    cpu-e-1021  24
[0] MPI startup(): 217     34187    cpu-e-1021  25
[0] MPI startup(): 218     34188    cpu-e-1021  26
[0] MPI startup(): 219     34189    cpu-e-1021  27
[0] MPI startup(): 220     34190    cpu-e-1021  28
[0] MPI startup(): 221     34191    cpu-e-1021  29
[0] MPI startup(): 222     34192    cpu-e-1021  30
[0] MPI startup(): 223     34193    cpu-e-1021  31
[0] MPI startup(): 224     33942    cpu-e-1022  0
[0] MPI startup(): 225     33943    cpu-e-1022  1
[0] MPI startup(): 226     33944    cpu-e-1022  2
[0] MPI startup(): 227     33945    cpu-e-1022  3
[0] MPI startup(): 228     33946    cpu-e-1022  4
[0] MPI startup(): 229     33947    cpu-e-1022  5
[0] MPI startup(): 230     33948    cpu-e-1022  6
[0] MPI startup(): 231     33949    cpu-e-1022  7
[0] MPI startup(): 232     33950    cpu-e-1022  8
[0] MPI startup(): 233     33951    cpu-e-1022  9
[0] MPI startup(): 234     33952    cpu-e-1022  10
[0] MPI startup(): 235     33953    cpu-e-1022  11
[0] MPI startup(): 236     33954    cpu-e-1022  12
[0] MPI startup(): 237     33955    cpu-e-1022  13
[0] MPI startup(): 238     33956    cpu-e-1022  14
[0] MPI startup(): 239     33957    cpu-e-1022  15
[0] MPI startup(): 240     33958    cpu-e-1022  16
[0] MPI startup(): 241     33959    cpu-e-1022  17
[0] MPI startup(): 242     33960    cpu-e-1022  18
[0] MPI startup(): 243     33961    cpu-e-1022  19
[0] MPI startup(): 244     33962    cpu-e-1022  20
[0] MPI startup(): 245     33963    cpu-e-1022  21
[0] MPI startup(): 246     33964    cpu-e-1022  22
[0] MPI startup(): 247     33965    cpu-e-1022  23
[0] MPI startup(): 248     33966    cpu-e-1022  24
[0] MPI startup(): 249     33967    cpu-e-1022  25
[0] MPI startup(): 250     33968    cpu-e-1022  26
[0] MPI startup(): 251     33969    cpu-e-1022  27
[0] MPI startup(): 252     33970    cpu-e-1022  28
[0] MPI startup(): 253     33971    cpu-e-1022  29
[0] MPI startup(): 254     33972    cpu-e-1022  30
[0] MPI startup(): 255     33973    cpu-e-1022  31
[0] MPI startup(): 256     33963    cpu-e-1149  0
[0] MPI startup(): 257     33964    cpu-e-1149  1
[0] MPI startup(): 258     33965    cpu-e-1149  2
[0] MPI startup(): 259     33966    cpu-e-1149  3
[0] MPI startup(): 260     33967    cpu-e-1149  4
[0] MPI startup(): 261     33968    cpu-e-1149  5
[0] MPI startup(): 262     33969    cpu-e-1149  6
[0] MPI startup(): 263     33970    cpu-e-1149  7
[0] MPI startup(): 264     33971    cpu-e-1149  8
[0] MPI startup(): 265     33972    cpu-e-1149  9
[0] MPI startup(): 266     33973    cpu-e-1149  10
[0] MPI startup(): 267     33974    cpu-e-1149  11
[0] MPI startup(): 268     33975    cpu-e-1149  12
[0] MPI startup(): 269     33976    cpu-e-1149  13
[0] MPI startup(): 270     33977    cpu-e-1149  14
[0] MPI startup(): 271     33978    cpu-e-1149  15
[0] MPI startup(): 272     33979    cpu-e-1149  16
[0] MPI startup(): 273     33980    cpu-e-1149  17
[0] MPI startup(): 274     33981    cpu-e-1149  18
[0] MPI startup(): 275     33982    cpu-e-1149  19
[0] MPI startup(): 276     33983    cpu-e-1149  20
[0] MPI startup(): 277     33984    cpu-e-1149  21
[0] MPI startup(): 278     33985    cpu-e-1149  22
[0] MPI startup(): 279     33986    cpu-e-1149  23
[0] MPI startup(): 280     33987    cpu-e-1149  24
[0] MPI startup(): 281     33988    cpu-e-1149  25
[0] MPI startup(): 282     33989    cpu-e-1149  26
[0] MPI startup(): 283     33990    cpu-e-1149  27
[0] MPI startup(): 284     33991    cpu-e-1149  28
[0] MPI startup(): 285     33992    cpu-e-1149  29
[0] MPI startup(): 286     33993    cpu-e-1149  30
[0] MPI startup(): 287     33994    cpu-e-1149  31
[0] MPI startup(): 288     33994    cpu-e-1150  0
[0] MPI startup(): 289     33995    cpu-e-1150  1
[0] MPI startup(): 290     33996    cpu-e-1150  2
[0] MPI startup(): 291     33997    cpu-e-1150  3
[0] MPI startup(): 292     33998    cpu-e-1150  4
[0] MPI startup(): 293     33999    cpu-e-1150  5
[0] MPI startup(): 294     34000    cpu-e-1150  6
[0] MPI startup(): 295     34001    cpu-e-1150  7
[0] MPI startup(): 296     34002    cpu-e-1150  8
[0] MPI startup(): 297     34003    cpu-e-1150  9
[0] MPI startup(): 298     34004    cpu-e-1150  10
[0] MPI startup(): 299     34005    cpu-e-1150  11
[0] MPI startup(): 300     34006    cpu-e-1150  12
[0] MPI startup(): 301     34007    cpu-e-1150  13
[0] MPI startup(): 302     34008    cpu-e-1150  14
[0] MPI startup(): 303     34009    cpu-e-1150  15
[0] MPI startup(): 304     34010    cpu-e-1150  16
[0] MPI startup(): 305     34011    cpu-e-1150  17
[0] MPI startup(): 306     34012    cpu-e-1150  18
[0] MPI startup(): 307     34013    cpu-e-1150  19
[0] MPI startup(): 308     34014    cpu-e-1150  20
[0] MPI startup(): 309     34015    cpu-e-1150  21
[0] MPI startup(): 310     34016    cpu-e-1150  22
[0] MPI startup(): 311     34017    cpu-e-1150  23
[0] MPI startup(): 312     34018    cpu-e-1150  24
[0] MPI startup(): 313     34019    cpu-e-1150  25
[0] MPI startup(): 314     34020    cpu-e-1150  26
[0] MPI startup(): 315     34021    cpu-e-1150  27
[0] MPI startup(): 316     34022    cpu-e-1150  28
[0] MPI startup(): 317     34023    cpu-e-1150  29
[0] MPI startup(): 318     34024    cpu-e-1150  30
[0] MPI startup(): 319     34025    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Fri Nov  8 00:06:02 2019
Command line        : /home/wjt27/io-500-sc19/bin/ior -w -s 1900000 -a MPIIO -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_hard/IOR_file -O stoneWallingStatusFile=/dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux cpu-e-829
TestID              : 0
StartTime           : Fri Nov  8 00:06:02 2019
Path                : /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_hard
FS                  : 405.5 TiB   Used FS: 25.9%   Inodes: 1820.8 Mi   Used Inodes: 10.0%

Options: 
api                 : MPIIO
apiVersion          : (3.1)
test filename       : /dac/fs1/mjr208/job16760824-2019-11-07-2352/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
nodes               : 10
tasks               : 320
clients per node    : 32
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 25.99 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  IOPS       Latency(s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ----       ----------  ---------- ---------  --------   --------   --------   --------   ----
stonewalling pairs accessed min: 68485 max: 181723 -- min data: 3.0 GiB mean data: 3.6 GiB time: 300.0s
WARNING: Expected aggregate file size       = 28580864000000.
WARNING: Stat() of aggregate file size      = 2733579130880.
WARNING: Using actual aggregate bytes moved = 2733579130880.
WARNING: maybe caused by deadlineForStonewalling
write     3258.41    72685      686.08      45.91      45.91      0.019191   800.04     0.009645   800.07     0   
Max Write: 3258.41 MiB/sec (3416.69 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        3258.41    3258.41    3258.41       0.00   72683.06   72683.06   72683.06       0.00  800.06758     300.02       3891.26     0    320  32    1   0     1        1         0    0 1900000    47008    47008 2606944.2 MPIIO      0
Finished            : Fri Nov  8 00:19:24 2019
mdtest_easy_delete
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       37009    cpu-e-829   0
[0] MPI startup(): 1       37010    cpu-e-829   1
[0] MPI startup(): 2       37011    cpu-e-829   2
[0] MPI startup(): 3       37012    cpu-e-829   3
[0] MPI startup(): 4       37013    cpu-e-829   4
[0] MPI startup(): 5       37014    cpu-e-829   5
[0] MPI startup(): 6       37015    cpu-e-829   6
[0] MPI startup(): 7       37016    cpu-e-829   7
[0] MPI startup(): 8       37017    cpu-e-829   8
[0] MPI startup(): 9       37018    cpu-e-829   9
[0] MPI startup(): 10      37019    cpu-e-829   10
[0] MPI startup(): 11      37020    cpu-e-829   11
[0] MPI startup(): 12      37021    cpu-e-829   12
[0] MPI startup(): 13      37022    cpu-e-829   13
[0] MPI startup(): 14      37023    cpu-e-829   14
[0] MPI startup(): 15      37024    cpu-e-829   15
[0] MPI startup(): 16      37025    cpu-e-829   16
[0] MPI startup(): 17      37026    cpu-e-829   17
[0] MPI startup(): 18      37027    cpu-e-829   18
[0] MPI startup(): 19      37028    cpu-e-829   19
[0] MPI startup(): 20      37029    cpu-e-829   20
[0] MPI startup(): 21      37030    cpu-e-829   21
[0] MPI startup(): 22      37031    cpu-e-829   22
[0] MPI startup(): 23      37032    cpu-e-829   23
[0] MPI startup(): 24      37033    cpu-e-829   24
[0] MPI startup(): 25      37034    cpu-e-829   25
[0] MPI startup(): 26      37035    cpu-e-829   26
[0] MPI startup(): 27      37036    cpu-e-829   27
[0] MPI startup(): 28      37037    cpu-e-829   28
[0] MPI startup(): 29      37038    cpu-e-829   29
[0] MPI startup(): 30      37039    cpu-e-829   30
[0] MPI startup(): 31      37040    cpu-e-829   31
[0] MPI startup(): 32      36182    cpu-e-830   0
[0] MPI startup(): 33      36184    cpu-e-830   1
[0] MPI startup(): 34      36185    cpu-e-830   2
[0] MPI startup(): 35      36186    cpu-e-830   3
[0] MPI startup(): 36      36187    cpu-e-830   4
[0] MPI startup(): 37      36188    cpu-e-830   5
[0] MPI startup(): 38      36189    cpu-e-830   6
[0] MPI startup(): 39      36190    cpu-e-830   7
[0] MPI startup(): 40      36191    cpu-e-830   8
[0] MPI startup(): 41      36192    cpu-e-830   9
[0] MPI startup(): 42      36193    cpu-e-830   10
[0] MPI startup(): 43      36194    cpu-e-830   11
[0] MPI startup(): 44      36195    cpu-e-830   12
[0] MPI startup(): 45      36196    cpu-e-830   13
[0] MPI startup(): 46      36197    cpu-e-830   14
[0] MPI startup(): 47      36198    cpu-e-830   15
[0] MPI startup(): 48      36199    cpu-e-830   16
[0] MPI startup(): 49      36200    cpu-e-830   17
[0] MPI startup(): 50      36201    cpu-e-830   18
[0] MPI startup(): 51      36202    cpu-e-830   19
[0] MPI startup(): 52      36203    cpu-e-830   20
[0] MPI startup(): 53      36204    cpu-e-830   21
[0] MPI startup(): 54      36205    cpu-e-830   22
[0] MPI startup(): 55      36206    cpu-e-830   23
[0] MPI startup(): 56      36207    cpu-e-830   24
[0] MPI startup(): 57      36208    cpu-e-830   25
[0] MPI startup(): 58      36209    cpu-e-830   26
[0] MPI startup(): 59      36210    cpu-e-830   27
[0] MPI startup(): 60      36211    cpu-e-830   28
[0] MPI startup(): 61      36212    cpu-e-830   29
[0] MPI startup(): 62      36213    cpu-e-830   30
[0] MPI startup(): 63      36214    cpu-e-830   31
[0] MPI startup(): 64      35824    cpu-e-893   0
[0] MPI startup(): 65      35825    cpu-e-893   1
[0] MPI startup(): 66      35826    cpu-e-893   2
[0] MPI startup(): 67      35827    cpu-e-893   3
[0] MPI startup(): 68      35828    cpu-e-893   4
[0] MPI startup(): 69      35829    cpu-e-893   5
[0] MPI startup(): 70      35830    cpu-e-893   6
[0] MPI startup(): 71      35831    cpu-e-893   7
[0] MPI startup(): 72      35832    cpu-e-893   8
[0] MPI startup(): 73      35833    cpu-e-893   9
[0] MPI startup(): 74      35834    cpu-e-893   10
[0] MPI startup(): 75      35835    cpu-e-893   11
[0] MPI startup(): 76      35836    cpu-e-893   12
[0] MPI startup(): 77      35837    cpu-e-893   13
[0] MPI startup(): 78      35838    cpu-e-893   14
[0] MPI startup(): 79      35839    cpu-e-893   15
[0] MPI startup(): 80      35840    cpu-e-893   16
[0] MPI startup(): 81      35841    cpu-e-893   17
[0] MPI startup(): 82      35842    cpu-e-893   18
[0] MPI startup(): 83      35843    cpu-e-893   19
[0] MPI startup(): 84      35844    cpu-e-893   20
[0] MPI startup(): 85      35845    cpu-e-893   21
[0] MPI startup(): 86      35846    cpu-e-893   22
[0] MPI startup(): 87      35847    cpu-e-893   23
[0] MPI startup(): 88      35848    cpu-e-893   24
[0] MPI startup(): 89      35849    cpu-e-893   25
[0] MPI startup(): 90      35850    cpu-e-893   26
[0] MPI startup(): 91      35851    cpu-e-893   27
[0] MPI startup(): 92      35852    cpu-e-893   28
[0] MPI startup(): 93      35853    cpu-e-893   29
[0] MPI startup(): 94      35854    cpu-e-893   30
[0] MPI startup(): 95      35855    cpu-e-893   31
[0] MPI startup(): 96      35812    cpu-e-894   0
[0] MPI startup(): 97      35813    cpu-e-894   1
[0] MPI startup(): 98      35814    cpu-e-894   2
[0] MPI startup(): 99      35815    cpu-e-894   3
[0] MPI startup(): 100     35816    cpu-e-894   4
[0] MPI startup(): 101     35817    cpu-e-894   5
[0] MPI startup(): 102     35818    cpu-e-894   6
[0] MPI startup(): 103     35819    cpu-e-894   7
[0] MPI startup(): 104     35820    cpu-e-894   8
[0] MPI startup(): 105     35821    cpu-e-894   9
[0] MPI startup(): 106     35822    cpu-e-894   10
[0] MPI startup(): 107     35823    cpu-e-894   11
[0] MPI startup(): 108     35824    cpu-e-894   12
[0] MPI startup(): 109     35825    cpu-e-894   13
[0] MPI startup(): 110     35826    cpu-e-894   14
[0] MPI startup(): 111     35827    cpu-e-894   15
[0] MPI startup(): 112     35829    cpu-e-894   16
[0] MPI startup(): 113     35830    cpu-e-894   17
[0] MPI startup(): 114     35831    cpu-e-894   18
[0] MPI startup(): 115     35832    cpu-e-894   19
[0] MPI startup(): 116     35833    cpu-e-894   20
[0] MPI startup(): 117     35834    cpu-e-894   21
[0] MPI startup(): 118     35835    cpu-e-894   22
[0] MPI startup(): 119     35836    cpu-e-894   23
[0] MPI startup(): 120     35837    cpu-e-894   24
[0] MPI startup(): 121     35838    cpu-e-894   25
[0] MPI startup(): 122     35839    cpu-e-894   26
[0] MPI startup(): 123     35840    cpu-e-894   27
[0] MPI startup(): 124     35841    cpu-e-894   28
[0] MPI startup(): 125     35842    cpu-e-894   29
[0] MPI startup(): 126     35843    cpu-e-894   30
[0] MPI startup(): 127     35844    cpu-e-894   31
[0] MPI startup(): 128     35889    cpu-e-957   0
[0] MPI startup(): 129     35890    cpu-e-957   1
[0] MPI startup(): 130     35891    cpu-e-957   2
[0] MPI startup(): 131     35892    cpu-e-957   3
[0] MPI startup(): 132     35893    cpu-e-957   4
[0] MPI startup(): 133     35894    cpu-e-957   5
[0] MPI startup(): 134     35895    cpu-e-957   6
[0] MPI startup(): 135     35896    cpu-e-957   7
[0] MPI startup(): 136     35897    cpu-e-957   8
[0] MPI startup(): 137     35898    cpu-e-957   9
[0] MPI startup(): 138     35899    cpu-e-957   10
[0] MPI startup(): 139     35900    cpu-e-957   11
[0] MPI startup(): 140     35901    cpu-e-957   12
[0] MPI startup(): 141     35902    cpu-e-957   13
[0] MPI startup(): 142     35903    cpu-e-957   14
[0] MPI startup(): 143     35904    cpu-e-957   15
[0] MPI startup(): 144     35905    cpu-e-957   16
[0] MPI startup(): 145     35906    cpu-e-957   17
[0] MPI startup(): 146     35907    cpu-e-957   18
[0] MPI startup(): 147     35908    cpu-e-957   19
[0] MPI startup(): 148     35909    cpu-e-957   20
[0] MPI startup(): 149     35910    cpu-e-957   21
[0] MPI startup(): 150     35911    cpu-e-957   22
[0] MPI startup(): 151     35912    cpu-e-957   23
[0] MPI startup(): 152     35913    cpu-e-957   24
[0] MPI startup(): 153     35914    cpu-e-957   25
[0] MPI startup(): 154     35915    cpu-e-957   26
[0] MPI startup(): 155     35916    cpu-e-957   27
[0] MPI startup(): 156     35917    cpu-e-957   28
[0] MPI startup(): 157     35918    cpu-e-957   29
[0] MPI startup(): 158     35919    cpu-e-957   30
[0] MPI startup(): 159     35920    cpu-e-957   31
[0] MPI startup(): 160     35845    cpu-e-958   0
[0] MPI startup(): 161     35846    cpu-e-958   1
[0] MPI startup(): 162     35847    cpu-e-958   2
[0] MPI startup(): 163     35848    cpu-e-958   3
[0] MPI startup(): 164     35849    cpu-e-958   4
[0] MPI startup(): 165     35850    cpu-e-958   5
[0] MPI startup(): 166     35851    cpu-e-958   6
[0] MPI startup(): 167     35852    cpu-e-958   7
[0] MPI startup(): 168     35853    cpu-e-958   8
[0] MPI startup(): 169     35854    cpu-e-958   9
[0] MPI startup(): 170     35855    cpu-e-958   10
[0] MPI startup(): 171     35856    cpu-e-958   11
[0] MPI startup(): 172     35857    cpu-e-958   12
[0] MPI startup(): 173     35858    cpu-e-958   13
[0] MPI startup(): 174     35859    cpu-e-958   14
[0] MPI startup(): 175     35860    cpu-e-958   15
[0] MPI startup(): 176     35861    cpu-e-958   16
[0] MPI startup(): 177     35862    cpu-e-958   17
[0] MPI startup(): 178     35863    cpu-e-958   18
[0] MPI startup(): 179     35864    cpu-e-958   19
[0] MPI startup(): 180     35865    cpu-e-958   20
[0] MPI startup(): 181     35866    cpu-e-958   21
[0] MPI startup(): 182     35867    cpu-e-958   22
[0] MPI startup(): 183     35868    cpu-e-958   23
[0] MPI startup(): 184     35869    cpu-e-958   24
[0] MPI startup(): 185     35870    cpu-e-958   25
[0] MPI startup(): 186     35871    cpu-e-958   26
[0] MPI startup(): 187     35872    cpu-e-958   27
[0] MPI startup(): 188     35873    cpu-e-958   28
[0] MPI startup(): 189     35874    cpu-e-958   29
[0] MPI startup(): 190     35875    cpu-e-958   30
[0] MPI startup(): 191     35876    cpu-e-958   31
[0] MPI startup(): 192     36098    cpu-e-1021  0
[0] MPI startup(): 193     36099    cpu-e-1021  1
[0] MPI startup(): 194     36100    cpu-e-1021  2
[0] MPI startup(): 195     36101    cpu-e-1021  3
[0] MPI startup(): 196     36102    cpu-e-1021  4
[0] MPI startup(): 197     36103    cpu-e-1021  5
[0] MPI startup(): 198     36104    cpu-e-1021  6
[0] MPI startup(): 199     36105    cpu-e-1021  7
[0] MPI startup(): 200     36106    cpu-e-1021  8
[0] MPI startup(): 201     36107    cpu-e-1021  9
[0] MPI startup(): 202     36108    cpu-e-1021  10
[0] MPI startup(): 203     36109    cpu-e-1021  11
[0] MPI startup(): 204     36110    cpu-e-1021  12
[0] MPI startup(): 205     36111    cpu-e-1021  13
[0] MPI startup(): 206     36112    cpu-e-1021  14
[0] MPI startup(): 207     36113    cpu-e-1021  15
[0] MPI startup(): 208     36114    cpu-e-1021  16
[0] MPI startup(): 209     36115    cpu-e-1021  17
[0] MPI startup(): 210     36116    cpu-e-1021  18
[0] MPI startup(): 211     36117    cpu-e-1021  19
[0] MPI startup(): 212     36118    cpu-e-1021  20
[0] MPI startup(): 213     36119    cpu-e-1021  21
[0] MPI startup(): 214     36120    cpu-e-1021  22
[0] MPI startup(): 215     36121    cpu-e-1021  23
[0] MPI startup(): 216     36122    cpu-e-1021  24
[0] MPI startup(): 217     36123    cpu-e-1021  25
[0] MPI startup(): 218     36124    cpu-e-1021  26
[0] MPI startup(): 219     36125    cpu-e-1021  27
[0] MPI startup(): 220     36126    cpu-e-1021  28
[0] MPI startup(): 221     36127    cpu-e-1021  29
[0] MPI startup(): 222     36128    cpu-e-1021  30
[0] MPI startup(): 223     36129    cpu-e-1021  31
[0] MPI startup(): 224     35868    cpu-e-1022  0
[0] MPI startup(): 225     35869    cpu-e-1022  1
[0] MPI startup(): 226     35870    cpu-e-1022  2
[0] MPI startup(): 227     35871    cpu-e-1022  3
[0] MPI startup(): 228     35872    cpu-e-1022  4
[0] MPI startup(): 229     35873    cpu-e-1022  5
[0] MPI startup(): 230     35874    cpu-e-1022  6
[0] MPI startup(): 231     35875    cpu-e-1022  7
[0] MPI startup(): 232     35876    cpu-e-1022  8
[0] MPI startup(): 233     35877    cpu-e-1022  9
[0] MPI startup(): 234     35878    cpu-e-1022  10
[0] MPI startup(): 235     35879    cpu-e-1022  11
[0] MPI startup(): 236     35880    cpu-e-1022  12
[0] MPI startup(): 237     35881    cpu-e-1022  13
[0] MPI startup(): 238     35882    cpu-e-1022  14
[0] MPI startup(): 239     35883    cpu-e-1022  15
[0] MPI startup(): 240     35884    cpu-e-1022  16
[0] MPI startup(): 241     35885    cpu-e-1022  17
[0] MPI startup(): 242     35886    cpu-e-1022  18
[0] MPI startup(): 243     35887    cpu-e-1022  19
[0] MPI startup(): 244     35888    cpu-e-1022  20
[0] MPI startup(): 245     35889    cpu-e-1022  21
[0] MPI startup(): 246     35890    cpu-e-1022  22
[0] MPI startup(): 247     35891    cpu-e-1022  23
[0] MPI startup(): 248     35892    cpu-e-1022  24
[0] MPI startup(): 249     35893    cpu-e-1022  25
[0] MPI startup(): 250     35894    cpu-e-1022  26
[0] MPI startup(): 251     35895    cpu-e-1022  27
[0] MPI startup(): 252     35896    cpu-e-1022  28
[0] MPI startup(): 253     35897    cpu-e-1022  29
[0] MPI startup(): 254     35898    cpu-e-1022  30
[0] MPI startup(): 255     35899    cpu-e-1022  31
[0] MPI startup(): 256     35903    cpu-e-1149  0
[0] MPI startup(): 257     35904    cpu-e-1149  1
[0] MPI startup(): 258     35905    cpu-e-1149  2
[0] MPI startup(): 259     35906    cpu-e-1149  3
[0] MPI startup(): 260     35907    cpu-e-1149  4
[0] MPI startup(): 261     35908    cpu-e-1149  5
[0] MPI startup(): 262     35909    cpu-e-1149  6
[0] MPI startup(): 263     35910    cpu-e-1149  7
[0] MPI startup(): 264     35911    cpu-e-1149  8
[0] MPI startup(): 265     35912    cpu-e-1149  9
[0] MPI startup(): 266     35913    cpu-e-1149  10
[0] MPI startup(): 267     35914    cpu-e-1149  11
[0] MPI startup(): 268     35915    cpu-e-1149  12
[0] MPI startup(): 269     35916    cpu-e-1149  13
[0] MPI startup(): 270     35917    cpu-e-1149  14
[0] MPI startup(): 271     35918    cpu-e-1149  15
[0] MPI startup(): 272     35919    cpu-e-1149  16
[0] MPI startup(): 273     35920    cpu-e-1149  17
[0] MPI startup(): 274     35921    cpu-e-1149  18
[0] MPI startup(): 275     35922    cpu-e-1149  19
[0] MPI startup(): 276     35923    cpu-e-1149  20
[0] MPI startup(): 277     35924    cpu-e-1149  21
[0] MPI startup(): 278     35925    cpu-e-1149  22
[0] MPI startup(): 279     35926    cpu-e-1149  23
[0] MPI startup(): 280     35927    cpu-e-1149  24
[0] MPI startup(): 281     35928    cpu-e-1149  25
[0] MPI startup(): 282     35929    cpu-e-1149  26
[0] MPI startup(): 283     35930    cpu-e-1149  27
[0] MPI startup(): 284     35931    cpu-e-1149  28
[0] MPI startup(): 285     35932    cpu-e-1149  29
[0] MPI startup(): 286     35933    cpu-e-1149  30
[0] MPI startup(): 287     35934    cpu-e-1149  31
[0] MPI startup(): 288     35963    cpu-e-1150  0
[0] MPI startup(): 289     35964    cpu-e-1150  1
[0] MPI startup(): 290     35965    cpu-e-1150  2
[0] MPI startup(): 291     35966    cpu-e-1150  3
[0] MPI startup(): 292     35967    cpu-e-1150  4
[0] MPI startup(): 293     35968    cpu-e-1150  5
[0] MPI startup(): 294     35969    cpu-e-1150  6
[0] MPI startup(): 295     35970    cpu-e-1150  7
[0] MPI startup(): 296     35971    cpu-e-1150  8
[0] MPI startup(): 297     35972    cpu-e-1150  9
[0] MPI startup(): 298     35973    cpu-e-1150  10
[0] MPI startup(): 299     35974    cpu-e-1150  11
[0] MPI startup(): 300     35975    cpu-e-1150  12
[0] MPI startup(): 301     35976    cpu-e-1150  13
[0] MPI startup(): 302     35977    cpu-e-1150  14
[0] MPI startup(): 303     35978    cpu-e-1150  15
[0] MPI startup(): 304     35979    cpu-e-1150  16
[0] MPI startup(): 305     35980    cpu-e-1150  17
[0] MPI startup(): 306     35981    cpu-e-1150  18
[0] MPI startup(): 307     35982    cpu-e-1150  19
[0] MPI startup(): 308     35983    cpu-e-1150  20
[0] MPI startup(): 309     35984    cpu-e-1150  21
[0] MPI startup(): 310     35985    cpu-e-1150  22
[0] MPI startup(): 311     35986    cpu-e-1150  23
[0] MPI startup(): 312     35987    cpu-e-1150  24
[0] MPI startup(): 313     35988    cpu-e-1150  25
[0] MPI startup(): 314     35989    cpu-e-1150  26
[0] MPI startup(): 315     35990    cpu-e-1150  27
[0] MPI startup(): 316     35991    cpu-e-1150  28
[0] MPI startup(): 317     35992    cpu-e-1150  29
[0] MPI startup(): 318     35993    cpu-e-1150  30
[0] MPI startup(): 319     35994    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
-- started at 11/08/2019 00:47:54 --

mdtest-3.3.0+dev was launched with 320 total task(s) on 10 node(s)
Command line used: /home/wjt27/io-500-sc19/bin/mdtest '-r' '-F' '-P' '-d' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_easy-stonewall' '-N' '1'
Path: /dac/fs1/mjr208/job16760824-2019-11-07-2352
FS: 405.5 TiB   Used FS: 26.5%   Inodes: 1820.8 Mi   Used Inodes: 10.5%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 32 for each phase.
320 tasks, 288000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :     129618.031     129612.762     129613.275          0.344
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.650          0.650          0.650          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       1472.206       1472.146       1472.200          0.004
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.539          1.539          1.539          0.000
-- finished at 11/08/2019 01:12:34 --

mdtest_easy_stat
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       36059    cpu-e-829   0
[0] MPI startup(): 1       36060    cpu-e-829   1
[0] MPI startup(): 2       36061    cpu-e-829   2
[0] MPI startup(): 3       36062    cpu-e-829   3
[0] MPI startup(): 4       36063    cpu-e-829   4
[0] MPI startup(): 5       36064    cpu-e-829   5
[0] MPI startup(): 6       36065    cpu-e-829   6
[0] MPI startup(): 7       36066    cpu-e-829   7
[0] MPI startup(): 8       36067    cpu-e-829   8
[0] MPI startup(): 9       36068    cpu-e-829   9
[0] MPI startup(): 10      36069    cpu-e-829   10
[0] MPI startup(): 11      36070    cpu-e-829   11
[0] MPI startup(): 12      36071    cpu-e-829   12
[0] MPI startup(): 13      36072    cpu-e-829   13
[0] MPI startup(): 14      36073    cpu-e-829   14
[0] MPI startup(): 15      36074    cpu-e-829   15
[0] MPI startup(): 16      36075    cpu-e-829   16
[0] MPI startup(): 17      36076    cpu-e-829   17
[0] MPI startup(): 18      36077    cpu-e-829   18
[0] MPI startup(): 19      36078    cpu-e-829   19
[0] MPI startup(): 20      36079    cpu-e-829   20
[0] MPI startup(): 21      36080    cpu-e-829   21
[0] MPI startup(): 22      36081    cpu-e-829   22
[0] MPI startup(): 23      36082    cpu-e-829   23
[0] MPI startup(): 24      36083    cpu-e-829   24
[0] MPI startup(): 25      36084    cpu-e-829   25
[0] MPI startup(): 26      36085    cpu-e-829   26
[0] MPI startup(): 27      36086    cpu-e-829   27
[0] MPI startup(): 28      36087    cpu-e-829   28
[0] MPI startup(): 29      36088    cpu-e-829   29
[0] MPI startup(): 30      36089    cpu-e-829   30
[0] MPI startup(): 31      36090    cpu-e-829   31
[0] MPI startup(): 32      35318    cpu-e-830   0
[0] MPI startup(): 33      35319    cpu-e-830   1
[0] MPI startup(): 34      35320    cpu-e-830   2
[0] MPI startup(): 35      35321    cpu-e-830   3
[0] MPI startup(): 36      35322    cpu-e-830   4
[0] MPI startup(): 37      35323    cpu-e-830   5
[0] MPI startup(): 38      35324    cpu-e-830   6
[0] MPI startup(): 39      35325    cpu-e-830   7
[0] MPI startup(): 40      35326    cpu-e-830   8
[0] MPI startup(): 41      35327    cpu-e-830   9
[0] MPI startup(): 42      35328    cpu-e-830   10
[0] MPI startup(): 43      35329    cpu-e-830   11
[0] MPI startup(): 44      35330    cpu-e-830   12
[0] MPI startup(): 45      35331    cpu-e-830   13
[0] MPI startup(): 46      35332    cpu-e-830   14
[0] MPI startup(): 47      35333    cpu-e-830   15
[0] MPI startup(): 48      35334    cpu-e-830   16
[0] MPI startup(): 49      35335    cpu-e-830   17
[0] MPI startup(): 50      35336    cpu-e-830   18
[0] MPI startup(): 51      35337    cpu-e-830   19
[0] MPI startup(): 52      35338    cpu-e-830   20
[0] MPI startup(): 53      35339    cpu-e-830   21
[0] MPI startup(): 54      35340    cpu-e-830   22
[0] MPI startup(): 55      35341    cpu-e-830   23
[0] MPI startup(): 56      35342    cpu-e-830   24
[0] MPI startup(): 57      35343    cpu-e-830   25
[0] MPI startup(): 58      35344    cpu-e-830   26
[0] MPI startup(): 59      35345    cpu-e-830   27
[0] MPI startup(): 60      35346    cpu-e-830   28
[0] MPI startup(): 61      35347    cpu-e-830   29
[0] MPI startup(): 62      35348    cpu-e-830   30
[0] MPI startup(): 63      35349    cpu-e-830   31
[0] MPI startup(): 64      35055    cpu-e-893   0
[0] MPI startup(): 65      35056    cpu-e-893   1
[0] MPI startup(): 66      35057    cpu-e-893   2
[0] MPI startup(): 67      35058    cpu-e-893   3
[0] MPI startup(): 68      35059    cpu-e-893   4
[0] MPI startup(): 69      35060    cpu-e-893   5
[0] MPI startup(): 70      35061    cpu-e-893   6
[0] MPI startup(): 71      35062    cpu-e-893   7
[0] MPI startup(): 72      35063    cpu-e-893   8
[0] MPI startup(): 73      35064    cpu-e-893   9
[0] MPI startup(): 74      35065    cpu-e-893   10
[0] MPI startup(): 75      35066    cpu-e-893   11
[0] MPI startup(): 76      35067    cpu-e-893   12
[0] MPI startup(): 77      35068    cpu-e-893   13
[0] MPI startup(): 78      35069    cpu-e-893   14
[0] MPI startup(): 79      35070    cpu-e-893   15
[0] MPI startup(): 80      35071    cpu-e-893   16
[0] MPI startup(): 81      35072    cpu-e-893   17
[0] MPI startup(): 82      35073    cpu-e-893   18
[0] MPI startup(): 83      35074    cpu-e-893   19
[0] MPI startup(): 84      35075    cpu-e-893   20
[0] MPI startup(): 85      35076    cpu-e-893   21
[0] MPI startup(): 86      35077    cpu-e-893   22
[0] MPI startup(): 87      35078    cpu-e-893   23
[0] MPI startup(): 88      35079    cpu-e-893   24
[0] MPI startup(): 89      35080    cpu-e-893   25
[0] MPI startup(): 90      35081    cpu-e-893   26
[0] MPI startup(): 91      35082    cpu-e-893   27
[0] MPI startup(): 92      35083    cpu-e-893   28
[0] MPI startup(): 93      35084    cpu-e-893   29
[0] MPI startup(): 94      35085    cpu-e-893   30
[0] MPI startup(): 95      35086    cpu-e-893   31
[0] MPI startup(): 96      35032    cpu-e-894   0
[0] MPI startup(): 97      35033    cpu-e-894   1
[0] MPI startup(): 98      35034    cpu-e-894   2
[0] MPI startup(): 99      35035    cpu-e-894   3
[0] MPI startup(): 100     35036    cpu-e-894   4
[0] MPI startup(): 101     35037    cpu-e-894   5
[0] MPI startup(): 102     35038    cpu-e-894   6
[0] MPI startup(): 103     35039    cpu-e-894   7
[0] MPI startup(): 104     35040    cpu-e-894   8
[0] MPI startup(): 105     35041    cpu-e-894   9
[0] MPI startup(): 106     35042    cpu-e-894   10
[0] MPI startup(): 107     35043    cpu-e-894   11
[0] MPI startup(): 108     35044    cpu-e-894   12
[0] MPI startup(): 109     35045    cpu-e-894   13
[0] MPI startup(): 110     35046    cpu-e-894   14
[0] MPI startup(): 111     35047    cpu-e-894   15
[0] MPI startup(): 112     35048    cpu-e-894   16
[0] MPI startup(): 113     35049    cpu-e-894   17
[0] MPI startup(): 114     35050    cpu-e-894   18
[0] MPI startup(): 115     35051    cpu-e-894   19
[0] MPI startup(): 116     35052    cpu-e-894   20
[0] MPI startup(): 117     35053    cpu-e-894   21
[0] MPI startup(): 118     35054    cpu-e-894   22
[0] MPI startup(): 119     35055    cpu-e-894   23
[0] MPI startup(): 120     35056    cpu-e-894   24
[0] MPI startup(): 121     35057    cpu-e-894   25
[0] MPI startup(): 122     35058    cpu-e-894   26
[0] MPI startup(): 123     35059    cpu-e-894   27
[0] MPI startup(): 124     35060    cpu-e-894   28
[0] MPI startup(): 125     35061    cpu-e-894   29
[0] MPI startup(): 126     35062    cpu-e-894   30
[0] MPI startup(): 127     35063    cpu-e-894   31
[0] MPI startup(): 128     35179    cpu-e-957   0
[0] MPI startup(): 129     35180    cpu-e-957   1
[0] MPI startup(): 130     35181    cpu-e-957   2
[0] MPI startup(): 131     35182    cpu-e-957   3
[0] MPI startup(): 132     35183    cpu-e-957   4
[0] MPI startup(): 133     35184    cpu-e-957   5
[0] MPI startup(): 134     35185    cpu-e-957   6
[0] MPI startup(): 135     35186    cpu-e-957   7
[0] MPI startup(): 136     35187    cpu-e-957   8
[0] MPI startup(): 137     35188    cpu-e-957   9
[0] MPI startup(): 138     35189    cpu-e-957   10
[0] MPI startup(): 139     35190    cpu-e-957   11
[0] MPI startup(): 140     35191    cpu-e-957   12
[0] MPI startup(): 141     35192    cpu-e-957   13
[0] MPI startup(): 142     35193    cpu-e-957   14
[0] MPI startup(): 143     35194    cpu-e-957   15
[0] MPI startup(): 144     35195    cpu-e-957   16
[0] MPI startup(): 145     35196    cpu-e-957   17
[0] MPI startup(): 146     35197    cpu-e-957   18
[0] MPI startup(): 147     35198    cpu-e-957   19
[0] MPI startup(): 148     35199    cpu-e-957   20
[0] MPI startup(): 149     35200    cpu-e-957   21
[0] MPI startup(): 150     35201    cpu-e-957   22
[0] MPI startup(): 151     35202    cpu-e-957   23
[0] MPI startup(): 152     35203    cpu-e-957   24
[0] MPI startup(): 153     35204    cpu-e-957   25
[0] MPI startup(): 154     35205    cpu-e-957   26
[0] MPI startup(): 155     35206    cpu-e-957   27
[0] MPI startup(): 156     35207    cpu-e-957   28
[0] MPI startup(): 157     35208    cpu-e-957   29
[0] MPI startup(): 158     35209    cpu-e-957   30
[0] MPI startup(): 159     35210    cpu-e-957   31
[0] MPI startup(): 160     35122    cpu-e-958   0
[0] MPI startup(): 161     35123    cpu-e-958   1
[0] MPI startup(): 162     35124    cpu-e-958   2
[0] MPI startup(): 163     35125    cpu-e-958   3
[0] MPI startup(): 164     35126    cpu-e-958   4
[0] MPI startup(): 165     35127    cpu-e-958   5
[0] MPI startup(): 166     35128    cpu-e-958   6
[0] MPI startup(): 167     35129    cpu-e-958   7
[0] MPI startup(): 168     35130    cpu-e-958   8
[0] MPI startup(): 169     35131    cpu-e-958   9
[0] MPI startup(): 170     35132    cpu-e-958   10
[0] MPI startup(): 171     35133    cpu-e-958   11
[0] MPI startup(): 172     35134    cpu-e-958   12
[0] MPI startup(): 173     35135    cpu-e-958   13
[0] MPI startup(): 174     35136    cpu-e-958   14
[0] MPI startup(): 175     35137    cpu-e-958   15
[0] MPI startup(): 176     35138    cpu-e-958   16
[0] MPI startup(): 177     35139    cpu-e-958   17
[0] MPI startup(): 178     35140    cpu-e-958   18
[0] MPI startup(): 179     35141    cpu-e-958   19
[0] MPI startup(): 180     35142    cpu-e-958   20
[0] MPI startup(): 181     35143    cpu-e-958   21
[0] MPI startup(): 182     35144    cpu-e-958   22
[0] MPI startup(): 183     35145    cpu-e-958   23
[0] MPI startup(): 184     35146    cpu-e-958   24
[0] MPI startup(): 185     35147    cpu-e-958   25
[0] MPI startup(): 186     35148    cpu-e-958   26
[0] MPI startup(): 187     35149    cpu-e-958   27
[0] MPI startup(): 188     35150    cpu-e-958   28
[0] MPI startup(): 189     35151    cpu-e-958   29
[0] MPI startup(): 190     35152    cpu-e-958   30
[0] MPI startup(): 191     35153    cpu-e-958   31
[0] MPI startup(): 192     35383    cpu-e-1021  0
[0] MPI startup(): 193     35384    cpu-e-1021  1
[0] MPI startup(): 194     35385    cpu-e-1021  2
[0] MPI startup(): 195     35386    cpu-e-1021  3
[0] MPI startup(): 196     35387    cpu-e-1021  4
[0] MPI startup(): 197     35388    cpu-e-1021  5
[0] MPI startup(): 198     35389    cpu-e-1021  6
[0] MPI startup(): 199     35390    cpu-e-1021  7
[0] MPI startup(): 200     35391    cpu-e-1021  8
[0] MPI startup(): 201     35392    cpu-e-1021  9
[0] MPI startup(): 202     35393    cpu-e-1021  10
[0] MPI startup(): 203     35394    cpu-e-1021  11
[0] MPI startup(): 204     35395    cpu-e-1021  12
[0] MPI startup(): 205     35396    cpu-e-1021  13
[0] MPI startup(): 206     35397    cpu-e-1021  14
[0] MPI startup(): 207     35398    cpu-e-1021  15
[0] MPI startup(): 208     35399    cpu-e-1021  16
[0] MPI startup(): 209     35400    cpu-e-1021  17
[0] MPI startup(): 210     35401    cpu-e-1021  18
[0] MPI startup(): 211     35402    cpu-e-1021  19
[0] MPI startup(): 212     35403    cpu-e-1021  20
[0] MPI startup(): 213     35404    cpu-e-1021  21
[0] MPI startup(): 214     35405    cpu-e-1021  22
[0] MPI startup(): 215     35406    cpu-e-1021  23
[0] MPI startup(): 216     35407    cpu-e-1021  24
[0] MPI startup(): 217     35408    cpu-e-1021  25
[0] MPI startup(): 218     35409    cpu-e-1021  26
[0] MPI startup(): 219     35410    cpu-e-1021  27
[0] MPI startup(): 220     35411    cpu-e-1021  28
[0] MPI startup(): 221     35412    cpu-e-1021  29
[0] MPI startup(): 222     35413    cpu-e-1021  30
[0] MPI startup(): 223     35414    cpu-e-1021  31
[0] MPI startup(): 224     35145    cpu-e-1022  0
[0] MPI startup(): 225     35146    cpu-e-1022  1
[0] MPI startup(): 226     35147    cpu-e-1022  2
[0] MPI startup(): 227     35148    cpu-e-1022  3
[0] MPI startup(): 228     35149    cpu-e-1022  4
[0] MPI startup(): 229     35150    cpu-e-1022  5
[0] MPI startup(): 230     35151    cpu-e-1022  6
[0] MPI startup(): 231     35152    cpu-e-1022  7
[0] MPI startup(): 232     35153    cpu-e-1022  8
[0] MPI startup(): 233     35154    cpu-e-1022  9
[0] MPI startup(): 234     35155    cpu-e-1022  10
[0] MPI startup(): 235     35156    cpu-e-1022  11
[0] MPI startup(): 236     35157    cpu-e-1022  12
[0] MPI startup(): 237     35158    cpu-e-1022  13
[0] MPI startup(): 238     35159    cpu-e-1022  14
[0] MPI startup(): 239     35160    cpu-e-1022  15
[0] MPI startup(): 240     35161    cpu-e-1022  16
[0] MPI startup(): 241     35162    cpu-e-1022  17
[0] MPI startup(): 242     35163    cpu-e-1022  18
[0] MPI startup(): 243     35164    cpu-e-1022  19
[0] MPI startup(): 244     35165    cpu-e-1022  20
[0] MPI startup(): 245     35166    cpu-e-1022  21
[0] MPI startup(): 246     35167    cpu-e-1022  22
[0] MPI startup(): 247     35168    cpu-e-1022  23
[0] MPI startup(): 248     35169    cpu-e-1022  24
[0] MPI startup(): 249     35170    cpu-e-1022  25
[0] MPI startup(): 250     35171    cpu-e-1022  26
[0] MPI startup(): 251     35172    cpu-e-1022  27
[0] MPI startup(): 252     35173    cpu-e-1022  28
[0] MPI startup(): 253     35174    cpu-e-1022  29
[0] MPI startup(): 254     35175    cpu-e-1022  30
[0] MPI startup(): 255     35176    cpu-e-1022  31
[0] MPI startup(): 256     35165    cpu-e-1149  0
[0] MPI startup(): 257     35166    cpu-e-1149  1
[0] MPI startup(): 258     35167    cpu-e-1149  2
[0] MPI startup(): 259     35168    cpu-e-1149  3
[0] MPI startup(): 260     35169    cpu-e-1149  4
[0] MPI startup(): 261     35170    cpu-e-1149  5
[0] MPI startup(): 262     35171    cpu-e-1149  6
[0] MPI startup(): 263     35172    cpu-e-1149  7
[0] MPI startup(): 264     35173    cpu-e-1149  8
[0] MPI startup(): 265     35174    cpu-e-1149  9
[0] MPI startup(): 266     35175    cpu-e-1149  10
[0] MPI startup(): 267     35176    cpu-e-1149  11
[0] MPI startup(): 268     35177    cpu-e-1149  12
[0] MPI startup(): 269     35178    cpu-e-1149  13
[0] MPI startup(): 270     35179    cpu-e-1149  14
[0] MPI startup(): 271     35180    cpu-e-1149  15
[0] MPI startup(): 272     35181    cpu-e-1149  16
[0] MPI startup(): 273     35182    cpu-e-1149  17
[0] MPI startup(): 274     35183    cpu-e-1149  18
[0] MPI startup(): 275     35184    cpu-e-1149  19
[0] MPI startup(): 276     35185    cpu-e-1149  20
[0] MPI startup(): 277     35186    cpu-e-1149  21
[0] MPI startup(): 278     35187    cpu-e-1149  22
[0] MPI startup(): 279     35188    cpu-e-1149  23
[0] MPI startup(): 280     35189    cpu-e-1149  24
[0] MPI startup(): 281     35190    cpu-e-1149  25
[0] MPI startup(): 282     35191    cpu-e-1149  26
[0] MPI startup(): 283     35192    cpu-e-1149  27
[0] MPI startup(): 284     35193    cpu-e-1149  28
[0] MPI startup(): 285     35194    cpu-e-1149  29
[0] MPI startup(): 286     35195    cpu-e-1149  30
[0] MPI startup(): 287     35196    cpu-e-1149  31
[0] MPI startup(): 288     35238    cpu-e-1150  0
[0] MPI startup(): 289     35239    cpu-e-1150  1
[0] MPI startup(): 290     35240    cpu-e-1150  2
[0] MPI startup(): 291     35241    cpu-e-1150  3
[0] MPI startup(): 292     35242    cpu-e-1150  4
[0] MPI startup(): 293     35243    cpu-e-1150  5
[0] MPI startup(): 294     35244    cpu-e-1150  6
[0] MPI startup(): 295     35245    cpu-e-1150  7
[0] MPI startup(): 296     35246    cpu-e-1150  8
[0] MPI startup(): 297     35247    cpu-e-1150  9
[0] MPI startup(): 298     35248    cpu-e-1150  10
[0] MPI startup(): 299     35249    cpu-e-1150  11
[0] MPI startup(): 300     35250    cpu-e-1150  12
[0] MPI startup(): 301     35251    cpu-e-1150  13
[0] MPI startup(): 302     35252    cpu-e-1150  14
[0] MPI startup(): 303     35253    cpu-e-1150  15
[0] MPI startup(): 304     35254    cpu-e-1150  16
[0] MPI startup(): 305     35255    cpu-e-1150  17
[0] MPI startup(): 306     35256    cpu-e-1150  18
[0] MPI startup(): 307     35257    cpu-e-1150  19
[0] MPI startup(): 308     35258    cpu-e-1150  20
[0] MPI startup(): 309     35259    cpu-e-1150  21
[0] MPI startup(): 310     35260    cpu-e-1150  22
[0] MPI startup(): 311     35261    cpu-e-1150  23
[0] MPI startup(): 312     35262    cpu-e-1150  24
[0] MPI startup(): 313     35263    cpu-e-1150  25
[0] MPI startup(): 314     35264    cpu-e-1150  26
[0] MPI startup(): 315     35265    cpu-e-1150  27
[0] MPI startup(): 316     35266    cpu-e-1150  28
[0] MPI startup(): 317     35267    cpu-e-1150  29
[0] MPI startup(): 318     35268    cpu-e-1150  30
[0] MPI startup(): 319     35269    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
-- started at 11/08/2019 00:33:10 --

mdtest-3.3.0+dev was launched with 320 total task(s) on 10 node(s)
Command line used: /home/wjt27/io-500-sc19/bin/mdtest '-T' '-F' '-P' '-d' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_easy-stonewall' '-N' '1'
Path: /dac/fs1/mjr208/job16760824-2019-11-07-2352
FS: 405.5 TiB   Used FS: 26.5%   Inodes: 1820.8 Mi   Used Inodes: 10.5%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 32 for each phase.
320 tasks, 288000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    1184267.062    1184217.057    1184222.210          3.519
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        161.133        161.126        161.132          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/08/2019 00:35:51 --

mdtest_easy_write
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       34214    cpu-e-829   0
[0] MPI startup(): 1       34215    cpu-e-829   1
[0] MPI startup(): 2       34216    cpu-e-829   2
[0] MPI startup(): 3       34217    cpu-e-829   3
[0] MPI startup(): 4       34218    cpu-e-829   4
[0] MPI startup(): 5       34219    cpu-e-829   5
[0] MPI startup(): 6       34220    cpu-e-829   6
[0] MPI startup(): 7       34221    cpu-e-829   7
[0] MPI startup(): 8       34222    cpu-e-829   8
[0] MPI startup(): 9       34223    cpu-e-829   9
[0] MPI startup(): 10      34224    cpu-e-829   10
[0] MPI startup(): 11      34225    cpu-e-829   11
[0] MPI startup(): 12      34226    cpu-e-829   12
[0] MPI startup(): 13      34227    cpu-e-829   13
[0] MPI startup(): 14      34228    cpu-e-829   14
[0] MPI startup(): 15      34229    cpu-e-829   15
[0] MPI startup(): 16      34230    cpu-e-829   16
[0] MPI startup(): 17      34231    cpu-e-829   17
[0] MPI startup(): 18      34232    cpu-e-829   18
[0] MPI startup(): 19      34233    cpu-e-829   19
[0] MPI startup(): 20      34234    cpu-e-829   20
[0] MPI startup(): 21      34235    cpu-e-829   21
[0] MPI startup(): 22      34236    cpu-e-829   22
[0] MPI startup(): 23      34237    cpu-e-829   23
[0] MPI startup(): 24      34238    cpu-e-829   24
[0] MPI startup(): 25      34239    cpu-e-829   25
[0] MPI startup(): 26      34240    cpu-e-829   26
[0] MPI startup(): 27      34241    cpu-e-829   27
[0] MPI startup(): 28      34242    cpu-e-829   28
[0] MPI startup(): 29      34243    cpu-e-829   29
[0] MPI startup(): 30      34244    cpu-e-829   30
[0] MPI startup(): 31      34245    cpu-e-829   31
[0] MPI startup(): 32      33703    cpu-e-830   0
[0] MPI startup(): 33      33704    cpu-e-830   1
[0] MPI startup(): 34      33705    cpu-e-830   2
[0] MPI startup(): 35      33706    cpu-e-830   3
[0] MPI startup(): 36      33707    cpu-e-830   4
[0] MPI startup(): 37      33708    cpu-e-830   5
[0] MPI startup(): 38      33709    cpu-e-830   6
[0] MPI startup(): 39      33710    cpu-e-830   7
[0] MPI startup(): 40      33711    cpu-e-830   8
[0] MPI startup(): 41      33712    cpu-e-830   9
[0] MPI startup(): 42      33713    cpu-e-830   10
[0] MPI startup(): 43      33714    cpu-e-830   11
[0] MPI startup(): 44      33715    cpu-e-830   12
[0] MPI startup(): 45      33716    cpu-e-830   13
[0] MPI startup(): 46      33717    cpu-e-830   14
[0] MPI startup(): 47      33718    cpu-e-830   15
[0] MPI startup(): 48      33719    cpu-e-830   16
[0] MPI startup(): 49      33720    cpu-e-830   17
[0] MPI startup(): 50      33721    cpu-e-830   18
[0] MPI startup(): 51      33722    cpu-e-830   19
[0] MPI startup(): 52      33723    cpu-e-830   20
[0] MPI startup(): 53      33724    cpu-e-830   21
[0] MPI startup(): 54      33725    cpu-e-830   22
[0] MPI startup(): 55      33726    cpu-e-830   23
[0] MPI startup(): 56      33727    cpu-e-830   24
[0] MPI startup(): 57      33728    cpu-e-830   25
[0] MPI startup(): 58      33729    cpu-e-830   26
[0] MPI startup(): 59      33730    cpu-e-830   27
[0] MPI startup(): 60      33731    cpu-e-830   28
[0] MPI startup(): 61      33732    cpu-e-830   29
[0] MPI startup(): 62      33733    cpu-e-830   30
[0] MPI startup(): 63      33734    cpu-e-830   31
[0] MPI startup(): 64      33524    cpu-e-893   0
[0] MPI startup(): 65      33525    cpu-e-893   1
[0] MPI startup(): 66      33526    cpu-e-893   2
[0] MPI startup(): 67      33527    cpu-e-893   3
[0] MPI startup(): 68      33528    cpu-e-893   4
[0] MPI startup(): 69      33529    cpu-e-893   5
[0] MPI startup(): 70      33530    cpu-e-893   6
[0] MPI startup(): 71      33531    cpu-e-893   7
[0] MPI startup(): 72      33532    cpu-e-893   8
[0] MPI startup(): 73      33533    cpu-e-893   9
[0] MPI startup(): 74      33534    cpu-e-893   10
[0] MPI startup(): 75      33535    cpu-e-893   11
[0] MPI startup(): 76      33536    cpu-e-893   12
[0] MPI startup(): 77      33537    cpu-e-893   13
[0] MPI startup(): 78      33538    cpu-e-893   14
[0] MPI startup(): 79      33539    cpu-e-893   15
[0] MPI startup(): 80      33540    cpu-e-893   16
[0] MPI startup(): 81      33541    cpu-e-893   17
[0] MPI startup(): 82      33542    cpu-e-893   18
[0] MPI startup(): 83      33543    cpu-e-893   19
[0] MPI startup(): 84      33544    cpu-e-893   20
[0] MPI startup(): 85      33545    cpu-e-893   21
[0] MPI startup(): 86      33546    cpu-e-893   22
[0] MPI startup(): 87      33547    cpu-e-893   23
[0] MPI startup(): 88      33548    cpu-e-893   24
[0] MPI startup(): 89      33549    cpu-e-893   25
[0] MPI startup(): 90      33550    cpu-e-893   26
[0] MPI startup(): 91      33551    cpu-e-893   27
[0] MPI startup(): 92      33552    cpu-e-893   28
[0] MPI startup(): 93      33553    cpu-e-893   29
[0] MPI startup(): 94      33554    cpu-e-893   30
[0] MPI startup(): 95      33555    cpu-e-893   31
[0] MPI startup(): 96      33511    cpu-e-894   0
[0] MPI startup(): 97      33512    cpu-e-894   1
[0] MPI startup(): 98      33513    cpu-e-894   2
[0] MPI startup(): 99      33514    cpu-e-894   3
[0] MPI startup(): 100     33515    cpu-e-894   4
[0] MPI startup(): 101     33516    cpu-e-894   5
[0] MPI startup(): 102     33517    cpu-e-894   6
[0] MPI startup(): 103     33518    cpu-e-894   7
[0] MPI startup(): 104     33519    cpu-e-894   8
[0] MPI startup(): 105     33520    cpu-e-894   9
[0] MPI startup(): 106     33521    cpu-e-894   10
[0] MPI startup(): 107     33522    cpu-e-894   11
[0] MPI startup(): 108     33523    cpu-e-894   12
[0] MPI startup(): 109     33524    cpu-e-894   13
[0] MPI startup(): 110     33525    cpu-e-894   14
[0] MPI startup(): 111     33526    cpu-e-894   15
[0] MPI startup(): 112     33527    cpu-e-894   16
[0] MPI startup(): 113     33528    cpu-e-894   17
[0] MPI startup(): 114     33529    cpu-e-894   18
[0] MPI startup(): 115     33530    cpu-e-894   19
[0] MPI startup(): 116     33531    cpu-e-894   20
[0] MPI startup(): 117     33532    cpu-e-894   21
[0] MPI startup(): 118     33533    cpu-e-894   22
[0] MPI startup(): 119     33534    cpu-e-894   23
[0] MPI startup(): 120     33535    cpu-e-894   24
[0] MPI startup(): 121     33536    cpu-e-894   25
[0] MPI startup(): 122     33537    cpu-e-894   26
[0] MPI startup(): 123     33538    cpu-e-894   27
[0] MPI startup(): 124     33539    cpu-e-894   28
[0] MPI startup(): 125     33540    cpu-e-894   29
[0] MPI startup(): 126     33541    cpu-e-894   30
[0] MPI startup(): 127     33542    cpu-e-894   31
[0] MPI startup(): 128     33644    cpu-e-957   0
[0] MPI startup(): 129     33645    cpu-e-957   1
[0] MPI startup(): 130     33646    cpu-e-957   2
[0] MPI startup(): 131     33647    cpu-e-957   3
[0] MPI startup(): 132     33648    cpu-e-957   4
[0] MPI startup(): 133     33649    cpu-e-957   5
[0] MPI startup(): 134     33650    cpu-e-957   6
[0] MPI startup(): 135     33651    cpu-e-957   7
[0] MPI startup(): 136     33652    cpu-e-957   8
[0] MPI startup(): 137     33653    cpu-e-957   9
[0] MPI startup(): 138     33654    cpu-e-957   10
[0] MPI startup(): 139     33655    cpu-e-957   11
[0] MPI startup(): 140     33656    cpu-e-957   12
[0] MPI startup(): 141     33657    cpu-e-957   13
[0] MPI startup(): 142     33658    cpu-e-957   14
[0] MPI startup(): 143     33659    cpu-e-957   15
[0] MPI startup(): 144     33660    cpu-e-957   16
[0] MPI startup(): 145     33661    cpu-e-957   17
[0] MPI startup(): 146     33662    cpu-e-957   18
[0] MPI startup(): 147     33663    cpu-e-957   19
[0] MPI startup(): 148     33664    cpu-e-957   20
[0] MPI startup(): 149     33665    cpu-e-957   21
[0] MPI startup(): 150     33666    cpu-e-957   22
[0] MPI startup(): 151     33667    cpu-e-957   23
[0] MPI startup(): 152     33668    cpu-e-957   24
[0] MPI startup(): 153     33669    cpu-e-957   25
[0] MPI startup(): 154     33670    cpu-e-957   26
[0] MPI startup(): 155     33671    cpu-e-957   27
[0] MPI startup(): 156     33672    cpu-e-957   28
[0] MPI startup(): 157     33673    cpu-e-957   29
[0] MPI startup(): 158     33674    cpu-e-957   30
[0] MPI startup(): 159     33675    cpu-e-957   31
[0] MPI startup(): 160     33603    cpu-e-958   0
[0] MPI startup(): 161     33604    cpu-e-958   1
[0] MPI startup(): 162     33605    cpu-e-958   2
[0] MPI startup(): 163     33606    cpu-e-958   3
[0] MPI startup(): 164     33607    cpu-e-958   4
[0] MPI startup(): 165     33608    cpu-e-958   5
[0] MPI startup(): 166     33609    cpu-e-958   6
[0] MPI startup(): 167     33610    cpu-e-958   7
[0] MPI startup(): 168     33611    cpu-e-958   8
[0] MPI startup(): 169     33612    cpu-e-958   9
[0] MPI startup(): 170     33613    cpu-e-958   10
[0] MPI startup(): 171     33614    cpu-e-958   11
[0] MPI startup(): 172     33615    cpu-e-958   12
[0] MPI startup(): 173     33616    cpu-e-958   13
[0] MPI startup(): 174     33617    cpu-e-958   14
[0] MPI startup(): 175     33618    cpu-e-958   15
[0] MPI startup(): 176     33619    cpu-e-958   16
[0] MPI startup(): 177     33620    cpu-e-958   17
[0] MPI startup(): 178     33621    cpu-e-958   18
[0] MPI startup(): 179     33622    cpu-e-958   19
[0] MPI startup(): 180     33623    cpu-e-958   20
[0] MPI startup(): 181     33624    cpu-e-958   21
[0] MPI startup(): 182     33625    cpu-e-958   22
[0] MPI startup(): 183     33626    cpu-e-958   23
[0] MPI startup(): 184     33627    cpu-e-958   24
[0] MPI startup(): 185     33628    cpu-e-958   25
[0] MPI startup(): 186     33629    cpu-e-958   26
[0] MPI startup(): 187     33630    cpu-e-958   27
[0] MPI startup(): 188     33631    cpu-e-958   28
[0] MPI startup(): 189     33632    cpu-e-958   29
[0] MPI startup(): 190     33633    cpu-e-958   30
[0] MPI startup(): 191     33634    cpu-e-958   31
[0] MPI startup(): 192     33831    cpu-e-1021  0
[0] MPI startup(): 193     33832    cpu-e-1021  1
[0] MPI startup(): 194     33833    cpu-e-1021  2
[0] MPI startup(): 195     33834    cpu-e-1021  3
[0] MPI startup(): 196     33835    cpu-e-1021  4
[0] MPI startup(): 197     33836    cpu-e-1021  5
[0] MPI startup(): 198     33837    cpu-e-1021  6
[0] MPI startup(): 199     33838    cpu-e-1021  7
[0] MPI startup(): 200     33839    cpu-e-1021  8
[0] MPI startup(): 201     33840    cpu-e-1021  9
[0] MPI startup(): 202     33841    cpu-e-1021  10
[0] MPI startup(): 203     33842    cpu-e-1021  11
[0] MPI startup(): 204     33843    cpu-e-1021  12
[0] MPI startup(): 205     33844    cpu-e-1021  13
[0] MPI startup(): 206     33845    cpu-e-1021  14
[0] MPI startup(): 207     33846    cpu-e-1021  15
[0] MPI startup(): 208     33847    cpu-e-1021  16
[0] MPI startup(): 209     33848    cpu-e-1021  17
[0] MPI startup(): 210     33849    cpu-e-1021  18
[0] MPI startup(): 211     33850    cpu-e-1021  19
[0] MPI startup(): 212     33851    cpu-e-1021  20
[0] MPI startup(): 213     33852    cpu-e-1021  21
[0] MPI startup(): 214     33853    cpu-e-1021  22
[0] MPI startup(): 215     33854    cpu-e-1021  23
[0] MPI startup(): 216     33855    cpu-e-1021  24
[0] MPI startup(): 217     33856    cpu-e-1021  25
[0] MPI startup(): 218     33857    cpu-e-1021  26
[0] MPI startup(): 219     33858    cpu-e-1021  27
[0] MPI startup(): 220     33859    cpu-e-1021  28
[0] MPI startup(): 221     33860    cpu-e-1021  29
[0] MPI startup(): 222     33861    cpu-e-1021  30
[0] MPI startup(): 223     33862    cpu-e-1021  31
[0] MPI startup(): 224     33606    cpu-e-1022  0
[0] MPI startup(): 225     33607    cpu-e-1022  1
[0] MPI startup(): 226     33608    cpu-e-1022  2
[0] MPI startup(): 227     33609    cpu-e-1022  3
[0] MPI startup(): 228     33610    cpu-e-1022  4
[0] MPI startup(): 229     33611    cpu-e-1022  5
[0] MPI startup(): 230     33612    cpu-e-1022  6
[0] MPI startup(): 231     33613    cpu-e-1022  7
[0] MPI startup(): 232     33614    cpu-e-1022  8
[0] MPI startup(): 233     33615    cpu-e-1022  9
[0] MPI startup(): 234     33616    cpu-e-1022  10
[0] MPI startup(): 235     33617    cpu-e-1022  11
[0] MPI startup(): 236     33618    cpu-e-1022  12
[0] MPI startup(): 237     33619    cpu-e-1022  13
[0] MPI startup(): 238     33620    cpu-e-1022  14
[0] MPI startup(): 239     33621    cpu-e-1022  15
[0] MPI startup(): 240     33622    cpu-e-1022  16
[0] MPI startup(): 241     33623    cpu-e-1022  17
[0] MPI startup(): 242     33624    cpu-e-1022  18
[0] MPI startup(): 243     33625    cpu-e-1022  19
[0] MPI startup(): 244     33626    cpu-e-1022  20
[0] MPI startup(): 245     33627    cpu-e-1022  21
[0] MPI startup(): 246     33628    cpu-e-1022  22
[0] MPI startup(): 247     33629    cpu-e-1022  23
[0] MPI startup(): 248     33630    cpu-e-1022  24
[0] MPI startup(): 249     33631    cpu-e-1022  25
[0] MPI startup(): 250     33632    cpu-e-1022  26
[0] MPI startup(): 251     33633    cpu-e-1022  27
[0] MPI startup(): 252     33634    cpu-e-1022  28
[0] MPI startup(): 253     33635    cpu-e-1022  29
[0] MPI startup(): 254     33636    cpu-e-1022  30
[0] MPI startup(): 255     33637    cpu-e-1022  31
[0] MPI startup(): 256     33575    cpu-e-1149  0
[0] MPI startup(): 257     33576    cpu-e-1149  1
[0] MPI startup(): 258     33577    cpu-e-1149  2
[0] MPI startup(): 259     33578    cpu-e-1149  3
[0] MPI startup(): 260     33579    cpu-e-1149  4
[0] MPI startup(): 261     33580    cpu-e-1149  5
[0] MPI startup(): 262     33581    cpu-e-1149  6
[0] MPI startup(): 263     33582    cpu-e-1149  7
[0] MPI startup(): 264     33583    cpu-e-1149  8
[0] MPI startup(): 265     33584    cpu-e-1149  9
[0] MPI startup(): 266     33585    cpu-e-1149  10
[0] MPI startup(): 267     33586    cpu-e-1149  11
[0] MPI startup(): 268     33587    cpu-e-1149  12
[0] MPI startup(): 269     33588    cpu-e-1149  13
[0] MPI startup(): 270     33589    cpu-e-1149  14
[0] MPI startup(): 271     33590    cpu-e-1149  15
[0] MPI startup(): 272     33591    cpu-e-1149  16
[0] MPI startup(): 273     33592    cpu-e-1149  17
[0] MPI startup(): 274     33593    cpu-e-1149  18
[0] MPI startup(): 275     33594    cpu-e-1149  19
[0] MPI startup(): 276     33595    cpu-e-1149  20
[0] MPI startup(): 277     33596    cpu-e-1149  21
[0] MPI startup(): 278     33597    cpu-e-1149  22
[0] MPI startup(): 279     33598    cpu-e-1149  23
[0] MPI startup(): 280     33599    cpu-e-1149  24
[0] MPI startup(): 281     33600    cpu-e-1149  25
[0] MPI startup(): 282     33601    cpu-e-1149  26
[0] MPI startup(): 283     33602    cpu-e-1149  27
[0] MPI startup(): 284     33603    cpu-e-1149  28
[0] MPI startup(): 285     33604    cpu-e-1149  29
[0] MPI startup(): 286     33605    cpu-e-1149  30
[0] MPI startup(): 287     33606    cpu-e-1149  31
[0] MPI startup(): 288     33616    cpu-e-1150  0
[0] MPI startup(): 289     33617    cpu-e-1150  1
[0] MPI startup(): 290     33618    cpu-e-1150  2
[0] MPI startup(): 291     33619    cpu-e-1150  3
[0] MPI startup(): 292     33620    cpu-e-1150  4
[0] MPI startup(): 293     33621    cpu-e-1150  5
[0] MPI startup(): 294     33622    cpu-e-1150  6
[0] MPI startup(): 295     33623    cpu-e-1150  7
[0] MPI startup(): 296     33624    cpu-e-1150  8
[0] MPI startup(): 297     33625    cpu-e-1150  9
[0] MPI startup(): 298     33626    cpu-e-1150  10
[0] MPI startup(): 299     33627    cpu-e-1150  11
[0] MPI startup(): 300     33628    cpu-e-1150  12
[0] MPI startup(): 301     33629    cpu-e-1150  13
[0] MPI startup(): 302     33630    cpu-e-1150  14
[0] MPI startup(): 303     33631    cpu-e-1150  15
[0] MPI startup(): 304     33632    cpu-e-1150  16
[0] MPI startup(): 305     33633    cpu-e-1150  17
[0] MPI startup(): 306     33634    cpu-e-1150  18
[0] MPI startup(): 307     33635    cpu-e-1150  19
[0] MPI startup(): 308     33636    cpu-e-1150  20
[0] MPI startup(): 309     33637    cpu-e-1150  21
[0] MPI startup(): 310     33638    cpu-e-1150  22
[0] MPI startup(): 311     33639    cpu-e-1150  23
[0] MPI startup(): 312     33640    cpu-e-1150  24
[0] MPI startup(): 313     33641    cpu-e-1150  25
[0] MPI startup(): 314     33642    cpu-e-1150  26
[0] MPI startup(): 315     33643    cpu-e-1150  27
[0] MPI startup(): 316     33644    cpu-e-1150  28
[0] MPI startup(): 317     33645    cpu-e-1150  29
[0] MPI startup(): 318     33646    cpu-e-1150  30
[0] MPI startup(): 319     33647    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
-- started at 11/07/2019 23:58:42 --

mdtest-3.3.0+dev was launched with 320 total task(s) on 10 node(s)
Command line used: /home/wjt27/io-500-sc19/bin/mdtest '-C' '-F' '-P' '-d' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /dac/fs1/mjr208/job16760824-2019-11-07-2352
FS: 405.5 TiB   Used FS: 25.9%   Inodes: 1638.8 Mi   Used Inodes: 0.0%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 32 for each phase.
320 tasks, 288000000 files
Continue stonewall hit min: 378516 max: 596302 avg: 550437.8 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :     435701.084     435697.676     435699.349          0.732
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA     587800.087             NA
   Tree creation             :          3.157          3.157          3.157          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        437.957        437.953        437.955          0.001
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.660             NA
   Tree creation             :          0.317          0.317          0.317          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/08/2019 00:06:00 --

mdtest_hard_delete
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       38000    cpu-e-829   0
[0] MPI startup(): 1       38001    cpu-e-829   1
[0] MPI startup(): 2       38002    cpu-e-829   2
[0] MPI startup(): 3       38003    cpu-e-829   3
[0] MPI startup(): 4       38004    cpu-e-829   4
[0] MPI startup(): 5       38005    cpu-e-829   5
[0] MPI startup(): 6       38006    cpu-e-829   6
[0] MPI startup(): 7       38007    cpu-e-829   7
[0] MPI startup(): 8       38008    cpu-e-829   8
[0] MPI startup(): 9       38009    cpu-e-829   9
[0] MPI startup(): 10      38010    cpu-e-829   10
[0] MPI startup(): 11      38011    cpu-e-829   11
[0] MPI startup(): 12      38012    cpu-e-829   12
[0] MPI startup(): 13      38013    cpu-e-829   13
[0] MPI startup(): 14      38014    cpu-e-829   14
[0] MPI startup(): 15      38015    cpu-e-829   15
[0] MPI startup(): 16      38016    cpu-e-829   16
[0] MPI startup(): 17      38017    cpu-e-829   17
[0] MPI startup(): 18      38018    cpu-e-829   18
[0] MPI startup(): 19      38019    cpu-e-829   19
[0] MPI startup(): 20      38020    cpu-e-829   20
[0] MPI startup(): 21      38021    cpu-e-829   21
[0] MPI startup(): 22      38022    cpu-e-829   22
[0] MPI startup(): 23      38023    cpu-e-829   23
[0] MPI startup(): 24      38024    cpu-e-829   24
[0] MPI startup(): 25      38025    cpu-e-829   25
[0] MPI startup(): 26      38026    cpu-e-829   26
[0] MPI startup(): 27      38027    cpu-e-829   27
[0] MPI startup(): 28      38028    cpu-e-829   28
[0] MPI startup(): 29      38029    cpu-e-829   29
[0] MPI startup(): 30      38030    cpu-e-829   30
[0] MPI startup(): 31      38031    cpu-e-829   31
[0] MPI startup(): 32      37058    cpu-e-830   0
[0] MPI startup(): 33      37059    cpu-e-830   1
[0] MPI startup(): 34      37060    cpu-e-830   2
[0] MPI startup(): 35      37061    cpu-e-830   3
[0] MPI startup(): 36      37062    cpu-e-830   4
[0] MPI startup(): 37      37063    cpu-e-830   5
[0] MPI startup(): 38      37064    cpu-e-830   6
[0] MPI startup(): 39      37065    cpu-e-830   7
[0] MPI startup(): 40      37066    cpu-e-830   8
[0] MPI startup(): 41      37067    cpu-e-830   9
[0] MPI startup(): 42      37068    cpu-e-830   10
[0] MPI startup(): 43      37069    cpu-e-830   11
[0] MPI startup(): 44      37070    cpu-e-830   12
[0] MPI startup(): 45      37071    cpu-e-830   13
[0] MPI startup(): 46      37072    cpu-e-830   14
[0] MPI startup(): 47      37073    cpu-e-830   15
[0] MPI startup(): 48      37074    cpu-e-830   16
[0] MPI startup(): 49      37075    cpu-e-830   17
[0] MPI startup(): 50      37076    cpu-e-830   18
[0] MPI startup(): 51      37077    cpu-e-830   19
[0] MPI startup(): 52      37078    cpu-e-830   20
[0] MPI startup(): 53      37079    cpu-e-830   21
[0] MPI startup(): 54      37080    cpu-e-830   22
[0] MPI startup(): 55      37081    cpu-e-830   23
[0] MPI startup(): 56      37082    cpu-e-830   24
[0] MPI startup(): 57      37083    cpu-e-830   25
[0] MPI startup(): 58      37084    cpu-e-830   26
[0] MPI startup(): 59      37085    cpu-e-830   27
[0] MPI startup(): 60      37086    cpu-e-830   28
[0] MPI startup(): 61      37087    cpu-e-830   29
[0] MPI startup(): 62      37088    cpu-e-830   30
[0] MPI startup(): 63      37089    cpu-e-830   31
[0] MPI startup(): 64      36693    cpu-e-893   0
[0] MPI startup(): 65      36694    cpu-e-893   1
[0] MPI startup(): 66      36695    cpu-e-893   2
[0] MPI startup(): 67      36696    cpu-e-893   3
[0] MPI startup(): 68      36697    cpu-e-893   4
[0] MPI startup(): 69      36698    cpu-e-893   5
[0] MPI startup(): 70      36699    cpu-e-893   6
[0] MPI startup(): 71      36700    cpu-e-893   7
[0] MPI startup(): 72      36701    cpu-e-893   8
[0] MPI startup(): 73      36702    cpu-e-893   9
[0] MPI startup(): 74      36703    cpu-e-893   10
[0] MPI startup(): 75      36704    cpu-e-893   11
[0] MPI startup(): 76      36705    cpu-e-893   12
[0] MPI startup(): 77      36706    cpu-e-893   13
[0] MPI startup(): 78      36707    cpu-e-893   14
[0] MPI startup(): 79      36708    cpu-e-893   15
[0] MPI startup(): 80      36709    cpu-e-893   16
[0] MPI startup(): 81      36710    cpu-e-893   17
[0] MPI startup(): 82      36711    cpu-e-893   18
[0] MPI startup(): 83      36712    cpu-e-893   19
[0] MPI startup(): 84      36713    cpu-e-893   20
[0] MPI startup(): 85      36714    cpu-e-893   21
[0] MPI startup(): 86      36715    cpu-e-893   22
[0] MPI startup(): 87      36716    cpu-e-893   23
[0] MPI startup(): 88      36717    cpu-e-893   24
[0] MPI startup(): 89      36718    cpu-e-893   25
[0] MPI startup(): 90      36719    cpu-e-893   26
[0] MPI startup(): 91      36720    cpu-e-893   27
[0] MPI startup(): 92      36721    cpu-e-893   28
[0] MPI startup(): 93      36722    cpu-e-893   29
[0] MPI startup(): 94      36723    cpu-e-893   30
[0] MPI startup(): 95      36724    cpu-e-893   31
[0] MPI startup(): 96      36680    cpu-e-894   0
[0] MPI startup(): 97      36681    cpu-e-894   1
[0] MPI startup(): 98      36682    cpu-e-894   2
[0] MPI startup(): 99      36683    cpu-e-894   3
[0] MPI startup(): 100     36684    cpu-e-894   4
[0] MPI startup(): 101     36685    cpu-e-894   5
[0] MPI startup(): 102     36686    cpu-e-894   6
[0] MPI startup(): 103     36687    cpu-e-894   7
[0] MPI startup(): 104     36688    cpu-e-894   8
[0] MPI startup(): 105     36689    cpu-e-894   9
[0] MPI startup(): 106     36690    cpu-e-894   10
[0] MPI startup(): 107     36691    cpu-e-894   11
[0] MPI startup(): 108     36692    cpu-e-894   12
[0] MPI startup(): 109     36693    cpu-e-894   13
[0] MPI startup(): 110     36694    cpu-e-894   14
[0] MPI startup(): 111     36695    cpu-e-894   15
[0] MPI startup(): 112     36696    cpu-e-894   16
[0] MPI startup(): 113     36697    cpu-e-894   17
[0] MPI startup(): 114     36698    cpu-e-894   18
[0] MPI startup(): 115     36699    cpu-e-894   19
[0] MPI startup(): 116     36700    cpu-e-894   20
[0] MPI startup(): 117     36701    cpu-e-894   21
[0] MPI startup(): 118     36702    cpu-e-894   22
[0] MPI startup(): 119     36703    cpu-e-894   23
[0] MPI startup(): 120     36704    cpu-e-894   24
[0] MPI startup(): 121     36705    cpu-e-894   25
[0] MPI startup(): 122     36706    cpu-e-894   26
[0] MPI startup(): 123     36707    cpu-e-894   27
[0] MPI startup(): 124     36708    cpu-e-894   28
[0] MPI startup(): 125     36709    cpu-e-894   29
[0] MPI startup(): 126     36710    cpu-e-894   30
[0] MPI startup(): 127     36711    cpu-e-894   31
[0] MPI startup(): 128     36803    cpu-e-957   0
[0] MPI startup(): 129     36804    cpu-e-957   1
[0] MPI startup(): 130     36805    cpu-e-957   2
[0] MPI startup(): 131     36806    cpu-e-957   3
[0] MPI startup(): 132     36807    cpu-e-957   4
[0] MPI startup(): 133     36808    cpu-e-957   5
[0] MPI startup(): 134     36809    cpu-e-957   6
[0] MPI startup(): 135     36810    cpu-e-957   7
[0] MPI startup(): 136     36811    cpu-e-957   8
[0] MPI startup(): 137     36812    cpu-e-957   9
[0] MPI startup(): 138     36813    cpu-e-957   10
[0] MPI startup(): 139     36814    cpu-e-957   11
[0] MPI startup(): 140     36815    cpu-e-957   12
[0] MPI startup(): 141     36816    cpu-e-957   13
[0] MPI startup(): 142     36817    cpu-e-957   14
[0] MPI startup(): 143     36818    cpu-e-957   15
[0] MPI startup(): 144     36819    cpu-e-957   16
[0] MPI startup(): 145     36820    cpu-e-957   17
[0] MPI startup(): 146     36821    cpu-e-957   18
[0] MPI startup(): 147     36822    cpu-e-957   19
[0] MPI startup(): 148     36823    cpu-e-957   20
[0] MPI startup(): 149     36824    cpu-e-957   21
[0] MPI startup(): 150     36825    cpu-e-957   22
[0] MPI startup(): 151     36826    cpu-e-957   23
[0] MPI startup(): 152     36827    cpu-e-957   24
[0] MPI startup(): 153     36828    cpu-e-957   25
[0] MPI startup(): 154     36829    cpu-e-957   26
[0] MPI startup(): 155     36830    cpu-e-957   27
[0] MPI startup(): 156     36831    cpu-e-957   28
[0] MPI startup(): 157     36832    cpu-e-957   29
[0] MPI startup(): 158     36833    cpu-e-957   30
[0] MPI startup(): 159     36834    cpu-e-957   31
[0] MPI startup(): 160     36756    cpu-e-958   0
[0] MPI startup(): 161     36757    cpu-e-958   1
[0] MPI startup(): 162     36758    cpu-e-958   2
[0] MPI startup(): 163     36759    cpu-e-958   3
[0] MPI startup(): 164     36760    cpu-e-958   4
[0] MPI startup(): 165     36761    cpu-e-958   5
[0] MPI startup(): 166     36762    cpu-e-958   6
[0] MPI startup(): 167     36763    cpu-e-958   7
[0] MPI startup(): 168     36764    cpu-e-958   8
[0] MPI startup(): 169     36765    cpu-e-958   9
[0] MPI startup(): 170     36766    cpu-e-958   10
[0] MPI startup(): 171     36767    cpu-e-958   11
[0] MPI startup(): 172     36768    cpu-e-958   12
[0] MPI startup(): 173     36769    cpu-e-958   13
[0] MPI startup(): 174     36770    cpu-e-958   14
[0] MPI startup(): 175     36771    cpu-e-958   15
[0] MPI startup(): 176     36772    cpu-e-958   16
[0] MPI startup(): 177     36773    cpu-e-958   17
[0] MPI startup(): 178     36774    cpu-e-958   18
[0] MPI startup(): 179     36775    cpu-e-958   19
[0] MPI startup(): 180     36776    cpu-e-958   20
[0] MPI startup(): 181     36777    cpu-e-958   21
[0] MPI startup(): 182     36778    cpu-e-958   22
[0] MPI startup(): 183     36779    cpu-e-958   23
[0] MPI startup(): 184     36780    cpu-e-958   24
[0] MPI startup(): 185     36781    cpu-e-958   25
[0] MPI startup(): 186     36782    cpu-e-958   26
[0] MPI startup(): 187     36783    cpu-e-958   27
[0] MPI startup(): 188     36784    cpu-e-958   28
[0] MPI startup(): 189     36785    cpu-e-958   29
[0] MPI startup(): 190     36786    cpu-e-958   30
[0] MPI startup(): 191     36787    cpu-e-958   31
[0] MPI startup(): 192     36992    cpu-e-1021  0
[0] MPI startup(): 193     36993    cpu-e-1021  1
[0] MPI startup(): 194     36994    cpu-e-1021  2
[0] MPI startup(): 195     36995    cpu-e-1021  3
[0] MPI startup(): 196     36996    cpu-e-1021  4
[0] MPI startup(): 197     36997    cpu-e-1021  5
[0] MPI startup(): 198     36998    cpu-e-1021  6
[0] MPI startup(): 199     36999    cpu-e-1021  7
[0] MPI startup(): 200     37000    cpu-e-1021  8
[0] MPI startup(): 201     37001    cpu-e-1021  9
[0] MPI startup(): 202     37002    cpu-e-1021  10
[0] MPI startup(): 203     37003    cpu-e-1021  11
[0] MPI startup(): 204     37004    cpu-e-1021  12
[0] MPI startup(): 205     37005    cpu-e-1021  13
[0] MPI startup(): 206     37006    cpu-e-1021  14
[0] MPI startup(): 207     37007    cpu-e-1021  15
[0] MPI startup(): 208     37008    cpu-e-1021  16
[0] MPI startup(): 209     37009    cpu-e-1021  17
[0] MPI startup(): 210     37010    cpu-e-1021  18
[0] MPI startup(): 211     37011    cpu-e-1021  19
[0] MPI startup(): 212     37012    cpu-e-1021  20
[0] MPI startup(): 213     37013    cpu-e-1021  21
[0] MPI startup(): 214     37014    cpu-e-1021  22
[0] MPI startup(): 215     37015    cpu-e-1021  23
[0] MPI startup(): 216     37016    cpu-e-1021  24
[0] MPI startup(): 217     37017    cpu-e-1021  25
[0] MPI startup(): 218     37018    cpu-e-1021  26
[0] MPI startup(): 219     37019    cpu-e-1021  27
[0] MPI startup(): 220     37020    cpu-e-1021  28
[0] MPI startup(): 221     37021    cpu-e-1021  29
[0] MPI startup(): 222     37022    cpu-e-1021  30
[0] MPI startup(): 223     37023    cpu-e-1021  31
[0] MPI startup(): 224     36783    cpu-e-1022  0
[0] MPI startup(): 225     36784    cpu-e-1022  1
[0] MPI startup(): 226     36785    cpu-e-1022  2
[0] MPI startup(): 227     36786    cpu-e-1022  3
[0] MPI startup(): 228     36787    cpu-e-1022  4
[0] MPI startup(): 229     36788    cpu-e-1022  5
[0] MPI startup(): 230     36789    cpu-e-1022  6
[0] MPI startup(): 231     36790    cpu-e-1022  7
[0] MPI startup(): 232     36791    cpu-e-1022  8
[0] MPI startup(): 233     36792    cpu-e-1022  9
[0] MPI startup(): 234     36793    cpu-e-1022  10
[0] MPI startup(): 235     36794    cpu-e-1022  11
[0] MPI startup(): 236     36795    cpu-e-1022  12
[0] MPI startup(): 237     36796    cpu-e-1022  13
[0] MPI startup(): 238     36797    cpu-e-1022  14
[0] MPI startup(): 239     36798    cpu-e-1022  15
[0] MPI startup(): 240     36799    cpu-e-1022  16
[0] MPI startup(): 241     36800    cpu-e-1022  17
[0] MPI startup(): 242     36801    cpu-e-1022  18
[0] MPI startup(): 243     36802    cpu-e-1022  19
[0] MPI startup(): 244     36803    cpu-e-1022  20
[0] MPI startup(): 245     36804    cpu-e-1022  21
[0] MPI startup(): 246     36805    cpu-e-1022  22
[0] MPI startup(): 247     36806    cpu-e-1022  23
[0] MPI startup(): 248     36807    cpu-e-1022  24
[0] MPI startup(): 249     36808    cpu-e-1022  25
[0] MPI startup(): 250     36809    cpu-e-1022  26
[0] MPI startup(): 251     36810    cpu-e-1022  27
[0] MPI startup(): 252     36811    cpu-e-1022  28
[0] MPI startup(): 253     36812    cpu-e-1022  29
[0] MPI startup(): 254     36813    cpu-e-1022  30
[0] MPI startup(): 255     36814    cpu-e-1022  31
[0] MPI startup(): 256     36819    cpu-e-1149  0
[0] MPI startup(): 257     36820    cpu-e-1149  1
[0] MPI startup(): 258     36821    cpu-e-1149  2
[0] MPI startup(): 259     36822    cpu-e-1149  3
[0] MPI startup(): 260     36823    cpu-e-1149  4
[0] MPI startup(): 261     36824    cpu-e-1149  5
[0] MPI startup(): 262     36825    cpu-e-1149  6
[0] MPI startup(): 263     36826    cpu-e-1149  7
[0] MPI startup(): 264     36827    cpu-e-1149  8
[0] MPI startup(): 265     36828    cpu-e-1149  9
[0] MPI startup(): 266     36829    cpu-e-1149  10
[0] MPI startup(): 267     36830    cpu-e-1149  11
[0] MPI startup(): 268     36831    cpu-e-1149  12
[0] MPI startup(): 269     36832    cpu-e-1149  13
[0] MPI startup(): 270     36833    cpu-e-1149  14
[0] MPI startup(): 271     36834    cpu-e-1149  15
[0] MPI startup(): 272     36835    cpu-e-1149  16
[0] MPI startup(): 273     36836    cpu-e-1149  17
[0] MPI startup(): 274     36837    cpu-e-1149  18
[0] MPI startup(): 275     36838    cpu-e-1149  19
[0] MPI startup(): 276     36839    cpu-e-1149  20
[0] MPI startup(): 277     36840    cpu-e-1149  21
[0] MPI startup(): 278     36841    cpu-e-1149  22
[0] MPI startup(): 279     36842    cpu-e-1149  23
[0] MPI startup(): 280     36843    cpu-e-1149  24
[0] MPI startup(): 281     36844    cpu-e-1149  25
[0] MPI startup(): 282     36845    cpu-e-1149  26
[0] MPI startup(): 283     36846    cpu-e-1149  27
[0] MPI startup(): 284     36847    cpu-e-1149  28
[0] MPI startup(): 285     36848    cpu-e-1149  29
[0] MPI startup(): 286     36849    cpu-e-1149  30
[0] MPI startup(): 287     36850    cpu-e-1149  31
[0] MPI startup(): 288     36888    cpu-e-1150  0
[0] MPI startup(): 289     36889    cpu-e-1150  1
[0] MPI startup(): 290     36890    cpu-e-1150  2
[0] MPI startup(): 291     36891    cpu-e-1150  3
[0] MPI startup(): 292     36892    cpu-e-1150  4
[0] MPI startup(): 293     36893    cpu-e-1150  5
[0] MPI startup(): 294     36894    cpu-e-1150  6
[0] MPI startup(): 295     36895    cpu-e-1150  7
[0] MPI startup(): 296     36896    cpu-e-1150  8
[0] MPI startup(): 297     36897    cpu-e-1150  9
[0] MPI startup(): 298     36898    cpu-e-1150  10
[0] MPI startup(): 299     36899    cpu-e-1150  11
[0] MPI startup(): 300     36900    cpu-e-1150  12
[0] MPI startup(): 301     36901    cpu-e-1150  13
[0] MPI startup(): 302     36902    cpu-e-1150  14
[0] MPI startup(): 303     36903    cpu-e-1150  15
[0] MPI startup(): 304     36904    cpu-e-1150  16
[0] MPI startup(): 305     36905    cpu-e-1150  17
[0] MPI startup(): 306     36906    cpu-e-1150  18
[0] MPI startup(): 307     36907    cpu-e-1150  19
[0] MPI startup(): 308     36908    cpu-e-1150  20
[0] MPI startup(): 309     36909    cpu-e-1150  21
[0] MPI startup(): 310     36910    cpu-e-1150  22
[0] MPI startup(): 311     36911    cpu-e-1150  23
[0] MPI startup(): 312     36912    cpu-e-1150  24
[0] MPI startup(): 313     36913    cpu-e-1150  25
[0] MPI startup(): 314     36914    cpu-e-1150  26
[0] MPI startup(): 315     36915    cpu-e-1150  27
[0] MPI startup(): 316     36916    cpu-e-1150  28
[0] MPI startup(): 317     36917    cpu-e-1150  29
[0] MPI startup(): 318     36918    cpu-e-1150  30
[0] MPI startup(): 319     36919    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
-- started at 11/08/2019 01:15:19 --

mdtest-3.3.0+dev was launched with 320 total task(s) on 10 node(s)
Command line used: /home/wjt27/io-500-sc19/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_hard' '-n' '950000' '-x' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /dac/fs1/mjr208/job16760824-2019-11-07-2352
FS: 405.5 TiB   Used FS: 26.5%   Inodes: 1638.8 Mi   Used Inodes: 0.6%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 32 for each phase.
320 tasks, 304000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      10745.053      10744.968      10745.012          0.017
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.553          0.553          0.553          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        879.355        879.348        879.351          0.001
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.808          1.808          1.808          0.000
-- finished at 11/08/2019 01:30:00 --

mdtest_hard_read
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       37750    cpu-e-829   0
[0] MPI startup(): 1       37751    cpu-e-829   1
[0] MPI startup(): 2       37752    cpu-e-829   2
[0] MPI startup(): 3       37753    cpu-e-829   3
[0] MPI startup(): 4       37754    cpu-e-829   4
[0] MPI startup(): 5       37755    cpu-e-829   5
[0] MPI startup(): 6       37756    cpu-e-829   6
[0] MPI startup(): 7       37757    cpu-e-829   7
[0] MPI startup(): 8       37758    cpu-e-829   8
[0] MPI startup(): 9       37759    cpu-e-829   9
[0] MPI startup(): 10      37760    cpu-e-829   10
[0] MPI startup(): 11      37761    cpu-e-829   11
[0] MPI startup(): 12      37762    cpu-e-829   12
[0] MPI startup(): 13      37763    cpu-e-829   13
[0] MPI startup(): 14      37764    cpu-e-829   14
[0] MPI startup(): 15      37765    cpu-e-829   15
[0] MPI startup(): 16      37766    cpu-e-829   16
[0] MPI startup(): 17      37767    cpu-e-829   17
[0] MPI startup(): 18      37768    cpu-e-829   18
[0] MPI startup(): 19      37769    cpu-e-829   19
[0] MPI startup(): 20      37770    cpu-e-829   20
[0] MPI startup(): 21      37771    cpu-e-829   21
[0] MPI startup(): 22      37772    cpu-e-829   22
[0] MPI startup(): 23      37773    cpu-e-829   23
[0] MPI startup(): 24      37774    cpu-e-829   24
[0] MPI startup(): 25      37775    cpu-e-829   25
[0] MPI startup(): 26      37776    cpu-e-829   26
[0] MPI startup(): 27      37777    cpu-e-829   27
[0] MPI startup(): 28      37778    cpu-e-829   28
[0] MPI startup(): 29      37779    cpu-e-829   29
[0] MPI startup(): 30      37780    cpu-e-829   30
[0] MPI startup(): 31      37781    cpu-e-829   31
[0] MPI startup(): 32      36852    cpu-e-830   0
[0] MPI startup(): 33      36853    cpu-e-830   1
[0] MPI startup(): 34      36854    cpu-e-830   2
[0] MPI startup(): 35      36855    cpu-e-830   3
[0] MPI startup(): 36      36856    cpu-e-830   4
[0] MPI startup(): 37      36857    cpu-e-830   5
[0] MPI startup(): 38      36858    cpu-e-830   6
[0] MPI startup(): 39      36859    cpu-e-830   7
[0] MPI startup(): 40      36860    cpu-e-830   8
[0] MPI startup(): 41      36861    cpu-e-830   9
[0] MPI startup(): 42      36862    cpu-e-830   10
[0] MPI startup(): 43      36863    cpu-e-830   11
[0] MPI startup(): 44      36864    cpu-e-830   12
[0] MPI startup(): 45      36865    cpu-e-830   13
[0] MPI startup(): 46      36866    cpu-e-830   14
[0] MPI startup(): 47      36867    cpu-e-830   15
[0] MPI startup(): 48      36868    cpu-e-830   16
[0] MPI startup(): 49      36869    cpu-e-830   17
[0] MPI startup(): 50      36870    cpu-e-830   18
[0] MPI startup(): 51      36871    cpu-e-830   19
[0] MPI startup(): 52      36872    cpu-e-830   20
[0] MPI startup(): 53      36873    cpu-e-830   21
[0] MPI startup(): 54      36874    cpu-e-830   22
[0] MPI startup(): 55      36875    cpu-e-830   23
[0] MPI startup(): 56      36876    cpu-e-830   24
[0] MPI startup(): 57      36877    cpu-e-830   25
[0] MPI startup(): 58      36878    cpu-e-830   26
[0] MPI startup(): 59      36879    cpu-e-830   27
[0] MPI startup(): 60      36880    cpu-e-830   28
[0] MPI startup(): 61      36881    cpu-e-830   29
[0] MPI startup(): 62      36882    cpu-e-830   30
[0] MPI startup(): 63      36883    cpu-e-830   31
[0] MPI startup(): 64      36508    cpu-e-893   0
[0] MPI startup(): 65      36509    cpu-e-893   1
[0] MPI startup(): 66      36510    cpu-e-893   2
[0] MPI startup(): 67      36511    cpu-e-893   3
[0] MPI startup(): 68      36512    cpu-e-893   4
[0] MPI startup(): 69      36513    cpu-e-893   5
[0] MPI startup(): 70      36514    cpu-e-893   6
[0] MPI startup(): 71      36515    cpu-e-893   7
[0] MPI startup(): 72      36516    cpu-e-893   8
[0] MPI startup(): 73      36517    cpu-e-893   9
[0] MPI startup(): 74      36518    cpu-e-893   10
[0] MPI startup(): 75      36519    cpu-e-893   11
[0] MPI startup(): 76      36520    cpu-e-893   12
[0] MPI startup(): 77      36521    cpu-e-893   13
[0] MPI startup(): 78      36522    cpu-e-893   14
[0] MPI startup(): 79      36523    cpu-e-893   15
[0] MPI startup(): 80      36524    cpu-e-893   16
[0] MPI startup(): 81      36525    cpu-e-893   17
[0] MPI startup(): 82      36526    cpu-e-893   18
[0] MPI startup(): 83      36527    cpu-e-893   19
[0] MPI startup(): 84      36528    cpu-e-893   20
[0] MPI startup(): 85      36529    cpu-e-893   21
[0] MPI startup(): 86      36530    cpu-e-893   22
[0] MPI startup(): 87      36531    cpu-e-893   23
[0] MPI startup(): 88      36532    cpu-e-893   24
[0] MPI startup(): 89      36533    cpu-e-893   25
[0] MPI startup(): 90      36534    cpu-e-893   26
[0] MPI startup(): 91      36535    cpu-e-893   27
[0] MPI startup(): 92      36536    cpu-e-893   28
[0] MPI startup(): 93      36537    cpu-e-893   29
[0] MPI startup(): 94      36538    cpu-e-893   30
[0] MPI startup(): 95      36539    cpu-e-893   31
[0] MPI startup(): 96      36500    cpu-e-894   0
[0] MPI startup(): 97      36501    cpu-e-894   1
[0] MPI startup(): 98      36502    cpu-e-894   2
[0] MPI startup(): 99      36503    cpu-e-894   3
[0] MPI startup(): 100     36504    cpu-e-894   4
[0] MPI startup(): 101     36505    cpu-e-894   5
[0] MPI startup(): 102     36506    cpu-e-894   6
[0] MPI startup(): 103     36507    cpu-e-894   7
[0] MPI startup(): 104     36508    cpu-e-894   8
[0] MPI startup(): 105     36509    cpu-e-894   9
[0] MPI startup(): 106     36510    cpu-e-894   10
[0] MPI startup(): 107     36511    cpu-e-894   11
[0] MPI startup(): 108     36512    cpu-e-894   12
[0] MPI startup(): 109     36513    cpu-e-894   13
[0] MPI startup(): 110     36514    cpu-e-894   14
[0] MPI startup(): 111     36515    cpu-e-894   15
[0] MPI startup(): 112     36516    cpu-e-894   16
[0] MPI startup(): 113     36517    cpu-e-894   17
[0] MPI startup(): 114     36518    cpu-e-894   18
[0] MPI startup(): 115     36519    cpu-e-894   19
[0] MPI startup(): 116     36520    cpu-e-894   20
[0] MPI startup(): 117     36521    cpu-e-894   21
[0] MPI startup(): 118     36522    cpu-e-894   22
[0] MPI startup(): 119     36523    cpu-e-894   23
[0] MPI startup(): 120     36524    cpu-e-894   24
[0] MPI startup(): 121     36525    cpu-e-894   25
[0] MPI startup(): 122     36526    cpu-e-894   26
[0] MPI startup(): 123     36527    cpu-e-894   27
[0] MPI startup(): 124     36528    cpu-e-894   28
[0] MPI startup(): 125     36529    cpu-e-894   29
[0] MPI startup(): 126     36530    cpu-e-894   30
[0] MPI startup(): 127     36531    cpu-e-894   31
[0] MPI startup(): 128     36611    cpu-e-957   0
[0] MPI startup(): 129     36612    cpu-e-957   1
[0] MPI startup(): 130     36613    cpu-e-957   2
[0] MPI startup(): 131     36614    cpu-e-957   3
[0] MPI startup(): 132     36615    cpu-e-957   4
[0] MPI startup(): 133     36616    cpu-e-957   5
[0] MPI startup(): 134     36617    cpu-e-957   6
[0] MPI startup(): 135     36618    cpu-e-957   7
[0] MPI startup(): 136     36619    cpu-e-957   8
[0] MPI startup(): 137     36620    cpu-e-957   9
[0] MPI startup(): 138     36621    cpu-e-957   10
[0] MPI startup(): 139     36622    cpu-e-957   11
[0] MPI startup(): 140     36623    cpu-e-957   12
[0] MPI startup(): 141     36624    cpu-e-957   13
[0] MPI startup(): 142     36625    cpu-e-957   14
[0] MPI startup(): 143     36626    cpu-e-957   15
[0] MPI startup(): 144     36627    cpu-e-957   16
[0] MPI startup(): 145     36628    cpu-e-957   17
[0] MPI startup(): 146     36629    cpu-e-957   18
[0] MPI startup(): 147     36630    cpu-e-957   19
[0] MPI startup(): 148     36631    cpu-e-957   20
[0] MPI startup(): 149     36632    cpu-e-957   21
[0] MPI startup(): 150     36633    cpu-e-957   22
[0] MPI startup(): 151     36634    cpu-e-957   23
[0] MPI startup(): 152     36635    cpu-e-957   24
[0] MPI startup(): 153     36636    cpu-e-957   25
[0] MPI startup(): 154     36637    cpu-e-957   26
[0] MPI startup(): 155     36638    cpu-e-957   27
[0] MPI startup(): 156     36639    cpu-e-957   28
[0] MPI startup(): 157     36640    cpu-e-957   29
[0] MPI startup(): 158     36641    cpu-e-957   30
[0] MPI startup(): 159     36642    cpu-e-957   31
[0] MPI startup(): 160     36567    cpu-e-958   0
[0] MPI startup(): 161     36568    cpu-e-958   1
[0] MPI startup(): 162     36569    cpu-e-958   2
[0] MPI startup(): 163     36570    cpu-e-958   3
[0] MPI startup(): 164     36571    cpu-e-958   4
[0] MPI startup(): 165     36572    cpu-e-958   5
[0] MPI startup(): 166     36573    cpu-e-958   6
[0] MPI startup(): 167     36574    cpu-e-958   7
[0] MPI startup(): 168     36575    cpu-e-958   8
[0] MPI startup(): 169     36576    cpu-e-958   9
[0] MPI startup(): 170     36577    cpu-e-958   10
[0] MPI startup(): 171     36578    cpu-e-958   11
[0] MPI startup(): 172     36579    cpu-e-958   12
[0] MPI startup(): 173     36580    cpu-e-958   13
[0] MPI startup(): 174     36581    cpu-e-958   14
[0] MPI startup(): 175     36582    cpu-e-958   15
[0] MPI startup(): 176     36583    cpu-e-958   16
[0] MPI startup(): 177     36584    cpu-e-958   17
[0] MPI startup(): 178     36585    cpu-e-958   18
[0] MPI startup(): 179     36586    cpu-e-958   19
[0] MPI startup(): 180     36587    cpu-e-958   20
[0] MPI startup(): 181     36588    cpu-e-958   21
[0] MPI startup(): 182     36589    cpu-e-958   22
[0] MPI startup(): 183     36590    cpu-e-958   23
[0] MPI startup(): 184     36591    cpu-e-958   24
[0] MPI startup(): 185     36592    cpu-e-958   25
[0] MPI startup(): 186     36593    cpu-e-958   26
[0] MPI startup(): 187     36594    cpu-e-958   27
[0] MPI startup(): 188     36595    cpu-e-958   28
[0] MPI startup(): 189     36596    cpu-e-958   29
[0] MPI startup(): 190     36597    cpu-e-958   30
[0] MPI startup(): 191     36598    cpu-e-958   31
[0] MPI startup(): 192     36812    cpu-e-1021  0
[0] MPI startup(): 193     36813    cpu-e-1021  1
[0] MPI startup(): 194     36814    cpu-e-1021  2
[0] MPI startup(): 195     36815    cpu-e-1021  3
[0] MPI startup(): 196     36816    cpu-e-1021  4
[0] MPI startup(): 197     36817    cpu-e-1021  5
[0] MPI startup(): 198     36818    cpu-e-1021  6
[0] MPI startup(): 199     36819    cpu-e-1021  7
[0] MPI startup(): 200     36820    cpu-e-1021  8
[0] MPI startup(): 201     36821    cpu-e-1021  9
[0] MPI startup(): 202     36822    cpu-e-1021  10
[0] MPI startup(): 203     36823    cpu-e-1021  11
[0] MPI startup(): 204     36824    cpu-e-1021  12
[0] MPI startup(): 205     36825    cpu-e-1021  13
[0] MPI startup(): 206     36826    cpu-e-1021  14
[0] MPI startup(): 207     36827    cpu-e-1021  15
[0] MPI startup(): 208     36828    cpu-e-1021  16
[0] MPI startup(): 209     36829    cpu-e-1021  17
[0] MPI startup(): 210     36830    cpu-e-1021  18
[0] MPI startup(): 211     36831    cpu-e-1021  19
[0] MPI startup(): 212     36832    cpu-e-1021  20
[0] MPI startup(): 213     36833    cpu-e-1021  21
[0] MPI startup(): 214     36834    cpu-e-1021  22
[0] MPI startup(): 215     36835    cpu-e-1021  23
[0] MPI startup(): 216     36836    cpu-e-1021  24
[0] MPI startup(): 217     36837    cpu-e-1021  25
[0] MPI startup(): 218     36838    cpu-e-1021  26
[0] MPI startup(): 219     36839    cpu-e-1021  27
[0] MPI startup(): 220     36840    cpu-e-1021  28
[0] MPI startup(): 221     36841    cpu-e-1021  29
[0] MPI startup(): 222     36842    cpu-e-1021  30
[0] MPI startup(): 223     36843    cpu-e-1021  31
[0] MPI startup(): 224     36589    cpu-e-1022  0
[0] MPI startup(): 225     36590    cpu-e-1022  1
[0] MPI startup(): 226     36591    cpu-e-1022  2
[0] MPI startup(): 227     36592    cpu-e-1022  3
[0] MPI startup(): 228     36593    cpu-e-1022  4
[0] MPI startup(): 229     36594    cpu-e-1022  5
[0] MPI startup(): 230     36595    cpu-e-1022  6
[0] MPI startup(): 231     36596    cpu-e-1022  7
[0] MPI startup(): 232     36597    cpu-e-1022  8
[0] MPI startup(): 233     36598    cpu-e-1022  9
[0] MPI startup(): 234     36599    cpu-e-1022  10
[0] MPI startup(): 235     36600    cpu-e-1022  11
[0] MPI startup(): 236     36601    cpu-e-1022  12
[0] MPI startup(): 237     36602    cpu-e-1022  13
[0] MPI startup(): 238     36603    cpu-e-1022  14
[0] MPI startup(): 239     36604    cpu-e-1022  15
[0] MPI startup(): 240     36605    cpu-e-1022  16
[0] MPI startup(): 241     36606    cpu-e-1022  17
[0] MPI startup(): 242     36607    cpu-e-1022  18
[0] MPI startup(): 243     36608    cpu-e-1022  19
[0] MPI startup(): 244     36609    cpu-e-1022  20
[0] MPI startup(): 245     36610    cpu-e-1022  21
[0] MPI startup(): 246     36611    cpu-e-1022  22
[0] MPI startup(): 247     36612    cpu-e-1022  23
[0] MPI startup(): 248     36613    cpu-e-1022  24
[0] MPI startup(): 249     36614    cpu-e-1022  25
[0] MPI startup(): 250     36615    cpu-e-1022  26
[0] MPI startup(): 251     36616    cpu-e-1022  27
[0] MPI startup(): 252     36617    cpu-e-1022  28
[0] MPI startup(): 253     36618    cpu-e-1022  29
[0] MPI startup(): 254     36619    cpu-e-1022  30
[0] MPI startup(): 255     36620    cpu-e-1022  31
[0] MPI startup(): 256     36628    cpu-e-1149  0
[0] MPI startup(): 257     36629    cpu-e-1149  1
[0] MPI startup(): 258     36630    cpu-e-1149  2
[0] MPI startup(): 259     36631    cpu-e-1149  3
[0] MPI startup(): 260     36632    cpu-e-1149  4
[0] MPI startup(): 261     36633    cpu-e-1149  5
[0] MPI startup(): 262     36634    cpu-e-1149  6
[0] MPI startup(): 263     36635    cpu-e-1149  7
[0] MPI startup(): 264     36636    cpu-e-1149  8
[0] MPI startup(): 265     36637    cpu-e-1149  9
[0] MPI startup(): 266     36638    cpu-e-1149  10
[0] MPI startup(): 267     36639    cpu-e-1149  11
[0] MPI startup(): 268     36640    cpu-e-1149  12
[0] MPI startup(): 269     36641    cpu-e-1149  13
[0] MPI startup(): 270     36642    cpu-e-1149  14
[0] MPI startup(): 271     36643    cpu-e-1149  15
[0] MPI startup(): 272     36644    cpu-e-1149  16
[0] MPI startup(): 273     36645    cpu-e-1149  17
[0] MPI startup(): 274     36646    cpu-e-1149  18
[0] MPI startup(): 275     36647    cpu-e-1149  19
[0] MPI startup(): 276     36648    cpu-e-1149  20
[0] MPI startup(): 277     36649    cpu-e-1149  21
[0] MPI startup(): 278     36650    cpu-e-1149  22
[0] MPI startup(): 279     36651    cpu-e-1149  23
[0] MPI startup(): 280     36652    cpu-e-1149  24
[0] MPI startup(): 281     36653    cpu-e-1149  25
[0] MPI startup(): 282     36654    cpu-e-1149  26
[0] MPI startup(): 283     36655    cpu-e-1149  27
[0] MPI startup(): 284     36656    cpu-e-1149  28
[0] MPI startup(): 285     36657    cpu-e-1149  29
[0] MPI startup(): 286     36658    cpu-e-1149  30
[0] MPI startup(): 287     36659    cpu-e-1149  31
[0] MPI startup(): 288     36699    cpu-e-1150  0
[0] MPI startup(): 289     36700    cpu-e-1150  1
[0] MPI startup(): 290     36701    cpu-e-1150  2
[0] MPI startup(): 291     36702    cpu-e-1150  3
[0] MPI startup(): 292     36703    cpu-e-1150  4
[0] MPI startup(): 293     36704    cpu-e-1150  5
[0] MPI startup(): 294     36705    cpu-e-1150  6
[0] MPI startup(): 295     36706    cpu-e-1150  7
[0] MPI startup(): 296     36707    cpu-e-1150  8
[0] MPI startup(): 297     36708    cpu-e-1150  9
[0] MPI startup(): 298     36709    cpu-e-1150  10
[0] MPI startup(): 299     36710    cpu-e-1150  11
[0] MPI startup(): 300     36711    cpu-e-1150  12
[0] MPI startup(): 301     36712    cpu-e-1150  13
[0] MPI startup(): 302     36713    cpu-e-1150  14
[0] MPI startup(): 303     36714    cpu-e-1150  15
[0] MPI startup(): 304     36715    cpu-e-1150  16
[0] MPI startup(): 305     36716    cpu-e-1150  17
[0] MPI startup(): 306     36717    cpu-e-1150  18
[0] MPI startup(): 307     36718    cpu-e-1150  19
[0] MPI startup(): 308     36719    cpu-e-1150  20
[0] MPI startup(): 309     36720    cpu-e-1150  21
[0] MPI startup(): 310     36721    cpu-e-1150  22
[0] MPI startup(): 311     36722    cpu-e-1150  23
[0] MPI startup(): 312     36723    cpu-e-1150  24
[0] MPI startup(): 313     36724    cpu-e-1150  25
[0] MPI startup(): 314     36725    cpu-e-1150  26
[0] MPI startup(): 315     36726    cpu-e-1150  27
[0] MPI startup(): 316     36727    cpu-e-1150  28
[0] MPI startup(): 317     36728    cpu-e-1150  29
[0] MPI startup(): 318     36729    cpu-e-1150  30
[0] MPI startup(): 319     36730    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
-- started at 11/08/2019 01:12:36 --

mdtest-3.3.0+dev was launched with 320 total task(s) on 10 node(s)
Command line used: /home/wjt27/io-500-sc19/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_hard' '-n' '950000' '-x' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /dac/fs1/mjr208/job16760824-2019-11-07-2352
FS: 405.5 TiB   Used FS: 26.5%   Inodes: 1638.8 Mi   Used Inodes: 0.6%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 32 for each phase.
320 tasks, 304000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      59007.576      59004.661      59004.914          0.178
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :        160.134        160.126        160.133          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/08/2019 01:15:16 --

mdtest_hard_stat
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       36713    cpu-e-829   0
[0] MPI startup(): 1       36714    cpu-e-829   1
[0] MPI startup(): 2       36715    cpu-e-829   2
[0] MPI startup(): 3       36716    cpu-e-829   3
[0] MPI startup(): 4       36717    cpu-e-829   4
[0] MPI startup(): 5       36718    cpu-e-829   5
[0] MPI startup(): 6       36719    cpu-e-829   6
[0] MPI startup(): 7       36720    cpu-e-829   7
[0] MPI startup(): 8       36721    cpu-e-829   8
[0] MPI startup(): 9       36722    cpu-e-829   9
[0] MPI startup(): 10      36723    cpu-e-829   10
[0] MPI startup(): 11      36724    cpu-e-829   11
[0] MPI startup(): 12      36725    cpu-e-829   12
[0] MPI startup(): 13      36726    cpu-e-829   13
[0] MPI startup(): 14      36727    cpu-e-829   14
[0] MPI startup(): 15      36728    cpu-e-829   15
[0] MPI startup(): 16      36729    cpu-e-829   16
[0] MPI startup(): 17      36730    cpu-e-829   17
[0] MPI startup(): 18      36731    cpu-e-829   18
[0] MPI startup(): 19      36732    cpu-e-829   19
[0] MPI startup(): 20      36733    cpu-e-829   20
[0] MPI startup(): 21      36734    cpu-e-829   21
[0] MPI startup(): 22      36735    cpu-e-829   22
[0] MPI startup(): 23      36736    cpu-e-829   23
[0] MPI startup(): 24      36737    cpu-e-829   24
[0] MPI startup(): 25      36738    cpu-e-829   25
[0] MPI startup(): 26      36739    cpu-e-829   26
[0] MPI startup(): 27      36740    cpu-e-829   27
[0] MPI startup(): 28      36741    cpu-e-829   28
[0] MPI startup(): 29      36742    cpu-e-829   29
[0] MPI startup(): 30      36743    cpu-e-829   30
[0] MPI startup(): 31      36744    cpu-e-829   31
[0] MPI startup(): 32      35942    cpu-e-830   0
[0] MPI startup(): 33      35943    cpu-e-830   1
[0] MPI startup(): 34      35944    cpu-e-830   2
[0] MPI startup(): 35      35945    cpu-e-830   3
[0] MPI startup(): 36      35946    cpu-e-830   4
[0] MPI startup(): 37      35947    cpu-e-830   5
[0] MPI startup(): 38      35948    cpu-e-830   6
[0] MPI startup(): 39      35949    cpu-e-830   7
[0] MPI startup(): 40      35950    cpu-e-830   8
[0] MPI startup(): 41      35951    cpu-e-830   9
[0] MPI startup(): 42      35952    cpu-e-830   10
[0] MPI startup(): 43      35953    cpu-e-830   11
[0] MPI startup(): 44      35954    cpu-e-830   12
[0] MPI startup(): 45      35955    cpu-e-830   13
[0] MPI startup(): 46      35956    cpu-e-830   14
[0] MPI startup(): 47      35957    cpu-e-830   15
[0] MPI startup(): 48      35958    cpu-e-830   16
[0] MPI startup(): 49      35959    cpu-e-830   17
[0] MPI startup(): 50      35960    cpu-e-830   18
[0] MPI startup(): 51      35961    cpu-e-830   19
[0] MPI startup(): 52      35962    cpu-e-830   20
[0] MPI startup(): 53      35963    cpu-e-830   21
[0] MPI startup(): 54      35964    cpu-e-830   22
[0] MPI startup(): 55      35965    cpu-e-830   23
[0] MPI startup(): 56      35966    cpu-e-830   24
[0] MPI startup(): 57      35967    cpu-e-830   25
[0] MPI startup(): 58      35968    cpu-e-830   26
[0] MPI startup(): 59      35969    cpu-e-830   27
[0] MPI startup(): 60      35970    cpu-e-830   28
[0] MPI startup(): 61      35971    cpu-e-830   29
[0] MPI startup(): 62      35972    cpu-e-830   30
[0] MPI startup(): 63      35973    cpu-e-830   31
[0] MPI startup(): 64      35591    cpu-e-893   0
[0] MPI startup(): 65      35592    cpu-e-893   1
[0] MPI startup(): 66      35593    cpu-e-893   2
[0] MPI startup(): 67      35594    cpu-e-893   3
[0] MPI startup(): 68      35595    cpu-e-893   4
[0] MPI startup(): 69      35596    cpu-e-893   5
[0] MPI startup(): 70      35597    cpu-e-893   6
[0] MPI startup(): 71      35598    cpu-e-893   7
[0] MPI startup(): 72      35599    cpu-e-893   8
[0] MPI startup(): 73      35600    cpu-e-893   9
[0] MPI startup(): 74      35601    cpu-e-893   10
[0] MPI startup(): 75      35602    cpu-e-893   11
[0] MPI startup(): 76      35603    cpu-e-893   12
[0] MPI startup(): 77      35604    cpu-e-893   13
[0] MPI startup(): 78      35605    cpu-e-893   14
[0] MPI startup(): 79      35606    cpu-e-893   15
[0] MPI startup(): 80      35607    cpu-e-893   16
[0] MPI startup(): 81      35608    cpu-e-893   17
[0] MPI startup(): 82      35609    cpu-e-893   18
[0] MPI startup(): 83      35610    cpu-e-893   19
[0] MPI startup(): 84      35611    cpu-e-893   20
[0] MPI startup(): 85      35612    cpu-e-893   21
[0] MPI startup(): 86      35613    cpu-e-893   22
[0] MPI startup(): 87      35614    cpu-e-893   23
[0] MPI startup(): 88      35615    cpu-e-893   24
[0] MPI startup(): 89      35616    cpu-e-893   25
[0] MPI startup(): 90      35617    cpu-e-893   26
[0] MPI startup(): 91      35618    cpu-e-893   27
[0] MPI startup(): 92      35619    cpu-e-893   28
[0] MPI startup(): 93      35620    cpu-e-893   29
[0] MPI startup(): 94      35621    cpu-e-893   30
[0] MPI startup(): 95      35622    cpu-e-893   31
[0] MPI startup(): 96      35585    cpu-e-894   0
[0] MPI startup(): 97      35586    cpu-e-894   1
[0] MPI startup(): 98      35587    cpu-e-894   2
[0] MPI startup(): 99      35588    cpu-e-894   3
[0] MPI startup(): 100     35589    cpu-e-894   4
[0] MPI startup(): 101     35590    cpu-e-894   5
[0] MPI startup(): 102     35591    cpu-e-894   6
[0] MPI startup(): 103     35592    cpu-e-894   7
[0] MPI startup(): 104     35593    cpu-e-894   8
[0] MPI startup(): 105     35594    cpu-e-894   9
[0] MPI startup(): 106     35595    cpu-e-894   10
[0] MPI startup(): 107     35596    cpu-e-894   11
[0] MPI startup(): 108     35597    cpu-e-894   12
[0] MPI startup(): 109     35598    cpu-e-894   13
[0] MPI startup(): 110     35599    cpu-e-894   14
[0] MPI startup(): 111     35600    cpu-e-894   15
[0] MPI startup(): 112     35601    cpu-e-894   16
[0] MPI startup(): 113     35602    cpu-e-894   17
[0] MPI startup(): 114     35603    cpu-e-894   18
[0] MPI startup(): 115     35604    cpu-e-894   19
[0] MPI startup(): 116     35605    cpu-e-894   20
[0] MPI startup(): 117     35606    cpu-e-894   21
[0] MPI startup(): 118     35607    cpu-e-894   22
[0] MPI startup(): 119     35608    cpu-e-894   23
[0] MPI startup(): 120     35609    cpu-e-894   24
[0] MPI startup(): 121     35610    cpu-e-894   25
[0] MPI startup(): 122     35611    cpu-e-894   26
[0] MPI startup(): 123     35612    cpu-e-894   27
[0] MPI startup(): 124     35613    cpu-e-894   28
[0] MPI startup(): 125     35614    cpu-e-894   29
[0] MPI startup(): 126     35615    cpu-e-894   30
[0] MPI startup(): 127     35616    cpu-e-894   31
[0] MPI startup(): 128     35709    cpu-e-957   0
[0] MPI startup(): 129     35710    cpu-e-957   1
[0] MPI startup(): 130     35711    cpu-e-957   2
[0] MPI startup(): 131     35712    cpu-e-957   3
[0] MPI startup(): 132     35713    cpu-e-957   4
[0] MPI startup(): 133     35714    cpu-e-957   5
[0] MPI startup(): 134     35715    cpu-e-957   6
[0] MPI startup(): 135     35716    cpu-e-957   7
[0] MPI startup(): 136     35717    cpu-e-957   8
[0] MPI startup(): 137     35718    cpu-e-957   9
[0] MPI startup(): 138     35719    cpu-e-957   10
[0] MPI startup(): 139     35720    cpu-e-957   11
[0] MPI startup(): 140     35721    cpu-e-957   12
[0] MPI startup(): 141     35722    cpu-e-957   13
[0] MPI startup(): 142     35723    cpu-e-957   14
[0] MPI startup(): 143     35724    cpu-e-957   15
[0] MPI startup(): 144     35725    cpu-e-957   16
[0] MPI startup(): 145     35726    cpu-e-957   17
[0] MPI startup(): 146     35727    cpu-e-957   18
[0] MPI startup(): 147     35728    cpu-e-957   19
[0] MPI startup(): 148     35729    cpu-e-957   20
[0] MPI startup(): 149     35730    cpu-e-957   21
[0] MPI startup(): 150     35731    cpu-e-957   22
[0] MPI startup(): 151     35732    cpu-e-957   23
[0] MPI startup(): 152     35733    cpu-e-957   24
[0] MPI startup(): 153     35734    cpu-e-957   25
[0] MPI startup(): 154     35735    cpu-e-957   26
[0] MPI startup(): 155     35736    cpu-e-957   27
[0] MPI startup(): 156     35737    cpu-e-957   28
[0] MPI startup(): 157     35738    cpu-e-957   29
[0] MPI startup(): 158     35739    cpu-e-957   30
[0] MPI startup(): 159     35740    cpu-e-957   31
[0] MPI startup(): 160     35664    cpu-e-958   0
[0] MPI startup(): 161     35665    cpu-e-958   1
[0] MPI startup(): 162     35666    cpu-e-958   2
[0] MPI startup(): 163     35667    cpu-e-958   3
[0] MPI startup(): 164     35668    cpu-e-958   4
[0] MPI startup(): 165     35669    cpu-e-958   5
[0] MPI startup(): 166     35670    cpu-e-958   6
[0] MPI startup(): 167     35671    cpu-e-958   7
[0] MPI startup(): 168     35672    cpu-e-958   8
[0] MPI startup(): 169     35673    cpu-e-958   9
[0] MPI startup(): 170     35674    cpu-e-958   10
[0] MPI startup(): 171     35675    cpu-e-958   11
[0] MPI startup(): 172     35676    cpu-e-958   12
[0] MPI startup(): 173     35677    cpu-e-958   13
[0] MPI startup(): 174     35678    cpu-e-958   14
[0] MPI startup(): 175     35679    cpu-e-958   15
[0] MPI startup(): 176     35680    cpu-e-958   16
[0] MPI startup(): 177     35681    cpu-e-958   17
[0] MPI startup(): 178     35682    cpu-e-958   18
[0] MPI startup(): 179     35683    cpu-e-958   19
[0] MPI startup(): 180     35684    cpu-e-958   20
[0] MPI startup(): 181     35685    cpu-e-958   21
[0] MPI startup(): 182     35686    cpu-e-958   22
[0] MPI startup(): 183     35687    cpu-e-958   23
[0] MPI startup(): 184     35688    cpu-e-958   24
[0] MPI startup(): 185     35689    cpu-e-958   25
[0] MPI startup(): 186     35690    cpu-e-958   26
[0] MPI startup(): 187     35691    cpu-e-958   27
[0] MPI startup(): 188     35692    cpu-e-958   28
[0] MPI startup(): 189     35693    cpu-e-958   29
[0] MPI startup(): 190     35694    cpu-e-958   30
[0] MPI startup(): 191     35695    cpu-e-958   31
[0] MPI startup(): 192     35918    cpu-e-1021  0
[0] MPI startup(): 193     35919    cpu-e-1021  1
[0] MPI startup(): 194     35920    cpu-e-1021  2
[0] MPI startup(): 195     35921    cpu-e-1021  3
[0] MPI startup(): 196     35922    cpu-e-1021  4
[0] MPI startup(): 197     35923    cpu-e-1021  5
[0] MPI startup(): 198     35924    cpu-e-1021  6
[0] MPI startup(): 199     35925    cpu-e-1021  7
[0] MPI startup(): 200     35926    cpu-e-1021  8
[0] MPI startup(): 201     35927    cpu-e-1021  9
[0] MPI startup(): 202     35928    cpu-e-1021  10
[0] MPI startup(): 203     35929    cpu-e-1021  11
[0] MPI startup(): 204     35930    cpu-e-1021  12
[0] MPI startup(): 205     35931    cpu-e-1021  13
[0] MPI startup(): 206     35932    cpu-e-1021  14
[0] MPI startup(): 207     35933    cpu-e-1021  15
[0] MPI startup(): 208     35934    cpu-e-1021  16
[0] MPI startup(): 209     35935    cpu-e-1021  17
[0] MPI startup(): 210     35936    cpu-e-1021  18
[0] MPI startup(): 211     35937    cpu-e-1021  19
[0] MPI startup(): 212     35938    cpu-e-1021  20
[0] MPI startup(): 213     35939    cpu-e-1021  21
[0] MPI startup(): 214     35940    cpu-e-1021  22
[0] MPI startup(): 215     35941    cpu-e-1021  23
[0] MPI startup(): 216     35942    cpu-e-1021  24
[0] MPI startup(): 217     35943    cpu-e-1021  25
[0] MPI startup(): 218     35944    cpu-e-1021  26
[0] MPI startup(): 219     35945    cpu-e-1021  27
[0] MPI startup(): 220     35946    cpu-e-1021  28
[0] MPI startup(): 221     35947    cpu-e-1021  29
[0] MPI startup(): 222     35948    cpu-e-1021  30
[0] MPI startup(): 223     35949    cpu-e-1021  31
[0] MPI startup(): 224     35689    cpu-e-1022  0
[0] MPI startup(): 225     35690    cpu-e-1022  1
[0] MPI startup(): 226     35691    cpu-e-1022  2
[0] MPI startup(): 227     35692    cpu-e-1022  3
[0] MPI startup(): 228     35693    cpu-e-1022  4
[0] MPI startup(): 229     35694    cpu-e-1022  5
[0] MPI startup(): 230     35695    cpu-e-1022  6
[0] MPI startup(): 231     35696    cpu-e-1022  7
[0] MPI startup(): 232     35697    cpu-e-1022  8
[0] MPI startup(): 233     35698    cpu-e-1022  9
[0] MPI startup(): 234     35699    cpu-e-1022  10
[0] MPI startup(): 235     35700    cpu-e-1022  11
[0] MPI startup(): 236     35701    cpu-e-1022  12
[0] MPI startup(): 237     35702    cpu-e-1022  13
[0] MPI startup(): 238     35703    cpu-e-1022  14
[0] MPI startup(): 239     35704    cpu-e-1022  15
[0] MPI startup(): 240     35705    cpu-e-1022  16
[0] MPI startup(): 241     35706    cpu-e-1022  17
[0] MPI startup(): 242     35707    cpu-e-1022  18
[0] MPI startup(): 243     35708    cpu-e-1022  19
[0] MPI startup(): 244     35709    cpu-e-1022  20
[0] MPI startup(): 245     35710    cpu-e-1022  21
[0] MPI startup(): 246     35711    cpu-e-1022  22
[0] MPI startup(): 247     35712    cpu-e-1022  23
[0] MPI startup(): 248     35713    cpu-e-1022  24
[0] MPI startup(): 249     35714    cpu-e-1022  25
[0] MPI startup(): 250     35715    cpu-e-1022  26
[0] MPI startup(): 251     35716    cpu-e-1022  27
[0] MPI startup(): 252     35717    cpu-e-1022  28
[0] MPI startup(): 253     35718    cpu-e-1022  29
[0] MPI startup(): 254     35719    cpu-e-1022  30
[0] MPI startup(): 255     35720    cpu-e-1022  31
[0] MPI startup(): 256     35718    cpu-e-1149  0
[0] MPI startup(): 257     35719    cpu-e-1149  1
[0] MPI startup(): 258     35720    cpu-e-1149  2
[0] MPI startup(): 259     35721    cpu-e-1149  3
[0] MPI startup(): 260     35722    cpu-e-1149  4
[0] MPI startup(): 261     35723    cpu-e-1149  5
[0] MPI startup(): 262     35724    cpu-e-1149  6
[0] MPI startup(): 263     35725    cpu-e-1149  7
[0] MPI startup(): 264     35726    cpu-e-1149  8
[0] MPI startup(): 265     35727    cpu-e-1149  9
[0] MPI startup(): 266     35728    cpu-e-1149  10
[0] MPI startup(): 267     35729    cpu-e-1149  11
[0] MPI startup(): 268     35730    cpu-e-1149  12
[0] MPI startup(): 269     35731    cpu-e-1149  13
[0] MPI startup(): 270     35732    cpu-e-1149  14
[0] MPI startup(): 271     35733    cpu-e-1149  15
[0] MPI startup(): 272     35734    cpu-e-1149  16
[0] MPI startup(): 273     35735    cpu-e-1149  17
[0] MPI startup(): 274     35736    cpu-e-1149  18
[0] MPI startup(): 275     35737    cpu-e-1149  19
[0] MPI startup(): 276     35738    cpu-e-1149  20
[0] MPI startup(): 277     35739    cpu-e-1149  21
[0] MPI startup(): 278     35740    cpu-e-1149  22
[0] MPI startup(): 279     35741    cpu-e-1149  23
[0] MPI startup(): 280     35742    cpu-e-1149  24
[0] MPI startup(): 281     35743    cpu-e-1149  25
[0] MPI startup(): 282     35744    cpu-e-1149  26
[0] MPI startup(): 283     35745    cpu-e-1149  27
[0] MPI startup(): 284     35746    cpu-e-1149  28
[0] MPI startup(): 285     35747    cpu-e-1149  29
[0] MPI startup(): 286     35748    cpu-e-1149  30
[0] MPI startup(): 287     35749    cpu-e-1149  31
[0] MPI startup(): 288     35777    cpu-e-1150  0
[0] MPI startup(): 289     35778    cpu-e-1150  1
[0] MPI startup(): 290     35779    cpu-e-1150  2
[0] MPI startup(): 291     35780    cpu-e-1150  3
[0] MPI startup(): 292     35781    cpu-e-1150  4
[0] MPI startup(): 293     35782    cpu-e-1150  5
[0] MPI startup(): 294     35783    cpu-e-1150  6
[0] MPI startup(): 295     35784    cpu-e-1150  7
[0] MPI startup(): 296     35785    cpu-e-1150  8
[0] MPI startup(): 297     35786    cpu-e-1150  9
[0] MPI startup(): 298     35787    cpu-e-1150  10
[0] MPI startup(): 299     35788    cpu-e-1150  11
[0] MPI startup(): 300     35789    cpu-e-1150  12
[0] MPI startup(): 301     35790    cpu-e-1150  13
[0] MPI startup(): 302     35791    cpu-e-1150  14
[0] MPI startup(): 303     35792    cpu-e-1150  15
[0] MPI startup(): 304     35793    cpu-e-1150  16
[0] MPI startup(): 305     35794    cpu-e-1150  17
[0] MPI startup(): 306     35795    cpu-e-1150  18
[0] MPI startup(): 307     35796    cpu-e-1150  19
[0] MPI startup(): 308     35797    cpu-e-1150  20
[0] MPI startup(): 309     35798    cpu-e-1150  21
[0] MPI startup(): 310     35799    cpu-e-1150  22
[0] MPI startup(): 311     35800    cpu-e-1150  23
[0] MPI startup(): 312     35801    cpu-e-1150  24
[0] MPI startup(): 313     35802    cpu-e-1150  25
[0] MPI startup(): 314     35803    cpu-e-1150  26
[0] MPI startup(): 315     35804    cpu-e-1150  27
[0] MPI startup(): 316     35805    cpu-e-1150  28
[0] MPI startup(): 317     35806    cpu-e-1150  29
[0] MPI startup(): 318     35807    cpu-e-1150  30
[0] MPI startup(): 319     35808    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
-- started at 11/08/2019 00:45:21 --

mdtest-3.3.0+dev was launched with 320 total task(s) on 10 node(s)
Command line used: /home/wjt27/io-500-sc19/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_hard' '-n' '950000' '-x' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /dac/fs1/mjr208/job16760824-2019-11-07-2352
FS: 405.5 TiB   Used FS: 26.5%   Inodes: 1820.8 Mi   Used Inodes: 10.5%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 32 for each phase.
320 tasks, 304000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      65400.242      65397.334      65397.606          0.190
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        144.481        144.474        144.480          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/08/2019 00:47:47 --

mdtest_hard_write
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
MPI startup(): Imported environment partly inaccesible. Map=0 Info=0
[0] MPI startup(): libfabric version: 1.8.0a1-impi

[0] MPI startup(): libfabric provider: psm2

[0] MPI startup(): Rank    Pid      Node name   Pin cpu
[0] MPI startup(): 0       35142    cpu-e-829   0
[0] MPI startup(): 1       35143    cpu-e-829   1
[0] MPI startup(): 2       35144    cpu-e-829   2
[0] MPI startup(): 3       35145    cpu-e-829   3
[0] MPI startup(): 4       35146    cpu-e-829   4
[0] MPI startup(): 5       35147    cpu-e-829   5
[0] MPI startup(): 6       35148    cpu-e-829   6
[0] MPI startup(): 7       35149    cpu-e-829   7
[0] MPI startup(): 8       35150    cpu-e-829   8
[0] MPI startup(): 9       35151    cpu-e-829   9
[0] MPI startup(): 10      35152    cpu-e-829   10
[0] MPI startup(): 11      35153    cpu-e-829   11
[0] MPI startup(): 12      35154    cpu-e-829   12
[0] MPI startup(): 13      35155    cpu-e-829   13
[0] MPI startup(): 14      35156    cpu-e-829   14
[0] MPI startup(): 15      35157    cpu-e-829   15
[0] MPI startup(): 16      35158    cpu-e-829   16
[0] MPI startup(): 17      35159    cpu-e-829   17
[0] MPI startup(): 18      35160    cpu-e-829   18
[0] MPI startup(): 19      35161    cpu-e-829   19
[0] MPI startup(): 20      35162    cpu-e-829   20
[0] MPI startup(): 21      35163    cpu-e-829   21
[0] MPI startup(): 22      35164    cpu-e-829   22
[0] MPI startup(): 23      35165    cpu-e-829   23
[0] MPI startup(): 24      35166    cpu-e-829   24
[0] MPI startup(): 25      35167    cpu-e-829   25
[0] MPI startup(): 26      35168    cpu-e-829   26
[0] MPI startup(): 27      35169    cpu-e-829   27
[0] MPI startup(): 28      35170    cpu-e-829   28
[0] MPI startup(): 29      35171    cpu-e-829   29
[0] MPI startup(): 30      35172    cpu-e-829   30
[0] MPI startup(): 31      35173    cpu-e-829   31
[0] MPI startup(): 32      34535    cpu-e-830   0
[0] MPI startup(): 33      34536    cpu-e-830   1
[0] MPI startup(): 34      34537    cpu-e-830   2
[0] MPI startup(): 35      34538    cpu-e-830   3
[0] MPI startup(): 36      34539    cpu-e-830   4
[0] MPI startup(): 37      34540    cpu-e-830   5
[0] MPI startup(): 38      34541    cpu-e-830   6
[0] MPI startup(): 39      34542    cpu-e-830   7
[0] MPI startup(): 40      34543    cpu-e-830   8
[0] MPI startup(): 41      34544    cpu-e-830   9
[0] MPI startup(): 42      34545    cpu-e-830   10
[0] MPI startup(): 43      34546    cpu-e-830   11
[0] MPI startup(): 44      34547    cpu-e-830   12
[0] MPI startup(): 45      34548    cpu-e-830   13
[0] MPI startup(): 46      34549    cpu-e-830   14
[0] MPI startup(): 47      34550    cpu-e-830   15
[0] MPI startup(): 48      34551    cpu-e-830   16
[0] MPI startup(): 49      34552    cpu-e-830   17
[0] MPI startup(): 50      34553    cpu-e-830   18
[0] MPI startup(): 51      34554    cpu-e-830   19
[0] MPI startup(): 52      34555    cpu-e-830   20
[0] MPI startup(): 53      34556    cpu-e-830   21
[0] MPI startup(): 54      34557    cpu-e-830   22
[0] MPI startup(): 55      34558    cpu-e-830   23
[0] MPI startup(): 56      34559    cpu-e-830   24
[0] MPI startup(): 57      34560    cpu-e-830   25
[0] MPI startup(): 58      34561    cpu-e-830   26
[0] MPI startup(): 59      34562    cpu-e-830   27
[0] MPI startup(): 60      34563    cpu-e-830   28
[0] MPI startup(): 61      34564    cpu-e-830   29
[0] MPI startup(): 62      34565    cpu-e-830   30
[0] MPI startup(): 63      34566    cpu-e-830   31
[0] MPI startup(): 64      34315    cpu-e-893   0
[0] MPI startup(): 65      34316    cpu-e-893   1
[0] MPI startup(): 66      34317    cpu-e-893   2
[0] MPI startup(): 67      34318    cpu-e-893   3
[0] MPI startup(): 68      34319    cpu-e-893   4
[0] MPI startup(): 69      34320    cpu-e-893   5
[0] MPI startup(): 70      34321    cpu-e-893   6
[0] MPI startup(): 71      34322    cpu-e-893   7
[0] MPI startup(): 72      34323    cpu-e-893   8
[0] MPI startup(): 73      34324    cpu-e-893   9
[0] MPI startup(): 74      34325    cpu-e-893   10
[0] MPI startup(): 75      34326    cpu-e-893   11
[0] MPI startup(): 76      34327    cpu-e-893   12
[0] MPI startup(): 77      34328    cpu-e-893   13
[0] MPI startup(): 78      34329    cpu-e-893   14
[0] MPI startup(): 79      34330    cpu-e-893   15
[0] MPI startup(): 80      34331    cpu-e-893   16
[0] MPI startup(): 81      34332    cpu-e-893   17
[0] MPI startup(): 82      34333    cpu-e-893   18
[0] MPI startup(): 83      34334    cpu-e-893   19
[0] MPI startup(): 84      34335    cpu-e-893   20
[0] MPI startup(): 85      34336    cpu-e-893   21
[0] MPI startup(): 86      34337    cpu-e-893   22
[0] MPI startup(): 87      34338    cpu-e-893   23
[0] MPI startup(): 88      34339    cpu-e-893   24
[0] MPI startup(): 89      34340    cpu-e-893   25
[0] MPI startup(): 90      34341    cpu-e-893   26
[0] MPI startup(): 91      34342    cpu-e-893   27
[0] MPI startup(): 92      34343    cpu-e-893   28
[0] MPI startup(): 93      34344    cpu-e-893   29
[0] MPI startup(): 94      34345    cpu-e-893   30
[0] MPI startup(): 95      34346    cpu-e-893   31
[0] MPI startup(): 96      34297    cpu-e-894   0
[0] MPI startup(): 97      34298    cpu-e-894   1
[0] MPI startup(): 98      34299    cpu-e-894   2
[0] MPI startup(): 99      34300    cpu-e-894   3
[0] MPI startup(): 100     34301    cpu-e-894   4
[0] MPI startup(): 101     34302    cpu-e-894   5
[0] MPI startup(): 102     34303    cpu-e-894   6
[0] MPI startup(): 103     34304    cpu-e-894   7
[0] MPI startup(): 104     34305    cpu-e-894   8
[0] MPI startup(): 105     34306    cpu-e-894   9
[0] MPI startup(): 106     34307    cpu-e-894   10
[0] MPI startup(): 107     34308    cpu-e-894   11
[0] MPI startup(): 108     34309    cpu-e-894   12
[0] MPI startup(): 109     34310    cpu-e-894   13
[0] MPI startup(): 110     34311    cpu-e-894   14
[0] MPI startup(): 111     34312    cpu-e-894   15
[0] MPI startup(): 112     34313    cpu-e-894   16
[0] MPI startup(): 113     34314    cpu-e-894   17
[0] MPI startup(): 114     34315    cpu-e-894   18
[0] MPI startup(): 115     34316    cpu-e-894   19
[0] MPI startup(): 116     34317    cpu-e-894   20
[0] MPI startup(): 117     34318    cpu-e-894   21
[0] MPI startup(): 118     34319    cpu-e-894   22
[0] MPI startup(): 119     34320    cpu-e-894   23
[0] MPI startup(): 120     34321    cpu-e-894   24
[0] MPI startup(): 121     34322    cpu-e-894   25
[0] MPI startup(): 122     34323    cpu-e-894   26
[0] MPI startup(): 123     34324    cpu-e-894   27
[0] MPI startup(): 124     34325    cpu-e-894   28
[0] MPI startup(): 125     34326    cpu-e-894   29
[0] MPI startup(): 126     34327    cpu-e-894   30
[0] MPI startup(): 127     34328    cpu-e-894   31
[0] MPI startup(): 128     34434    cpu-e-957   0
[0] MPI startup(): 129     34435    cpu-e-957   1
[0] MPI startup(): 130     34436    cpu-e-957   2
[0] MPI startup(): 131     34437    cpu-e-957   3
[0] MPI startup(): 132     34438    cpu-e-957   4
[0] MPI startup(): 133     34439    cpu-e-957   5
[0] MPI startup(): 134     34440    cpu-e-957   6
[0] MPI startup(): 135     34441    cpu-e-957   7
[0] MPI startup(): 136     34442    cpu-e-957   8
[0] MPI startup(): 137     34443    cpu-e-957   9
[0] MPI startup(): 138     34444    cpu-e-957   10
[0] MPI startup(): 139     34445    cpu-e-957   11
[0] MPI startup(): 140     34446    cpu-e-957   12
[0] MPI startup(): 141     34447    cpu-e-957   13
[0] MPI startup(): 142     34448    cpu-e-957   14
[0] MPI startup(): 143     34449    cpu-e-957   15
[0] MPI startup(): 144     34450    cpu-e-957   16
[0] MPI startup(): 145     34451    cpu-e-957   17
[0] MPI startup(): 146     34452    cpu-e-957   18
[0] MPI startup(): 147     34453    cpu-e-957   19
[0] MPI startup(): 148     34454    cpu-e-957   20
[0] MPI startup(): 149     34455    cpu-e-957   21
[0] MPI startup(): 150     34456    cpu-e-957   22
[0] MPI startup(): 151     34457    cpu-e-957   23
[0] MPI startup(): 152     34458    cpu-e-957   24
[0] MPI startup(): 153     34459    cpu-e-957   25
[0] MPI startup(): 154     34460    cpu-e-957   26
[0] MPI startup(): 155     34461    cpu-e-957   27
[0] MPI startup(): 156     34462    cpu-e-957   28
[0] MPI startup(): 157     34463    cpu-e-957   29
[0] MPI startup(): 158     34464    cpu-e-957   30
[0] MPI startup(): 159     34465    cpu-e-957   31
[0] MPI startup(): 160     34385    cpu-e-958   0
[0] MPI startup(): 161     34386    cpu-e-958   1
[0] MPI startup(): 162     34387    cpu-e-958   2
[0] MPI startup(): 163     34388    cpu-e-958   3
[0] MPI startup(): 164     34389    cpu-e-958   4
[0] MPI startup(): 165     34390    cpu-e-958   5
[0] MPI startup(): 166     34391    cpu-e-958   6
[0] MPI startup(): 167     34392    cpu-e-958   7
[0] MPI startup(): 168     34393    cpu-e-958   8
[0] MPI startup(): 169     34394    cpu-e-958   9
[0] MPI startup(): 170     34395    cpu-e-958   10
[0] MPI startup(): 171     34396    cpu-e-958   11
[0] MPI startup(): 172     34397    cpu-e-958   12
[0] MPI startup(): 173     34398    cpu-e-958   13
[0] MPI startup(): 174     34399    cpu-e-958   14
[0] MPI startup(): 175     34400    cpu-e-958   15
[0] MPI startup(): 176     34401    cpu-e-958   16
[0] MPI startup(): 177     34402    cpu-e-958   17
[0] MPI startup(): 178     34403    cpu-e-958   18
[0] MPI startup(): 179     34404    cpu-e-958   19
[0] MPI startup(): 180     34405    cpu-e-958   20
[0] MPI startup(): 181     34406    cpu-e-958   21
[0] MPI startup(): 182     34407    cpu-e-958   22
[0] MPI startup(): 183     34408    cpu-e-958   23
[0] MPI startup(): 184     34409    cpu-e-958   24
[0] MPI startup(): 185     34410    cpu-e-958   25
[0] MPI startup(): 186     34411    cpu-e-958   26
[0] MPI startup(): 187     34412    cpu-e-958   27
[0] MPI startup(): 188     34413    cpu-e-958   28
[0] MPI startup(): 189     34414    cpu-e-958   29
[0] MPI startup(): 190     34415    cpu-e-958   30
[0] MPI startup(): 191     34416    cpu-e-958   31
[0] MPI startup(): 192     34629    cpu-e-1021  0
[0] MPI startup(): 193     34630    cpu-e-1021  1
[0] MPI startup(): 194     34631    cpu-e-1021  2
[0] MPI startup(): 195     34632    cpu-e-1021  3
[0] MPI startup(): 196     34633    cpu-e-1021  4
[0] MPI startup(): 197     34634    cpu-e-1021  5
[0] MPI startup(): 198     34635    cpu-e-1021  6
[0] MPI startup(): 199     34636    cpu-e-1021  7
[0] MPI startup(): 200     34637    cpu-e-1021  8
[0] MPI startup(): 201     34638    cpu-e-1021  9
[0] MPI startup(): 202     34639    cpu-e-1021  10
[0] MPI startup(): 203     34640    cpu-e-1021  11
[0] MPI startup(): 204     34641    cpu-e-1021  12
[0] MPI startup(): 205     34642    cpu-e-1021  13
[0] MPI startup(): 206     34643    cpu-e-1021  14
[0] MPI startup(): 207     34644    cpu-e-1021  15
[0] MPI startup(): 208     34645    cpu-e-1021  16
[0] MPI startup(): 209     34646    cpu-e-1021  17
[0] MPI startup(): 210     34647    cpu-e-1021  18
[0] MPI startup(): 211     34648    cpu-e-1021  19
[0] MPI startup(): 212     34649    cpu-e-1021  20
[0] MPI startup(): 213     34650    cpu-e-1021  21
[0] MPI startup(): 214     34651    cpu-e-1021  22
[0] MPI startup(): 215     34652    cpu-e-1021  23
[0] MPI startup(): 216     34653    cpu-e-1021  24
[0] MPI startup(): 217     34654    cpu-e-1021  25
[0] MPI startup(): 218     34655    cpu-e-1021  26
[0] MPI startup(): 219     34656    cpu-e-1021  27
[0] MPI startup(): 220     34657    cpu-e-1021  28
[0] MPI startup(): 221     34658    cpu-e-1021  29
[0] MPI startup(): 222     34659    cpu-e-1021  30
[0] MPI startup(): 223     34660    cpu-e-1021  31
[0] MPI startup(): 224     34392    cpu-e-1022  0
[0] MPI startup(): 225     34393    cpu-e-1022  1
[0] MPI startup(): 226     34394    cpu-e-1022  2
[0] MPI startup(): 227     34395    cpu-e-1022  3
[0] MPI startup(): 228     34396    cpu-e-1022  4
[0] MPI startup(): 229     34397    cpu-e-1022  5
[0] MPI startup(): 230     34398    cpu-e-1022  6
[0] MPI startup(): 231     34399    cpu-e-1022  7
[0] MPI startup(): 232     34400    cpu-e-1022  8
[0] MPI startup(): 233     34401    cpu-e-1022  9
[0] MPI startup(): 234     34402    cpu-e-1022  10
[0] MPI startup(): 235     34403    cpu-e-1022  11
[0] MPI startup(): 236     34404    cpu-e-1022  12
[0] MPI startup(): 237     34405    cpu-e-1022  13
[0] MPI startup(): 238     34406    cpu-e-1022  14
[0] MPI startup(): 239     34407    cpu-e-1022  15
[0] MPI startup(): 240     34408    cpu-e-1022  16
[0] MPI startup(): 241     34409    cpu-e-1022  17
[0] MPI startup(): 242     34410    cpu-e-1022  18
[0] MPI startup(): 243     34411    cpu-e-1022  19
[0] MPI startup(): 244     34412    cpu-e-1022  20
[0] MPI startup(): 245     34413    cpu-e-1022  21
[0] MPI startup(): 246     34414    cpu-e-1022  22
[0] MPI startup(): 247     34415    cpu-e-1022  23
[0] MPI startup(): 248     34416    cpu-e-1022  24
[0] MPI startup(): 249     34417    cpu-e-1022  25
[0] MPI startup(): 250     34418    cpu-e-1022  26
[0] MPI startup(): 251     34419    cpu-e-1022  27
[0] MPI startup(): 252     34420    cpu-e-1022  28
[0] MPI startup(): 253     34421    cpu-e-1022  29
[0] MPI startup(): 254     34422    cpu-e-1022  30
[0] MPI startup(): 255     34423    cpu-e-1022  31
[0] MPI startup(): 256     34418    cpu-e-1149  0
[0] MPI startup(): 257     34419    cpu-e-1149  1
[0] MPI startup(): 258     34420    cpu-e-1149  2
[0] MPI startup(): 259     34421    cpu-e-1149  3
[0] MPI startup(): 260     34422    cpu-e-1149  4
[0] MPI startup(): 261     34423    cpu-e-1149  5
[0] MPI startup(): 262     34424    cpu-e-1149  6
[0] MPI startup(): 263     34425    cpu-e-1149  7
[0] MPI startup(): 264     34426    cpu-e-1149  8
[0] MPI startup(): 265     34427    cpu-e-1149  9
[0] MPI startup(): 266     34428    cpu-e-1149  10
[0] MPI startup(): 267     34429    cpu-e-1149  11
[0] MPI startup(): 268     34430    cpu-e-1149  12
[0] MPI startup(): 269     34431    cpu-e-1149  13
[0] MPI startup(): 270     34432    cpu-e-1149  14
[0] MPI startup(): 271     34433    cpu-e-1149  15
[0] MPI startup(): 272     34434    cpu-e-1149  16
[0] MPI startup(): 273     34435    cpu-e-1149  17
[0] MPI startup(): 274     34436    cpu-e-1149  18
[0] MPI startup(): 275     34437    cpu-e-1149  19
[0] MPI startup(): 276     34438    cpu-e-1149  20
[0] MPI startup(): 277     34439    cpu-e-1149  21
[0] MPI startup(): 278     34440    cpu-e-1149  22
[0] MPI startup(): 279     34441    cpu-e-1149  23
[0] MPI startup(): 280     34442    cpu-e-1149  24
[0] MPI startup(): 281     34443    cpu-e-1149  25
[0] MPI startup(): 282     34444    cpu-e-1149  26
[0] MPI startup(): 283     34445    cpu-e-1149  27
[0] MPI startup(): 284     34446    cpu-e-1149  28
[0] MPI startup(): 285     34447    cpu-e-1149  29
[0] MPI startup(): 286     34448    cpu-e-1149  30
[0] MPI startup(): 287     34449    cpu-e-1149  31
[0] MPI startup(): 288     34450    cpu-e-1150  0
[0] MPI startup(): 289     34451    cpu-e-1150  1
[0] MPI startup(): 290     34452    cpu-e-1150  2
[0] MPI startup(): 291     34453    cpu-e-1150  3
[0] MPI startup(): 292     34454    cpu-e-1150  4
[0] MPI startup(): 293     34455    cpu-e-1150  5
[0] MPI startup(): 294     34456    cpu-e-1150  6
[0] MPI startup(): 295     34457    cpu-e-1150  7
[0] MPI startup(): 296     34458    cpu-e-1150  8
[0] MPI startup(): 297     34459    cpu-e-1150  9
[0] MPI startup(): 298     34460    cpu-e-1150  10
[0] MPI startup(): 299     34461    cpu-e-1150  11
[0] MPI startup(): 300     34462    cpu-e-1150  12
[0] MPI startup(): 301     34463    cpu-e-1150  13
[0] MPI startup(): 302     34464    cpu-e-1150  14
[0] MPI startup(): 303     34465    cpu-e-1150  15
[0] MPI startup(): 304     34466    cpu-e-1150  16
[0] MPI startup(): 305     34467    cpu-e-1150  17
[0] MPI startup(): 306     34468    cpu-e-1150  18
[0] MPI startup(): 307     34469    cpu-e-1150  19
[0] MPI startup(): 308     34470    cpu-e-1150  20
[0] MPI startup(): 309     34471    cpu-e-1150  21
[0] MPI startup(): 310     34472    cpu-e-1150  22
[0] MPI startup(): 311     34473    cpu-e-1150  23
[0] MPI startup(): 312     34474    cpu-e-1150  24
[0] MPI startup(): 313     34475    cpu-e-1150  25
[0] MPI startup(): 314     34476    cpu-e-1150  26
[0] MPI startup(): 315     34477    cpu-e-1150  27
[0] MPI startup(): 316     34478    cpu-e-1150  28
[0] MPI startup(): 317     34479    cpu-e-1150  29
[0] MPI startup(): 318     34480    cpu-e-1150  30
[0] MPI startup(): 319     34481    cpu-e-1150  31
[0] MPI startup(): I_MPI_CC=icc
[0] MPI startup(): I_MPI_CXX=icpc
[0] MPI startup(): I_MPI_FC=ifort
[0] MPI startup(): I_MPI_F90=ifort
[0] MPI startup(): I_MPI_F77=ifort
[0] MPI startup(): I_MPI_ROOT=/usr/local/Cluster-Apps/intel/2019.3//compilers_and_libraries_2019.3.192/linux/mpi
[0] MPI startup(): I_MPI_HYDRA_RMK=slurm
[0] MPI startup(): I_MPI_JOB_RESPECT_PROCESS_PLACEMENT=off
[0] MPI startup(): I_MPI_PIN_DOMAIN=omp:compact
[0] MPI startup(): I_MPI_PIN_ORDER=scatter
[0] MPI startup(): I_MPI_FABRICS=shm:ofi
[0] MPI startup(): I_MPI_DEBUG=10
[0] MPI startup(): I_MPI_TMI_PROVIDER variable has been removed from the product, its value is ignored

[0] MPI startup(): I_MPI_TMI_PROVIDER environment variable is not supported.
[0] MPI startup(): Similar variables:
	 I_MPI_OFI_PROVIDER
[0] MPI startup(): To check the list of supported variables, use the impi_info utility or refer to https://software.intel.com/en-us/mpi-library/documentation/get-started.
-- started at 11/08/2019 00:19:34 --

mdtest-3.3.0+dev was launched with 320 total task(s) on 10 node(s)
Command line used: /home/wjt27/io-500-sc19/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_hard' '-n' '950000' '-x' '/dac/fs1/mjr208/job16760824-2019-11-07-2352/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /dac/fs1/mjr208/job16760824-2019-11-07-2352
FS: 405.5 TiB   Used FS: 26.5%   Inodes: 1820.8 Mi   Used Inodes: 10.0%

Nodemap: 11111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2137 Shifting ranks by 32 for each phase.
320 tasks, 304000000 files
Continue stonewall hit min: 26999 max: 29527 avg: 28479.7 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      29808.137      29804.652      29807.843          0.716
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      30393.493             NA
   Tree creation             :          6.396          6.396          6.396          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        317.019        316.982        316.985          0.008
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.850             NA
   Tree creation             :          0.156          0.156          0.156          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/08/2019 00:24:52 --

result_summary
[RESULT] BW   phase 1            ior_easy_write              142.401 GB/s : time 348.24 seconds
[RESULT] IOPS phase 1         mdtest_easy_write              435.701 kiops : time 437.96 seconds
[RESULT] BW   phase 2            ior_hard_write                3.182 GB/s : time 800.04 seconds
[RESULT] IOPS phase 2         mdtest_hard_write               29.808 kiops : time 317.02 seconds
[RESULT] IOPS phase 3                      find             7348.560 kiops : time  27.33 seconds
[RESULT] BW   phase 3             ior_easy_read              106.335 GB/s : time 466.38 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             1184.270 kiops : time 161.13 seconds
[RESULT] BW   phase 4             ior_hard_read                4.624 GB/s : time 550.00 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               65.400 kiops : time 144.48 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete              129.618 kiops : time 1472.21 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               59.008 kiops : time 160.13 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete               10.745 kiops : time 883.07 seconds
[SCORE] Bandwidth 21.7259 GB/s : IOPS 167.087 kiops : TOTAL 60.2504