SharedScratch

Institution Janelia Research Campus, HHMI
Client Procs Per Node
Client Operating System Scientific-Linux
Client Operating System Version 7.7
Client Kernel Version 3.10.0-957.10.1.el7.x86_64

DATA SERVER

Storage Type Flash
Volatile Memory 96GB
Storage Interface NVMe
Network 100GbE
Software Version 3.4.6
OS Version 7.7

INFORMATION

Client Nodes 10
Client Total Procs 400

METADATA

Easy Write 22.12 kIOP/s
Easy Stat 1,594.65 kIOP/s
Easy Delete 21.55 kIOP/s
Hard Write 3.89 kIOP/s
Hard Read 28.65 kIOP/s
Hard Stat 11.68 kIOP/s
Hard Delete 3.63 kIOP/s

Submitted Files

io500
#!/bin/bash
#
# INSTRUCTIONS:
# Edit this file as needed for your machine.
# This simplified version is just for running on a single node.
# It is a simplified version of the site-configs/sandia/startup.sh which include SLURM directives.
# Most of the variables set in here are needed for io500_fixed.sh which gets sourced at the end of this.
# Please also edit 'extra_description' function.

set -euo pipefail  # better error handling

# turn these to True successively while you debug and tune this benchmark.
# for each one that you turn to true, go and edit the appropriate function.
# to find the function name, see the 'main' function.
# These are listed in the order that they run.
io500_run_ior_easy="True" # does the write phase and enables the subsequent read
io500_run_md_easy="True"  # does the creat phase and enables the subsequent stat
io500_run_ior_hard="True" # does the write phase and enables the subsequent read
io500_run_md_hard="True"  # does the creat phase and enables the subsequent read
io500_run_find="True"
io500_run_ior_easy_read="True"
io500_run_md_easy_stat="True"
io500_run_ior_hard_read="True"
io500_run_md_hard_stat="True"
io500_run_md_hard_read="True"
io500_run_md_easy_delete="True" # turn this off if you want to just run find by itself
io500_run_md_hard_delete="True" # turn this off if you want to just run find by itself
io500_run_mdreal="False"  # this one is optional
io500_cleanup_workdir="False"  # this flag is currently ignored. You'll need to clean up your data files manually if you want to.
io500_clean_cache="False" # attempt to clean the cache after every benchmark, useful for validating the performance results and for testing with a local node; it uses the io500_clean_cache_cmd (can be overwritten); make sure the user can write to /proc/sys/vm/drop_caches
io500_stonewall_timer=300 # Stonewalling timer, set to 300 to be an official run; set to 0, if you never want to abort...
io500_rules="regular" # Choose regular for an official regular submission or scc for a Student Cluster Competition submission to execute the test cases for 30 seconds instead of 300 seconds

# to run this benchmark, find and edit each of these functions.
# please also edit 'extra_description' function to help us collect the required data.
function main {
  setup_directories
  setup_paths
  setup_ior_easy # required if you want a complete score
  setup_ior_hard # required if you want a complete score
  setup_mdt_easy # required if you want a complete score
  setup_mdt_hard # required if you want a complete score
  setup_find     # required if you want a complete score
  setup_mdreal   # optional
  run_benchmarks
}

function setup_directories {
  # set directories for where the benchmark files are created and where the results will go.
  # If you want to set up stripe tuning on your output directories or anything similar, then this is good place to do it.
  timestamp=`date +%Y.%m.%d-%H.%M.%S`           # create a uniquifier
  io500_workdir=$PWD/datafiles/io500.$timestamp # directory where the data will be stored
  io500_result_dir=$PWD/results/$timestamp      # the directory where the output results will be kept
  mkdir -p $io500_workdir $io500_result_dir
  export MPICH_MPIIO_HINTS="*:romio_cb_write=enable:romio_ds_write=disable:romio_cb_read=enable:romio_ds_read=disable"
}

function setup_paths {
  # Set the paths to the binaries.  If you ran ./utilities/prepare.sh successfully, then binaries are in ./bin/
  io500_ior_cmd=$PWD/bin/ior
  io500_mdtest_cmd=$PWD/bin/mdtest
  io500_mdreal_cmd=$PWD/bin/md-real-io
  io500_mpirun="mpirun"
  io500_mpiargs="--nolocal -np 760 -ppn 76 -f ./site-configs/hhmi/hostfile-10gpu.4 --bootstrap ssh"
}

function setup_ior_easy {
  io500_ior_easy_params="-t 32768k -b 9920000m -F"
  echo -n ""
}

function setup_mdt_easy {
  io500_mdtest_easy_params="-u -L" # unique dir per thread, files only at leaves
}

function setup_ior_hard {
  io500_ior_hard_api="POSIX"
  io500_ior_hard_api_specific_options="--posix.odirect"
}

function setup_mdt_hard {
  io500_mdtest_hard_api="POSIX"
  io500_mdtest_hard_api_specific_options=""
}

function setup_find {
  #
  # setup the find command. This is an area where innovation is allowed.
  #    There are three default options provided. One is a serial find, one is python
  #    parallel version, one is C parallel version.  Current default is to use serial.
  #    But it is very slow. We recommend to either customize or use the C parallel version.
  #    For GPFS, we recommend to use the provided mmfind wrapper described below.
  #    Instructions below.
  #    If a custom approach is used, please provide enough info so others can reproduce.

  # the serial version that should run (SLOWLY) without modification
  #io500_find_mpi="False"
  #io500_find_cmd=$PWD/bin/sfind.sh
  #io500_find_cmd_args=""

  # a parallel version in C, the -s adds a stonewall
  #   for a real run, turn -s (stonewall) off or set it at 300 or more
  #   to prepare this (assuming you've run ./utilities/prepare.sh already):
  #   > cd build/pfind
  #   > ./prepare.sh
  #   > ./compile.sh
  #   > cp pfind ../../bin/
  #   If you use io500_find_mpi="True", then this will run with the same
  #   number of MPI nodes and ranks as the other phases.
  #   If you prefer another number, and fewer might be better here,
  #   Then you can set io500_find_mpi to be "False" and write a wrapper
  #   script for this which sets up MPI as you would like.  Then change
  #   io500_find_cmd to point to your wrapper script.
  io500_find_mpi="True"
  io500_find_cmd="$PWD/bin/pfind"
  # uses stonewalling, run pfind
  io500_find_cmd_args="-N"

  # for GPFS systems, you should probably use the provided mmfind wrapper
  # if you used ./utilities/prepare.sh, you'll find this wrapper in ./bin/mmfind.sh
  #io500_find_mpi="False"
  #io500_find_cmd="$PWD/bin/mmfind.sh"
  #io500_find_cmd_args=""
}

function setup_mdreal {
  echo -n ""
}

function run_benchmarks {
  # Important: source the io500_fixed.sh script.  Do not change it. If you discover
  # a need to change it, please email the mailing list to discuss
  source ./utilities/io500_fixed.sh 2>&1 | tee $io500_result_dir/io-500-summary.$timestamp.txt
}

# Information fields; these provide information about your system hardware
# Use https://vi4io.org/io500-info-creator/ to generate information about your hardware
# that you want to include publicly!
function extra_description {
  # TODO: Please add your information using the info-creator!
  # EXAMPLE:
  echo io500_info_submitter='"Janelia Scientific Computing Systems"'
  echo io500_info_10nodechallenge='"true"'
  echo io500_info_institution='"Janelia Research Campus, Howard Hughes Medical Institute"'
  echo io500_info_system='"weka"'
  echo io500_info_storage_install_date='"2/2019"'
  echo io500_info_storage_vendor='"WekaIO"'
  echo io500_info_filesystem_name='"wekafs"'
  echo io500_info_filesystem_type='"wekaio"'
  echo io500_info_filesystem_version='"3.4.6"'
  echo io500_info_client_nodes='"10"'
  echo io500_info_client_procs_per_node='"40"'
  echo io500_info_client_operating_system='"Scientific-Linux"'
  echo io500_info_client_operating_system_version='"7.7"'
  echo io500_info_client_kernel_version='"3.10.0-957.10.1.el7.x86_64"'
  echo io500_info_ds_nodes='"20"'
  echo io500_info_ds_storage_devices='"4"'
  echo io500_info_ds_storage_type='"NVMe-SSD"'
  echo io500_info_ds_volatile_memory_capacity='"96GB"'
  echo io500_info_ds_storage_interface='"NVMe"'
  echo io500_info_ds_network='"Ethernet"'
  echo io500_info_ds_software_version='"3.4.6"'

}

main
ior_easy_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov  3 21:40:14 2019
Command line        : /io500/io500-sc19/bin/ior -r -R -t 32768k -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_easy/ior_file_easy -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_easy/stonewall
Machine             : Linux e11u05.int.janelia.org
TestID              : 0
StartTime           : Sun Nov  3 21:40:14 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_easy
FS                  : 100.0 TiB   Used FS: 41.4%   Inodes: 858.3 Mi   Used Inodes: 24.7%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 760
clients per node    : 76
repetitions         : 1
xfersize            : 32 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 7189.94 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
WARNING: Task 38, partial read(), 25165824 of 33554432 bytes at offset 0
WARNING: Task 36, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 25, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 67, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 40, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 45, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 63, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 50, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 64, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 13, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 74, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 73, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 65, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 69, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 46, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 44, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 51, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 23, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 42, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 14, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 8, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 17, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 16, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 2, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 27, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 47, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 49, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 52, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 19, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 55, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 62, partial read(), 25165824 of 33554432 bytes at offset 0
WARNING: Task 71, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 43, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 58, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 21, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 66, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 37, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 10, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 15, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 48, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 39, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 7, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 68, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 56, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 32, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 59, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 3, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 53, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 75, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 6, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 54, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 33, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 70, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 5, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 57, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 61, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 30, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 60, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 41, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 18, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 0, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 22, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 12, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 28, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 35, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 9, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 20, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 34, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 24, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 29, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 31, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 26, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 11, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 1, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 4, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 664, partial read(), 18874368 of 33554432 bytes at offset 0
WARNING: Task 354, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 192, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 206, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 750, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 258, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 667, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 234, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 452, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 182, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 613, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 653, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 410, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 636, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 669, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 733, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 340, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 753, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 695, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 745, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 577, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 125, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 604, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 564, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 406, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 478, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 262, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 126, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 557, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 551, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 597, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 301, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 601, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 331, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 742, partial read(), 25165824 of 33554432 bytes at offset 0
WARNING: Task 287, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 471, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 599, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 370, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 464, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 321, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 554, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 527, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 397, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 166, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 222, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 414, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 646, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 714, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 436, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 191, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 110, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 544, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 245, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 713, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 401, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 418, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 493, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 146, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 256, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 217, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 476, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 362, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 610, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 351, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 399, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 178, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 576, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 254, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 698, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 134, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 153, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 673, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 588, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 293, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 685, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 103, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 183, partial read(), 18874368 of 33554432 bytes at offset 0
WARNING: Task 598, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 355, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 295, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 482, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 429, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 718, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 106, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 198, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 620, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 552, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 353, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 277, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 529, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 381, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 711, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 94, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 209, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 642, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 587, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 338, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 491, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 428, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 734, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 98, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 154, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 618, partial read(), 23068672 of 33554432 bytes at offset 0
WARNING: Task 536, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 310, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 479, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 419, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 739, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 81, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 220, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 655, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 602, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 315, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 511, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 395, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 720, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 143, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 186, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 659, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 605, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 333, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 505, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 388, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 700, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 93, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 210, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 619, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 583, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 325, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 496, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 731, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 88, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 221, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 661, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 533, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 378, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 523, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 725, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 89, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 202, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 634, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 581, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 469, partial read(), 25165824 of 33554432 bytes at offset 0
WARNING: Task 408, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 692, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 76, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 226, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 623, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 538, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 501, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 453, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 716, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 161, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 677, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 547, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 494, partial read(), 25165824 of 33554432 bytes at offset 0
WARNING: Task 693, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 195, partial read(), 22020096 of 33554432 bytes at offset 0
WARNING: Task 648, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 594, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 490, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 684, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 678, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 590, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 458, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 741, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 617, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 542, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 530, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 736, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 647, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 571, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 456, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 689, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 627, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 562, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 525, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 697, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 631, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 555, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 492, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 729, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 615, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 584, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 465, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 757, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 654, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 560, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 457, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 735, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 651, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 559, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 481, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 690, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 656, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 553, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 460, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 687, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 621, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 549, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 466, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 738, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 629, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 603, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 463, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 744, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 662, partial read(), 24117248 of 33554432 bytes at offset 0
WARNING: Task 543, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 747, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 660, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 719, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 624, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 703, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 608, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 688, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 730, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 737, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 461, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 676, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 705, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 515, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 722, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 561, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 477, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 649, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 521, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 524, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 595, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 480, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 563, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 709, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 580, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 586, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 510, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 759, partial read(), 18874368 of 33554432 bytes at offset 0
WARNING: Task 702, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 486, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 528, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 694, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 565, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 751, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 484, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 625, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 732, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 545, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 686, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 639, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 550, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 558, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 670, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 645, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 462, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 638, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 579, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 591, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 520, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 671, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 228, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 500, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 699, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 665, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 267, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 487, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 541, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 282, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 459, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 263, partial read(), 14680064 of 33554432 bytes at offset 0
WARNING: Task 261, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 291, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 231, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 758, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 274, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 748, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 275, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 740, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 658, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 260, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 252, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 566, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 247, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 257, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 250, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 243, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 290, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 607, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 244, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 265, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 229, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 233, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 276, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 284, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 264, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 286, partial read(), 25165824 of 33554432 bytes at offset 0
WARNING: Task 652, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 268, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 269, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 246, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 546, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 298, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 242, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 508, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 272, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 230, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 475, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 302, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 756, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 539, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 297, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 704, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 160, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 249, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 426, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 727, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 171, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 292, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 445, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 193, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 294, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 440, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 168, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 682, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 303, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 451, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 708, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 172, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 283, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 396, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 691, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 159, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 278, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 502, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 403, partial read(), 22020096 of 33554432 bytes at offset 0
WARNING: Task 204, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 241, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 443, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 170, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 253, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 433, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 715, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 196, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 448, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 723, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 155, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 424, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 728, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 199, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 404, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 190, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 384, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 157, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 400, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 724, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 213, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 675, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 483, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 439, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 717, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 216, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 674, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 570, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 517, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 434, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 212, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 438, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 179, partial read(), 18874368 of 33554432 bytes at offset 0
WARNING: Task 398, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 174, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 514, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 394, partial read(), 23068672 of 33554432 bytes at offset 0
WARNING: Task 189, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 681, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 503, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 423, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 712, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 194, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 472, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 402, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 207, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 672, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 442, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 211, partial read(), 18874368 of 33554432 bytes at offset 0
WARNING: Task 432, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 227, partial read(), 19922944 of 33554432 bytes at offset 0
WARNING: Task 385, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 185, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 416, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 175, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 447, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 215, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 431, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 203, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 389, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 218, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 444, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 184, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 409, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 200, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 593, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 430, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 167, partial read(), 17825792 of 33554432 bytes at offset 0
WARNING: Task 512, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 417, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 188, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 411, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 173, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 387, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 169, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 518, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 446, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 197, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 435, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 205, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 532, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 393, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 158, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 383, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 208, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 380, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 707, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 201, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 644, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 422, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 224, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 592, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 390, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 181, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 407, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 176, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 420, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 156, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 413, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 180, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 449, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 392, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 391, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 441, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 498, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 749, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 721, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 513, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 657, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 437, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 710, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 412, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 497, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 405, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 504, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 421, partial read(), 16777216 of 33554432 bytes at offset 0
WARNING: Task 495, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 666, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 485, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 499, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 650, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 506, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 640, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 280, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 248, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 575, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 270, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 259, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 425, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 240, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 255, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 235, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 239, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 251, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 289, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 300, partial read(), 25165824 of 33554432 bytes at offset 0
WARNING: Task 473, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 236, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 467, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 232, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 537, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 572, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 266, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 540, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 279, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 296, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 273, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 568, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 754, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 450, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 299, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 382, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 569, partial read(), 32505856 of 33554432 bytes at offset 0
WARNING: Task 683, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 548, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 668, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 162, partial read(), 18874368 of 33554432 bytes at offset 0
WARNING: Task 177, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 187, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 219, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 614, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 589, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 680, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 643, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 609, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 746, partial read(), 25165824 of 33554432 bytes at offset 0
WARNING: Task 637, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 574, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 641, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 238, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 163, partial read(), 17825792 of 33554432 bytes at offset 0
WARNING: Task 626, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 622, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 600, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 535, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 632, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 386, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 635, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 628, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 606, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 164, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 288, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 616, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 77, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 630, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 573, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 271, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 427, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 726, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 109, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 165, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 611, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 454, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 743, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 97, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 578, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 415, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 130, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 582, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 455, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 150, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 116, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 112, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 104, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 120, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 132, partial read(), 22020096 of 33554432 bytes at offset 0
WARNING: Task 114, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 82, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 99, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 128, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 79, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 78, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 108, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 96, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 84, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 225, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 95, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 87, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 137, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 83, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 149, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 701, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 118, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 144, partial read(), 24117248 of 33554432 bytes at offset 0
WARNING: Task 119, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 115, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 107, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 140, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 101, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 127, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 105, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 585, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 141, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 121, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 113, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 90, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 755, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 123, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 131, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 85, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 148, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 145, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 129, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 136, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 80, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 135, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 100, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 102, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 111, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 124, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 117, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 122, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 86, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 91, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 214, partial read(), 22020096 of 33554432 bytes at offset 0
WARNING: Task 139, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 534, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 92, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 151, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 138, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 147, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 706, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 752, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 556, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 468, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 596, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 633, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 152, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 612, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 679, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 663, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 470, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 519, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 696, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 507, partial read(), 26214400 of 33554432 bytes at offset 0
WARNING: Task 516, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 474, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 509, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 285, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 237, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 281, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 379, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 317, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 359, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 367, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 323, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 358, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 327, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 368, partial read(), 31457280 of 33554432 bytes at offset 0
WARNING: Task 352, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 371, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 350, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 372, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 369, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 312, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 349, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 319, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 336, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 361, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 304, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 306, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 363, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 346, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 334, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 373, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 360, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 356, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 329, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 374, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 366, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 375, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 308, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 365, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 376, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 347, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 342, partial read(), 18874368 of 33554432 bytes at offset 0
WARNING: Task 345, partial read(), 29360128 of 33554432 bytes at offset 0
WARNING: Task 344, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 364, partial read(), 30408704 of 33554432 bytes at offset 0
WARNING: Task 357, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 377, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 348, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 343, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 311, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 322, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 320, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 341, partial read(), 28311552 of 33554432 bytes at offset 0
WARNING: Task 314, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 330, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 313, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 332, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 337, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 318, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 326, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 309, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 305, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 307, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 339, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 316, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 335, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 324, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 328, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 133, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 142, partial read(), 27262976 of 33554432 bytes at offset 0
WARNING: Task 290, partial read(), 29360128 of 33554432 bytes at offset 6241124352
WARNING: Task 303, partial read(), 28311552 of 33554432 bytes at offset 7012876288
WARNING: Task 288, partial read(), 29360128 of 33554432 bytes at offset 6308233216
WARNING: Task 230, partial read(), 28311552 of 33554432 bytes at offset 6744440832
WARNING: Task 248, partial read(), 28311552 of 33554432 bytes at offset 7046430720
WARNING: Task 255, partial read(), 27262976 of 33554432 bytes at offset 7616856064
WARNING: Task 243, partial read(), 27262976 of 33554432 bytes at offset 5637144576
WARNING: Task 283, partial read(), 28311552 of 33554432 bytes at offset 6912212992
WARNING: Task 299, partial read(), 28311552 of 33554432 bytes at offset 5200936960
WARNING: Task 301, partial read(), 28311552 of 33554432 bytes at offset 6878658560
WARNING: Task 266, partial read(), 28311552 of 33554432 bytes at offset 6174015488
WARNING: Task 297, partial read(), 28311552 of 33554432 bytes at offset 6878658560
WARNING: Task 237, partial read(), 27262976 of 33554432 bytes at offset 6039797760
WARNING: Task 232, partial read(), 28311552 of 33554432 bytes at offset 5972688896
WARNING: Task 261, partial read(), 28311552 of 33554432 bytes at offset 5670699008
WARNING: Task 259, partial read(), 27262976 of 33554432 bytes at offset 5637144576
WARNING: Task 281, partial read(), 28311552 of 33554432 bytes at offset 6073352192
WARNING: Task 246, partial read(), 26214400 of 33554432 bytes at offset 6308233216
WARNING: Task 239, partial read(), 28311552 of 33554432 bytes at offset 6845104128
WARNING: Task 272, partial read(), 29360128 of 33554432 bytes at offset 6073352192
WARNING: Task 264, partial read(), 29360128 of 33554432 bytes at offset 5200936960
WARNING: Task 292, partial read(), 28311552 of 33554432 bytes at offset 7046430720
WARNING: Task 270, partial read(), 28311552 of 33554432 bytes at offset 6408896512
WARNING: Task 277, partial read(), 28311552 of 33554432 bytes at offset 7482638336
WARNING: Task 284, partial read(), 28311552 of 33554432 bytes at offset 7214202880
WARNING: Task 228, partial read(), 28311552 of 33554432 bytes at offset 7650410496
WARNING: Task 263, partial read(), 28311552 of 33554432 bytes at offset 7314866176
WARNING: Task 295, partial read(), 28311552 of 33554432 bytes at offset 6912212992
WARNING: Task 235, partial read(), 28311552 of 33554432 bytes at offset 6274678784
WARNING: Task 241, partial read(), 27262976 of 33554432 bytes at offset 6610223104
WARNING: Task 286, partial read(), 29360128 of 33554432 bytes at offset 5972688896
WARNING: Task 268, partial read(), 28311552 of 33554432 bytes at offset 5771362304
WARNING: Task 244, partial read(), 28311552 of 33554432 bytes at offset 5435817984
WARNING: Task 250, partial read(), 28311552 of 33554432 bytes at offset 5905580032
WARNING: Task 252, partial read(), 28311552 of 33554432 bytes at offset 5838471168
WARNING: Task 275, partial read(), 28311552 of 33554432 bytes at offset 6375342080
WARNING: Task 257, partial read(), 27262976 of 33554432 bytes at offset 5637144576
WARNING: Task 279, partial read(), 28311552 of 33554432 bytes at offset 6241124352
WARNING: Expected aggregate file size       = 7905424179200000.
WARNING: Stat() of aggregate file size      = 11169599324160.
WARNING: Using actual aggregate bytes moved = 11169599324160.
read      18283      10158080000 32768      0.102441   582.52     0.002231   582.62     0   
Max Read:  18283.11 MiB/sec (19171.23 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        18283.11   18283.11   18283.11       0.00     571.35     571.35     571.35       0.00  582.62309         NA            NA     0    760  76    1   1     1        1         0    0      1 10401873920000 33554432 10652160.0 POSIX      0
Finished            : Sun Nov  3 21:49:57 2019
ior_easy_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov  3 20:41:07 2019
Command line        : /io500/io500-sc19/bin/ior -w -t 32768k -b 9920000m -F -i 1 -C -Q 1 -g -G 27 -k -e -o /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_easy/ior_file_easy -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_easy/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux e11u05.int.janelia.org
TestID              : 0
StartTime           : Sun Nov  3 20:41:07 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_easy
FS                  : 100.0 TiB   Used FS: 26.3%   Inodes: 858.3 Mi   Used Inodes: 17.7%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_easy/ior_file_easy
access              : file-per-process
type                : independent
segments            : 1
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 760
clients per node    : 76
repetitions         : 1
xfersize            : 32 MiB
blocksize           : 9.46 TiB
aggregate filesize  : 7189.94 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 187 max: 438 -- min data: 5.8 GiB mean data: 9.2 GiB time: 301.9s
WARNING: Expected aggregate file size       = 7905424179200000.
WARNING: Stat() of aggregate file size      = 11169599324160.
WARNING: Using actual aggregate bytes moved = 11169599324160.
WARNING: maybe caused by deadlineForStonewalling
write     22169      10158080000 32768      0.228817   480.25     0.027579   480.50     0   
Max Write: 22168.79 MiB/sec (23245.66 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write       22168.79   22168.79   22168.79       0.00     692.77     692.77     692.77       0.00  480.50262     301.89      23833.69     0    760  76    1   1     1        1         0    0      1 10401873920000 33554432 10652160.0 POSIX      0
Finished            : Sun Nov  3 20:49:08 2019
ior_hard_read
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov  3 21:50:41 2019
Command line        : /io500/io500-sc19/bin/ior -r -R -s 1900000 -a POSIX --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_hard/IOR_file -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_hard/stonewall
Machine             : Linux e11u05.int.janelia.org
TestID              : 0
StartTime           : Sun Nov  3 21:50:41 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_hard
FS                  : 100.0 TiB   Used FS: 41.4%   Inodes: 858.3 Mi   Used Inodes: 24.7%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 760
clients per node    : 76
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 61.74 TiB

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
WARNING: Expected aggregate file size       = 67879552000000.
WARNING: Stat() of aggregate file size      = 5154594548480.
WARNING: Using actual aggregate bytes moved = 5154594548480.
read      25792      45.91      45.91      0.004801   190.59     0.001465   190.60     0   
Max Read:  25791.80 MiB/sec (27044.66 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
read        25791.80   25791.80   25791.80       0.00  575320.34  575320.34  575320.34       0.00  190.59566         NA            NA     0    760  76    1   0     1        1         0    0 1900000    47008    47008 4915804.5 POSIX      0
Finished            : Sun Nov  3 21:53:52 2019
ior_hard_write
IOR-3.3.0+dev: MPI Coordinated Test of Parallel I/O
Began               : Sun Nov  3 21:23:16 2019
Command line        : /io500/io500-sc19/bin/ior -w -s 1900000 -a POSIX --posix.odirect -i 1 -C -Q 1 -g -G 27 -k -e -t 47008 -b 47008 -o /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_hard/IOR_file -O stoneWallingStatusFile=/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_hard/stonewall -O stoneWallingWearOut=1 -D 300
Machine             : Linux e11u05.int.janelia.org
TestID              : 0
StartTime           : Sun Nov  3 21:23:16 2019
Path                : /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_hard
FS                  : 100.0 TiB   Used FS: 36.7%   Inodes: 858.3 Mi   Used Inodes: 24.0%

Options: 
api                 : POSIX
apiVersion          : 
test filename       : /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/ior_hard/IOR_file
access              : single-shared-file
type                : independent
segments            : 1900000
ordering in a file  : sequential
ordering inter file : constant task offset
task offset         : 1
tasks               : 760
clients per node    : 76
repetitions         : 1
xfersize            : 47008 bytes
blocksize           : 47008 bytes
aggregate filesize  : 61.74 TiB
stonewallingTime    : 300
stoneWallingWearOut : 1

Results: 

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s) iter
------    ---------  ---------- ---------  --------   --------   --------   -------- ----
stonewalling pairs accessed min: 16 max: 144281 -- min data: 0.0 GiB mean data: 0.1 GiB time: 301.7s
WARNING: Expected aggregate file size       = 67879552000000.
WARNING: Stat() of aggregate file size      = 5154594548480.
WARNING: Using actual aggregate bytes moved = 5154594548480.
WARNING: maybe caused by deadlineForStonewalling
write     7459       45.91      45.91      0.018384   658.99     0.013144   659.02     0   
Max Write: 7459.21 MiB/sec (7821.55 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev   Max(OPs)   Min(OPs)  Mean(OPs)     StdDev    Mean(s) Stonewall(s) Stonewall(MiB) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt   blksiz    xsize aggs(MiB)   API RefNum
write        7459.21    7459.21    7459.21       0.00  166387.61  166387.61  166387.61       0.00  659.02478     301.68        252.07     0    760  76    1   0     1        1         0    0 1900000    47008    47008 4915804.5 POSIX      0
Finished            : Sun Nov  3 21:34:15 2019
mdtest_easy_delete
-- started at 11/03/2019 21:55:45 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-r' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01
FS: 100.0 TiB   Used FS: 41.4%   Inodes: 858.3 Mi   Used Inodes: 24.7%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 76 for each phase.
760 tasks, 684000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :      21552.039      21552.014      21552.029          0.004
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.384          0.384          0.384          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       2088.729       2088.727       2088.728          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          2.605          2.605          2.605          0.000
-- finished at 11/03/2019 22:30:41 --

mdtest_easy_stat
-- started at 11/03/2019 21:50:07 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-T' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_easy-stonewall' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01
FS: 100.0 TiB   Used FS: 41.4%   Inodes: 858.3 Mi   Used Inodes: 24.7%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 76 for each phase.
760 tasks, 684000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :    1594647.772    1594440.380    1594622.296         22.795
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :         28.233         28.230         28.230          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/03/2019 21:50:35 --

mdtest_easy_write
-- started at 11/03/2019 20:49:14 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-C' '-F' '-P' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_easy' '-n' '900000' '-u' '-L' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_easy-stonewall' '-N' '1' '-W' '300'
Path: /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01
FS: 100.0 TiB   Used FS: 36.5%   Inodes: 858.3 Mi   Used Inodes: 18.9%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 76 for each phase.
760 tasks, 684000000 files
Continue stonewall hit min: 5065 max: 59232 avg: 13973.5 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :      22118.762      22118.523      22118.739          0.030
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA      35411.796             NA
   Tree creation             :          3.819          3.819          3.819          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       2035.232       2035.210       2035.212          0.003
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        299.897             NA
   Tree creation             :          0.262          0.262          0.262          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/03/2019 21:23:10 --

mdtest_hard_delete
-- started at 11/03/2019 22:31:34 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-r' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01
FS: 100.0 TiB   Used FS: 41.3%   Inodes: 858.3 Mi   Used Inodes: 19.7%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :       3630.254       3629.733       3630.244          0.050
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          1.423          1.423          1.423          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :        324.961        324.914        324.915          0.004
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.703          0.703          0.703          0.000
-- finished at 11/03/2019 22:37:00 --

mdtest_hard_read
-- started at 11/03/2019 22:30:47 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-X' '-E' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01
FS: 100.0 TiB   Used FS: 41.3%   Inodes: 858.3 Mi   Used Inodes: 19.7%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :      28652.822      28652.745      28652.773          0.019
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :         41.166         41.166         41.166          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/03/2019 22:31:29 --

mdtest_hard_stat
-- started at 11/03/2019 21:53:58 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-T' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1'
Path: /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01
FS: 100.0 TiB   Used FS: 41.4%   Inodes: 858.3 Mi   Used Inodes: 24.7%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files

SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :      11682.473      11682.136      11682.454          0.035
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :          0.000          0.000          0.000          0.000
   File stat                 :        100.968        100.965        100.965          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   Tree creation             :          0.000          0.000          0.000          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/03/2019 21:55:39 --

mdtest_hard_write
-- started at 11/03/2019 21:34:22 --

mdtest-3.3.0+dev was launched with 760 total task(s) on 10 node(s)
Command line used: /io500/io500-sc19/bin/mdtest '-C' '-t' '-F' '-P' '-w' '3901' '-e' '3901' '-d' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_hard' '-n' '950000' '-x' '/io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01/mdt_hard-stonewall' '-a' 'POSIX' '-N' '1' '-W' '300'
Path: /io500/io500-sc19/datafiles/io500.2019.11.03-20.41.01
FS: 100.0 TiB   Used FS: 41.4%   Inodes: 858.3 Mi   Used Inodes: 24.5%

Nodemap: 1111111111111111111111111111111111111111111111111111111111111111111111111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
V-0: Rank   0 Line  2128 Shifting ranks by 76 for each phase.
760 tasks, 722000000 files
Continue stonewall hit min: 1531 max: 1552 avg: 1542.1 


SUMMARY rate: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :       3892.752       3892.742       3892.747          0.003
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA       3904.734             NA
   Tree creation             :        116.453        116.453        116.453          0.000
   Tree removal              :          0.000          0.000          0.000          0.000

SUMMARY time: (of 1 iterations)
   Operation                      Max            Min           Mean        Std Dev
   ---------                      ---            ---           ----        -------
   File creation             :        303.005        303.004        303.005          0.000
   File stat                 :          0.000          0.000          0.000          0.000
   File read                 :          0.000          0.000          0.000          0.000
   File removal              :          0.000          0.000          0.000          0.000
   File create (stonewall)   :             NA             NA        300.140             NA
   Tree creation             :          0.009          0.009          0.009          0.000
   Tree removal              :          0.000          0.000          0.000          0.000
-- finished at 11/03/2019 21:39:25 --

result_summary
[RESULT] BW   phase 1            ior_easy_write               21.649 GB/s : time 480.50 seconds
[RESULT] IOPS phase 1         mdtest_easy_write               22.119 kiops : time 2035.23 seconds
[RESULT] BW   phase 2            ior_hard_write                7.284 GB/s : time 659.02 seconds
[RESULT] IOPS phase 2         mdtest_hard_write                3.893 kiops : time 303.01 seconds
[RESULT] IOPS phase 3                      find             1058.550 kiops : time  43.75 seconds
[RESULT] BW   phase 3             ior_easy_read               17.854 GB/s : time 582.62 seconds
[RESULT] IOPS phase 4          mdtest_easy_stat             1594.650 kiops : time  28.23 seconds
[RESULT] BW   phase 4             ior_hard_read               25.188 GB/s : time 190.60 seconds
[RESULT] IOPS phase 5          mdtest_hard_stat               11.682 kiops : time 100.97 seconds
[RESULT] IOPS phase 6        mdtest_easy_delete               21.552 kiops : time 2088.73 seconds
[RESULT] IOPS phase 7          mdtest_hard_read               28.653 kiops : time  41.17 seconds
[RESULT] IOPS phase 8        mdtest_hard_delete                3.630 kiops : time 331.65 seconds
[SCORE] Bandwidth 16.3189 GB/s : IOPS 37.3736 kiops : TOTAL 24.6961