#!/usr/bin/bash
# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; tab-width: 8 -*-

# This is a script to run the fio benchmark

script_path=`dirname $0`
script_name=`basename $0`
pbench_bin="`cd ${script_path}/..; /bin/pwd`"

# source the base script
. "$pbench_bin"/base

export benchmark="fio"
export benchmark_run_dir=""

job_file="${script_path}/templates/fio.job"

# Every bench-script follows a similar sequence:
# 1) process bench script arguments
# 2) ensure the right version of the benchmark is installed
# 3) gather pre-run state
# 4) run the benchmark and start/stop perf analysis tools
# 5) postprocess analysis tool data
# 6) gather post-run state
# 7) postprocess benchmark data

orig_cmd="$*"

# Defaults
run_dir=""
keep_failed_tool_data="y"
tar_nonref_data="y"
postprocess_only="n"
nr_samples=5
maxstddevpct=5 # maximum allowable standard deviation in percent
max_failures=6 # after N failed attempts to hit below ${maxstddevpct}, move on to the next test
supported_test_types="read,write,rw,randread,randwrite,randrw"
test_types="read,randread"		# default is -non- destructive
pre_check="n"
export config=""
rate_iops=""
block_sizes="4,64,1024"
targets="/tmp/fio"
numjobs=""
runtime=""
ramptime=""
iodepth=""
ioengine=""
pre_iteration_script=""
job_mode_def="concurrent"
job_mode=${job_mode_def} # serial or concurrent targets
file_size=""
direct="" # don't cache IO's by default
sync="" # don't sync IO's by default
clients=""
client_file=""
tool_label_pattern="fio-"
tool_group="default"
max_key_length=20
primary_metric="readwrite_IOPS"
histogram_interval_sec=$(pbench-config histogram-interval-sec pbench-fio)
if [ -z "$histogram_interval_sec" ] ;then
	histogram_interval_sec=10
fi
sysinfo="default"
declare -a client_names=()
declare -A client_ports=()
unique_ports=0

function usage {
	printf "The following options are available:\n"
	printf "\n"
	printf -- "\t-t str[,str...] --test-types=str[,str...]\n"
	printf "\t\tone or more of %s\n" $supported_test_types
	printf "\n"
	printf -- "\t--direct=[0/1]\n"
	printf "\t\t1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled\n"
	printf "\n"
	printf -- "\t--sync=[0/1]\n"
	printf "\t\t1 = O_SYNC enabled, 0 = O_SYNC disabled (default)\n"
	printf "\n"
	printf -- "\t--rate-iops=int\n"
	printf "\t\tdo not exceeed this IOP rate (per job, per client)\n"
	printf "\n"
	printf -- "\t-r int --runtime=int\n"
	printf "\t\truntime in seconds (default is $runtime)\n"
	printf "\n"
	printf -- "\t--ramptime=int\n"
	printf "\t\ttime in seconds to warm up test before taking measurements (default is $ramptime)\n"
	printf "\n"
	printf -- "\t-b int[,int...] --block-sizes=int[,int...] (default is $block_sizes)\n"
	printf "\t\tone or more block sizes in KiB\n"
	printf "\n"
	printf -- "\t-s int[,int...] --file-size=int[,int...] (no default)\n"
	printf "\t\tfile sizes in bytes (accepts modifiers, e.g. '128m', '1500k', etc.,\n"
	printf "\t\tsee https://fio.readthedocs.io/en/latest/fio_doc.html#int)\n"
	printf "\n"
	printf -- "\t-d str[,str...] --targets=str[,str...]\n"
	printf "\t\tone or more files, block devices, or directories (default is $targets);\n"
	printf "\t\tuse of persistent names for block devices is highly recommended;\n"
	printf "\t\twhen a directory is specified, one must also provide a job file\n"
	printf "\t\tvia '--job-file=<path>' which employs the 'directory = \$target'\n"
	printf "\t\tsyntax, or use the predefined job file provided at:\n"
	printf "\t\t  ${script_path}/templates/fio-shared-fs.job\n"
	printf "\n"
	printf -- "\t-j str --job-mode=str    str=[serial|concurrent]  (default is '${job_mode_def}')\n"
	printf "\t\tdirects how --targets parameter(s) is/are used; with 'serial' mode all combinations\n"
	printf "\t\tof fio job parameters are run against each target one at a time, while with 'concurrent'\n"
	printf "\t\tmode all target devices are used at the same time.\n"
	printf "\n"
	printf -- "\t--ioengine=str           str= any ioengine fio supports (default is $ioengine)\n"
	printf "\n"
	printf -- "\t--iodepth=<int>"
	printf "\t\tSet the iodepth config variable in the fio job file\n"
	printf "\n"
	printf -- "\t-c str[,str...] --clients=str[,str...]      str= a list of one or more host names (hosta,hostb,hostc) where you want fio to run\n"
	printf "\t\tIf no clients are specified, fio is run locally\n"
	printf "\t\tNote: the pbench-agent must be installed on each of the client systems already.\n"
	printf "\n"
	printf -- "\t--client-file=str        str= file (with absolute path) which contains 1 client per line\n"
	printf "\n"
	printf -- "\t--config=str\n"
	printf "\t\tname of the test configuration\n"
	printf "\n"
	printf -- "\t--tool-group=str\n"
	printf "\n"
	printf -- "\t--postprocess-only=[y|n]\n"
	printf "\t\tuse this only if you want to postprocess an existing result again\n"
	printf "\t\tyou must use --run-dir option with this\n"
	printf "\n"
	printf -- "\t--run-dir=<path>\n"
	printf "\t\tprovide the path of an existig result (typically somewhere in $pbench_run\n"
	printf "\n"
	printf -- "\t--numjobs=<int>\n"
	printf "\t\tnumber of jobs to run, if not given then fio default of numjobs=1 will be used\n"
	printf "\n"
	printf -- "\t--job-file=<path>\n"
	printf "\t\tprovide the path of a fio job config file, (default is $job_file)\n"
	printf "\n"
	printf -- "\t--pre-iteration-script=str\n"
	printf "\t\tuse executable script/program to prepare the system for test iteration\n"
	printf "\t\texample: --pre-iteration-script=\$HOME/drop-cache.sh\n"
	printf "\n"
	printf -- "\t--samples=<int>\n"
	printf "\t\tnumber of samples to use per test iteration (default is $nr_samples)\n"
	printf "\n"
	printf -- "\t--max-stddev=<int>\n"
	printf "\t\tthe maximum percent stddev allowed to pass\n"
	printf "\n"
	printf -- "\t--max-failures=<int>\n"
	printf "\t\tthe maximum number of failures to get below stddev\n"
	printf "\n"
	printf -- "\t--histogram-interval-sec=<int>\n"
	printf "\t\tset the histogram logging interval in seconds (default $histogram_interval_sec)\n"
	printf "\n"
	printf -- "\t--sysinfo=str            str= comma separated values of sysinfo to be collected\n"
	printf "\t\tavailable: $(pbench-display-sysinfo-options)\n"
	printf "\n"
	printf -- "\t--unique-ports           Use unique ports for each server\n"
}

function fio_process_options() {
	local opts
	opts=$(getopt -q -o b:c:d:hj:r:s:t: --longoptions "help,max-stddev:,max-failures:,samples:,direct:,sync:,pre-check,clients:,client-file:,iodepth:,ioengine:,config:,jobs-per-dev:,job-mode:,rate-iops:,ramptime:,runtime:,test-types:,block-sizes:,file-size:,targets:,tool-group:,postprocess-only:,run-dir:,numjobs:,job-file:,sysinfo:,pre-iteration-script:,histogram-interval-sec:,histogram-interval-msec:,unique-ports" -n "getopt.sh" -- "$@")
	if [ $? -ne 0 ]; then
		printf -- "${script_name} $*\n"
		printf -- "\n"
		printf -- "\tunrecognized option specified\n\n"
		usage >&2
		exit 1
	fi
	eval set -- "$opts"
	local arg
	while true; do
		arg="${1}"
		shift
		case "${arg}" in
		-h|--help)
			usage
			exit 0
			;;
		--pre-check)
			if [[ "$postprocess_only" != "n" ]]; then
				printf -- "${script_name} $*\n"
				printf -- "\n"
				printf -- "\t--pre-check not compatible with specified \"--postprocess-only ${postprocess_only}\" option\n\n"
				usage >&2
				exit 1
			fi
			pre_check="y"
			;;
		--max-stddev)
			if [ -n "$1" ]; then
				maxstddevpct="$1"
				shift
			fi
			;;
		--max-failures)
			if [ -n "$1" ]; then
				max_failures="$1"
				shift
			fi
			;;
		--samples)
			if [ -n "$1" ]; then
				nr_samples="$1"
				shift
			fi
			;;
		--direct)
			if [ -n "$1" ]; then
				direct=$1
				shift
			fi
			;;
		--sync)
			if [ -n "$1" ]; then
				sync=$1
				shift
			fi
			;;
		-t|--test-types)
			if [ -n "$1" ]; then
				test_types="$1"
				shift
			fi
			;;
		-b|--block-sizes)
			if [ -n "$1" ]; then
				block_sizes="$1"
				shift
			fi
			;;
		-s|--file-size)
			if [ -n "$1" ]; then
				file_size="$1"
				shift
			fi
			;;
		--ramptime)
			if [ -n "$1" ]; then
				ramptime="$1"
				shift
			fi
			;;
		--rate-iops)
			if [ -n "$1" ]; then
				rate_iops="$1"
				shift
			fi
			;;
		-r|--runtime)
			if [ -n "$1" ]; then
				runtime="$1"
				shift
			fi
			;;
		-c|--clients)
			if [ -n "$client_file" ] ;then
				printf -- "${script_name} $*\n"
				printf -- "\n"
				printf -- "\t-c|--clients and --client-file are mutually exclusive\n\n"
				usage >&2
				exit 1
			fi
			if [ -n "$1" ]; then
				clients="$1"
				shift
			fi
			;;
		--client-file)
			if [ -n "${clients}" ] ;then
				printf -- "${script_name} $*\n"
				printf -- "\n"
				printf -- "\t--client-file and -c|--clients are mutually exclusive\n\n"
				usage >&2
				exit 1
			fi
			client_file=$1
			shift
			if [ ! -e "${client_file}" ]; then
				printf -- "${script_name} $*\n"
				printf -- "\n"
				printf -- "\tSpecified client file, ${client_file}, does not exist\n\n"
				usage >&2
				exit 1
			fi
			if [[ "$client_file" != /* ]] ;then
				# make it absolute
				client_file=$PWD/$client_file
			fi
			while read line; do
				clients="${clients},${line}"
			done < ${client_file}
			clients=${clients:1}
			;;
		-d|--targets)
			if [ -n "$1" ]; then
				targets="$1"
				shift
			fi
			;;
		-j|--job-mode)
			if [ -n "$1" ]; then
				job_mode="$1"
				shift
			fi
			;;
		--config)
			if [ -n "$1" ]; then
				config="$1"
				shift
			fi
			;;
		--ioengine)
			if [ -n "$1" ]; then
				ioengine="$1"
				shift
			fi
			;;
		--iodepth)
			if [ -n "$1" ]; then
				iodepth="$1"
				shift
			fi
			;;
		--tool-group)
			if [ -n "$1" ]; then
				tool_group="$1"
				shift
			fi
			;;
		--postprocess-only)
			if [ -n "$1" ]; then
				postprocess_only="$1"
				shift
				if [[ "$postprocess_only" != "n" && "$pre_check" == "y" ]]; then
					printf -- "${script_name} $*\n"
					printf -- "\n"
					printf -- "\t\"--postprocess-only ${postprocess_only}\" not compatible with specified --pre-check option\n\n"
					usage >&2
					exit 1
				fi
			fi
			;;
		--run-dir)
			if [ -n "$1" ]; then
				run_dir="$1"
				shift
			fi
			;;
		--numjobs)
			if [ -n "$1" ]; then
				numjobs="$1"
				shift
			fi
			;;
		--job-file)
			if [ -n "$1" ]; then
				job_file="$1"
				shift

			fi
			;;
		--pre-iteration-script)
			if [ -n "$1" ]; then
				pre_iteration_script="$1"
				if [ ! -x $pre_iteration_script ]; then
					printf "ERROR: $pre_iteration_script must be executable\n"
					exit 1
				fi
				shift
			fi
			;;
		--histogram-interval-sec)
			histogram_interval_sec="$1"
			shift
			;;
			# histogram-interval-msec is only kept for backwards compatibility
			# it is deprecated and is not documented in the --help output
		--histogram-interval-msec)
			(( histogram_interval_sec = $1 / 1000 ))
			shift
			;;
		--sysinfo)
			if [ -n "$1" ]; then
				sysinfo="$1"
				shift
			fi
			;;
		--unique-ports)
			unique_ports=1
			;;
		--)
			break
			;;
		*)
			usage >&2
			exit 1
			;;
		esac
	done

	if [[ "${pre_check}" == "y" ]]; then
		# The extra arguments are the expected version number and match
		# condition the controller found in its configuration file. This
		# means we do NOT use the local client's configuration file for
		# the `fio` version.
		local_pre_check "${targets}" ${@}
		exit ${?}
	fi

	verify_common_bench_script_options $tool_group $sysinfo

	if [[ ${job_mode} != "serial" && ${job_mode} != "concurrent" ]]; then
		error_log "Unsupported --job-mode=${job_mode} encountered"
		usage >&2
		exit 1
	fi

	local tt=""
	local space_sep_stt=${supported_test_types//,/ }
	for tt in ${test_types//,/ }; do
		local found=0
		for vtt in ${space_sep_stt}; do
			if [[ "${tt}" == "${vtt}" ]]; then
				found=1
				break
			fi
		done
		if [[ ${found} -ne 1 ]]; then
			error_log "Unsupported --test-types=${test_types} encountered"
			usage >&2
			exit 1
		fi
	done

	if [ "$postprocess_only" = "n" ]; then
		benchmark_fullname="${benchmark}_${config}_${date_suffix}"
		benchmark_run_dir="$pbench_run/${benchmark_fullname}"
	else
		if [ -z "$run_dir" ]; then
			error_log "Please specify a directory if postprocessing an existing result (--run-dir=)"
			exit 1
		fi
		benchmark_fullname="$(basename $run_dir)"
		benchmark_run_dir="$run_dir"
	fi
}

function record_iteration {
	local count=${1}
	local test_type=${2}
	local block_size=${3}
	local dev=${4}
	local iteration=${5}

	local mdlog=${benchmark_run_dir}/metadata.log
	echo ${iteration} >> ${benchmark_iterations}
	echo ${count} | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_number
	echo ${test_type} | pbench-add-metalog-option ${mdlog} iterations/${iteration} test_type
	echo ${block_size} | pbench-add-metalog-option ${mdlog} iterations/${iteration} block_size_KiB
	echo ${dev} | pbench-add-metalog-option ${mdlog} iterations/${iteration} dev
	echo ${iteration} | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_name
}

function local_pre_check() {
	local devs="${1}"
	local ver="${2}"
	local match="${3}"

	debug_log "local_pre_check(devs=\"${devs}\", ver=\"${ver}\", match=\"${match}\")"

	# First check for the supported version of fio in use.

	local _stdout
	_stdout=$(${benchmark} --version 2>&1)
	if [[ ${?} -ne 0 ]]; then
		error_log "[${script_name}] ${benchmark} failed reporting its version: '${_stdout}'"
		exit 1
	fi
	local ver_installed=${_stdout##*-}

	vercmp "${benchmark}" "${match}" "${ver}" "${ver_installed}"
	if [[ ${?} -ne 0 ]]; then
		return 1
	fi

	# Now if we have any block devices specified, verify them.

	local dev=""
	local rc=0
	for dev in ${devs//,/ }; do
		if [[ ${dev} =~ ^/dev/ ]]; then
			local rp_dev=$(realpath ${dev})
			if [[ ! -b ${rp_dev} ]]; then
				error_log "${pbench_hostname_full} does not have block device ${dev}"
			fi
		fi
	done

	return ${rc}
}

function fio_pre_check() {
	local targets="${1}"
	local clients="${2}"
	local ver="${3}"
	local match="${4}"

	local rc=0
	local do_local_pre_check=0

	debug_log "fio_pre_check(targets=\"${targets}\", clients=\"${clients}\", ver=\"${ver}\", match=\"${match}\")"
	if [ -n "${clients}" ]; then
		local client=""
		for client in "${client_names[@]}" ; do
			if pbench-is-local ${client}; then
				do_local_pre_check=1
			else
				debug_log "Running pre-check on client ${client}"
				ssh ${ssh_opts} ${client} pbench-fio --pre-check --targets="${targets}" "${ver}" "${match}" || rc=1
			fi
		done
	else
		do_local_pre_check=1
	fi
	if [[ ${do_local_pre_check} -eq 1 ]]; then
		debug_log "Running pre-check locally"
		local_pre_check "${devs}" "${ver}" "${match}" || rc=${?}
	fi
	if [[ ${rc} -ne 0 ]]; then
		error_log "pre-check(s) failed, exiting"
		exit 1
	fi
}

function fio_create_jobfile() {
	local fio_job_file="${13}"
	local fio_job_file_path=$(dirname "${fio_job_file}")
	mkdir -p "${fio_job_file_path}"
	if [ ${?} -ne 0 ]; then
		error_log "Failed to create path to job file \"${fio_job_file}\""
		exit 1
	fi
	"${script_path}/templates/make-fio-jobfile.py" -j ${job_file} \
		-rw="${1}" \
		-ioengine="${2}" \
		-bs="$(printf %sk ${3})" \
		-iodepth="${4}" \
		-direct="${5}" \
		-sync="${6}" \
		-runtime="${7}" \
		-ramp_time="${8}" \
		-size="${9}" \
		-rate_iops="${10}" \
		-log_hist_msec="${11}" \
		-targets ${12//,/ } \
		-numjobs="${14}" \
		| sed -e 's/ = /=/g' > ${fio_job_file}

	if [ ${?} -ne 0 ]; then
		error_log "Failed to create job file \"${fio_job_file}\""
		exit 1
	fi
}

function does_wildcard_exist() {
	local fn_pat=$1
	local found=1
	for fn in ${fn_pat}; do
		if [[ -e ${fn} ]]; then
			found=0
			break
		fi
	done
	return ${found}
}

function fio_run_job() {
	local iteration="$1"
	local benchmark_results_dir="$2"
	local sample_name="$(basename ${benchmark_results_dir})"
	local fio_job_file="$3"
	local bench_opts="--output-format=json $fio_job_file"
	local tgts="$4"
	local clients="$5"

	mkdir -p $benchmark_results_dir/clients
	if [[ ! -d $benchmark_results_dir/clients ]]; then
		error_log "Failed to create results directory hierarchy: $benchmark_results_dir/clients"
		exit 1
	fi

	echo "running fio job: $fio_job_file ($(basename $benchmark_results_dir))"

	if [ -n "${clients}" ]; then
		for client in "${client_names[@]}"; do
			if pbench-is-local ${client}; then
				debug_log "killing any old local ${benchmark} processes and starting new a ${benchmark} process for the local client"
				${script_path}/driver/${benchmark}-client ${benchmark_results_dir} ${client_ports[${client}]}
			else
				debug_log "killing any old ${benchmark} processes, creating directories, and starting new a ${benchmark} process on remote client ${client}:${client_ports[${client}]}"
				ssh ${ssh_opts} ${client} "mkdir -p ${benchmark_results_dir} && ${script_path}/driver/${benchmark}-client ${benchmark_results_dir} ${client_ports[${client}]}"
			fi
		done
		debug_log "waiting for all ${benchmark} client processes to start on the specified clients"
		for client in "${client_names[@]}"; do
			timeout --kill-after=1 60 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' "${client}" "${client_ports[${client}]}"
			if [[ $? -eq 124 ]]; then
				error_log "${benchmark} process took more than 60 seconds to start in client ${client}, failing"
				exit 1
			fi
		done
	fi

	local client_opts=""

	if [ -n "${clients}" ]; then
		# If a client-file parameter was passed, then clients is already
		# initialized to its contents.
		local max_jobs=0
		local nclients=0

		if [ -n "${numjobs}" ]; then
			# So that a separate filename will be generated for each (host,
			# job-number) pair.
			max_jobs=${numjobs}
		else
			for t in ${tgts//,/ }; do
				(( max_jobs++ ))
			done
		fi

		let nclients=`wc -l < $_client_file`
		let max_jobs=$max_jobs*$nclients
		client_opts="$client_opts --client=$_client_file --max-jobs=$max_jobs"
	fi

	# Certain test preparation steps such as cache dropping can be a bit
	# hard on the system, give it a few seconds before actually starting
	# test by putting this before pbench-start-tools.
	if [ -n "$pre_iteration_script" ] ; then
		printf "running pre-iteration-script command: $pre_iteration_script\n"
		eval "$pre_iteration_script"
	fi

	pbench-start-tools --group=$tool_group --dir=$benchmark_results_dir

	# create a command file and keep it with the results for debugging later, or user can run outside of pbench
	echo "$benchmark $client_opts $bench_opts" >$benchmark_results_dir/fio.cmd
	chmod +x $benchmark_results_dir/fio.cmd
	debug_log "$benchmark: Going to run [$benchmark $bench_opts $client_opts]"
	pushd $benchmark_results_dir >/dev/null
	$benchmark_results_dir/fio.cmd >$benchmark_results_dir/fio-result.txt
	local _fio_exit_code=$?
	popd >/dev/null

	pbench-stop-tools --group=$tool_group --dir=$benchmark_results_dir
	pbench-send-tools --group=$tool_group --dir=$benchmark_results_dir

	local histogram_params="--time-quantum ${histogram_interval_sec} --percentiles 0 50 90 95 99 99.5 100"
	local histogram_cmd="fio-histo-log-pctiles.py ${histogram_params} -- "

	# First move all fio log files to the appropriate client directory.
	local _clients_list=""
	if [ -n "${clients}" ]; then
		local client
		for client in "${client_names[@]}"; do
			_clients_list="${_clients_list} ${client}"
			mkdir -p ${benchmark_results_dir}/clients/${client}
			/bin/mv ${benchmark_results_dir}/*.log.${client} ${benchmark_results_dir}/clients/${client}/ 2> /dev/null
			# remove the trailing client name in the log file names
			pushd ${benchmark_results_dir}/clients/${client} >/dev/null
			for i in $(/bin/ls | grep log); do
				mv ${i} $(echo ${i} | sed -e s/\.${client}//)
			done
			popd >/dev/null
		done
	else
		_clients_list="localhost"
		mkdir -p ${benchmark_results_dir}/clients/localhost/
		/bin/mv ${benchmark_results_dir}/*.log ${benchmark_results_dir}/clients/localhost/ 2> /dev/null
	fi

	if grep -q "^log_hist_msec" ${fio_job_file}; then
		for client in ${_clients_list}; do
			# Process the histograms
			# Determine histogram interval based on value used in job file.
			#my $log_interval = `cat $dir/fio.job | grep "^log_hist_msec"`;
			#if [ ! $log_interval =~ /^\s*$/ ]; then
			local log_dir="${benchmark_results_dir}/clients/${client}"
			does_wildcard_exist ${log_dir}/'fio_clat_hist.*.log*'
			if [[ ${?} -eq 0 ]]; then
				local hist_dir="${log_dir}/hist"
				mkdir -p ${hist_dir}
				# compute per-client latency percentiles
				${histogram_cmd} ${log_dir}/fio_clat_hist.*.log* >${hist_dir}/hist.csv
				${script_path}/postprocess/${benchmark}-postprocess-viz.py --job-file=${fio_job_file} ${histogram_params} -- ${hist_dir}
			else
				warn_log "log_hist_msec specified in job file but failed to find any log files (.../clients/${client}/fio_clat_hist.*.log*) to process for histograms"
			fi
		done
		does_wildcard_exist ${benchmark_results_dir}/clients/'*/fio_clat_hist.*.log*'
		if [[ ${?} -eq 0 ]]; then
			local hist_dir="${benchmark_results_dir}/hist"
			mkdir -p ${hist_dir}
			# compute cluster-wide latency percentiles
			${histogram_cmd} ${benchmark_results_dir}/clients/*/fio_clat_hist.*.log* >${hist_dir}/hist.csv
			${script_path}/postprocess/${benchmark}-postprocess-viz.py --job-file=${fio_job_file} ${histogram_params} -- ${benchmark_results_dir}/hist
		else
			warn_log "log_hist_msec specified in job file but failed to find any log files (.../clients/*/fio_clat_hist.*.log*) to process for histograms"
		fi
	fi

	if [[ ${_fio_exit_code} -eq 0 ]]; then
		printf "fio job complete\n"
	else
		printf "fio job failed, status code %s\n" "${_fio_exit_code}"
	fi
	return ${_fio_exit_code}
}

# Run the benchmark and start/stop perf analysis tools
function fio_run_benchmark() {
	(( histogram_interval_msec = histogram_interval_sec * 1000 ))

	if [[ -n "$client_file" ]] ;then
		# Copy the clients file given on the command line into the run
		# directory to keep it with the run.
		_client_file=$benchmark_run_dir/fio-client.file
		cp $client_file ${_client_file}
	else
		if [[ -n "${clients}" ]]; then
			# Record the -c|--clients list into the
			# fio-client.file.  This makes the fio command shorter
			# by always using a client file regardless of how
			# clients were specified.  We only need to save the
			# client list once in the benchmark run directory.
			_client_file=$benchmark_run_dir/fio-client.file
			for client in "${client_names[@]}"; do
				echo ip:$client,${client_ports[$client]}
			done > $_client_file
		fi
	fi

	local count=1
	if [ "$job_mode" = "serial" ]; then
		# if each target is separated by a space, there will be one job for each in next for loop
		targets=${targets//,/ }
	fi
	local _targets_arr=(${targets})
	typeset -i ntargets=${#_targets_arr[@]}
	local _test_types_arr=(${test_types})
	typeset -i ntesttypes=${#_test_types_arr[@]}
	local _block_sizes_arr=(${block_sizes})
	typeset -i nblocksizes=${#_block_sizes_arr[@]}
	typeset -i total_iterations=$ntargets*$ntesttypes*$nblocksizes
	local iteration_failures=0
	for dev in $targets; do
		for test_type in ${test_types//,/ }; do
			for block_size in ${block_sizes//,/ }; do
				local job_num=1
				local iteration="${count}-${test_type}-${block_size}KiB"
				if [[ "$job_mode" = "serial" ]]; then
					dev_short_name="$(basename $dev)"
					# easier to identify what job used what device when having 1 job per device
					iteration="${iteration}-${dev_short_name}"
				fi
				record_iteration ${count} ${test_type} ${block_size} ${dev} ${iteration}
				local iteration_dir=$benchmark_run_dir/$iteration
				local failures=0
				while true; do
					if [[ $failures -gt 0 ]]; then
						echo "Restarting iteration $iteration ($count of $total_iterations)"
						log "Restarting iteration $iteration ($count of $total_iterations)"
					fi
					if [ "$postprocess_only" = "n" ]; then
						mkdir -p $iteration_dir
					else
						if [ ! -e $iteration_dir ]; then
							# if the iteration dir does not exist, look for a failed result directory or archive
							fail_nr=$failures
							((fail_nr++))
							fail_tag="-fail$fail_nr"
							failed_iteration_dir="$iteration_dir$fail_tag"
							if [ -e $failed_iteration_dir ]; then
								mv $failed_iteration_dir $iteration_dir || exit 1
							else
								failed_iteration_archive="$iteration_dir$fail_tag.tar.xz"
								if [ -e $failed_iteration_archive ]; then
									echo "using $failed_iteration_archive as $iteration_dir"
									tar -C $benchmark_run_dir -J -x -f $failed_iteration_archive || exit 1
									mv $failed_iteration_dir $iteration_dir || exit 1
								else
									echo "Could not find $iteration_dir, $failed_iteration_dir, or $failed_iteration_archive"
								fi
							fi
						fi
					fi
					if [ -e $iteration_dir ]; then
						local fio_job_file="$iteration_dir/fio.job"
						local _action=""
						if [ "$postprocess_only" = "n" ]; then
							_action="Created"
							#             ARG   1            2           3             4          5         6       7          8           9            10           11                         12     13              14
							fio_create_jobfile "$test_type" "$ioengine" "$block_size" "$iodepth" "$direct" "$sync" "$runtime" "$ramptime" "$file_size" "$rate_iops" "$histogram_interval_msec" "$dev" "$fio_job_file" "$numjobs"
						else
							_action="Found"
						fi
						echo "${_action} the following job file (${fio_job_file}):"
						cat ${fio_job_file}

						for sample in `seq 1 $nr_samples`; do
							# each attempt at a test config requires multiple samples to get stddev
							local benchmark_results_dir="$iteration_dir/sample$sample"
							if [ "$postprocess_only" = "n" ]; then
								if [ -e $benchmark_results_dir ]; then
									# FIXME - Shouldn't we error and exit out instead of removing data?
									/bin/rm -rf $benchmark_results_dir
								fi
								mkdir -p $benchmark_results_dir
								fio_run_job "$iteration" "$benchmark_results_dir" "$fio_job_file" "$dev" "${clients}"
								rc=$?
								if [ $rc -ne 0 ]; then
									debug_log "failed: fio_run_job \"$iteration\" \"$benchmark_results_dir\" \"$fio_job_file\" \"$dev\" \"${clients}\""
									# we need all samples to be good, so bust out of testing samples now
									((iteration_failures++))
									break 2
								fi
							else
								# we are only postprocessing, so we might have to untar an existing result
								if [ ! -e $benchmark_results_dir ]; then
									if [ -e ${iteration_dir}/sample${sample}.tar.xz ]; then
										pushd $iteration_dir >/dev/null
										tar Jxf sample$sample.tar.xz && /bin/rm sample$sample.tar.xz
										local _tar_rc=$?
										popd >/dev/null
										if [[ ${_tar_rc} -ne 0 ]]; then
											error_log "could not unpack sample tar ball, ${iteration_dir}/sample${sample}.tar.xz"
											((iteration_failures++))
											break 2
										fi
									else
										echo "sample $sample missing.  There should be $nr_samples samples"
										((iteration_failures++))
										break 2
									fi
								fi
							fi
							debug_log "post-processing tool data"
							pbench-postprocess-tools --group=$tool_group --dir=$benchmark_results_dir
							debug_log "post-processing fio result"
							echo "$script_path/postprocess/$benchmark-postprocess \"$benchmark_results_dir\" \"$tool_label_pattern\" \"$tool_group\"" >"$benchmark_results_dir/$benchmark-postprocess.cmd"
							chmod +x "$benchmark_results_dir/$benchmark-postprocess.cmd"
							$benchmark_results_dir/$benchmark-postprocess.cmd
							rc=$?
							# if for any reason the benchmark postprocessing script fails, consider this a failure to get a sample
							if [ $rc -ne 0 ]; then
								debug_log "failed: $(cat $benchmark_results_dir/$benchmark-postprocess.cmd)"
								# we need all samples to be good, so bust out of testing samples now
								((iteration_failures++))
								break 2
							fi
						done
					fi
					echo "$script_path/postprocess/process-iteration-samples \"$iteration_dir\" \"$primary_metric\" \"$maxstddevpct\" \"$failures\" \"$max_failures\" \"$tar_nonref_data\" \"$keep_failed_tool_data\"" >"$iteration_dir/process-iteration-samples.cmd"
					chmod +x "$iteration_dir/process-iteration-samples.cmd"
					$iteration_dir/process-iteration-samples.cmd
					fail=$?
					if [[ $fail -eq 0 || "$postprocess_only" = "n" ]]; then
						# Either this sample was successful, or we are in
						# post-processing only mode, so break out of the retry
						# loop.
						break
					fi
					# The sample was not within the max stddev.
					((failures++))
					if [ $failures -ge $max_failures ]; then
						# We consider this iteration to be a failure since too
						# many sample failures were encountered.
						((iteration_failures++))
						break
					fi
				done
				let count=$count+1 # now we can move to the next iteration
			done
		done
	done
	return ${iteration_failures}
}

fio_process_options "$@"

if [ "$postprocess_only" = "n" ]; then
	ver="$(pbench-config version ${benchmark})"
	if [[ -z "${ver}" ]]; then
		error_log "${script_name}: 'version' is missing in the [${benchmark}] section of the config file"
		exit 1
	fi
	match="$(pbench-config match ${benchmark})"
	if [[ -z "${match}" ]]; then
		error_log "${script_name}: 'match' is missing in the [${benchmark}] section of the config file"
		exit 1
	fi
	fio_server_port="$(pbench-config server_port ${benchmark})"
	if [[ -z "${fio_server_port}" ]]; then
		error_log "${script_name}: 'server_port' is missing in the [${benchmark}] section of the config file"
		exit 1
	fi

	if [[ -n "${clients}" ]] ; then
		let local_client_count=0
		for client in ${clients//,/ } ; do
			if [[ $client = *':'* ]] ; then
				client_name=${client%%:*}
				client_port=${client#*:}
			else
				client_name=${client}
				client_port=${fio_server_port}
				if (( unique_ports )) ; then
					fio_server_port=$((fio_server_port+1))
				fi
			fi
			if pbench-is-local ${client}; then
				(( local_client_count++ ))
				if [[ ${local_client_count} -gt 1 ]]; then
					error_log "${script_name}: more than one client refers to the local host"
					exit 1
				fi
			fi
			client_names+=("${client_name}")
			client_ports["${client_name}"]=${client_port}
		done
	fi

	fio_pre_check "${targets}" "${clients}" "${ver}" "${match}"

	mkdir -p $benchmark_run_dir/.running
	if [[ $? -ne 0 ]]; then
		error_log "Unable to create $benchmark_run_dir path and/or .running marker directory"
		exit 1
	fi
fi

benchmark_iterations="${benchmark_run_dir}/.iterations"
> ${benchmark_iterations}

if [[ "$postprocess_only" == "n" ]]; then
	# Start the tool meisters on each registered local/remote host
	pbench-tool-meister-start --sysinfo="${sysinfo}" "${tool_group}"
	if [[ ${?} != 0 ]]; then
		error_log "[${script_name}]: failed to start the tool meisters."
		exit 1
	fi

	trap "interrupt" INT QUIT TERM
fi

fio_run_benchmark
_frb_sts=${?}

echo "$script_path/postprocess/generate-benchmark-summary \"$benchmark\" \"$orig_cmd\" \"$benchmark_run_dir\"" >"$benchmark_run_dir/generate-benchmark-summary.cmd"
chmod +x "$benchmark_run_dir/generate-benchmark-summary.cmd"
$benchmark_run_dir/generate-benchmark-summary.cmd
_gbs_sts=${?}

if [[ "$postprocess_only" == "n" ]]; then
	# Stop the tool meisters on each registered local/remote host
	pbench-tool-meister-stop --sysinfo="${sysinfo}" "${tool_group}"
	if [[ ${?} != 0 ]]; then
		error_log "[${script_name}]: failed to stop the tool meisters."
	fi
	rmdir $benchmark_run_dir/.running
fi

# FYI this is tricky using the exit status of `[[` itself, where we only exit
# with 0 if both statuses are 0, otherwise we exit with 1 indicating an error.
[[ ${_frb_sts} -eq 0 && ${_gbs_sts} -eq 0 ]]
