#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; tab-width: 8 -*-

# This is a script to run the fio benchmark

script_path=`dirname $0`
script_name=`basename $0`
pbench_bin="`cd ${script_path}/..; /bin/pwd`"

# source the base script
. "$pbench_bin"/base

benchmark="fio"
benchmark_rpm=${benchmark}
export benchmark_run_dir=""
# allow unit tests to override
if [[ -z "${benchmark_bin}" ]]; then
	benchmark_bin="$(resolve_benchmark_bin "${benchmark}")"
	if [[ -z "${benchmark_bin}" ]]; then
		error_log "[${script_name}] ${benchmark} executable not found on PATH=${PATH}"
		exit 1
	fi
fi
ver="$(getconf.py version ${benchmark})"
if [[ -z "${ver}" ]]; then
	error_log "${script_name}: package version is missing in config file"
	exit 1
fi
fio_server_port="$(getconf.py server_port ${benchmark})"
if [[ -z "${fio_server_port}" ]]; then
	error_log "${script_name}: server_port is missing in config file"
	exit 1
fi

job_file="${script_path}/templates/fio.job"

# Every bench-script follows a similar sequence:
# 1) process bench script arguments
# 2) ensure the right version of the benchmark is installed
# 3) gather pre-run state
# 4) run the benchmark and start/stop perf analysis tools
# 5) postprocess analysis tool data
# 6) gather post-run state
# 7) postprocess benchmark data

orig_cmd="$*"

# Defaults
run_dir=""
keep_failed_tool_data="y"
tar_nonref_data="y"
postprocess_only="n"
nr_samples=5
maxstddevpct=5 # maximum allowable standard deviation in percent
max_failures=6 # after N failed attempts to hit below $maxstddevpct, move on to the nest test
supported_test_types="read,write,rw,randread,randwrite,randrw"
test_types="read,randread"		# default is -non- destructive
install_only="n"
config=""
rate_iops=""
block_sizes="4,64,1024"
targets="/tmp/fio"
numjobs=""
runtime=""
ramptime=""
iodepth=""
ioengine=""
pre_iteration_script=""
job_mode_def="concurrent"
job_mode=${job_mode_def} # serial or concurrent targets
file_size=""
direct="" # don't cache IO's by default
sync="" # don't sync IO's by default
clients=""
client_file=""
tool_label_pattern="fio-"
tool_group="default"
max_key_length=20
primary_metric="readwrite_IOPS"
histogram_interval_sec=$(getconf.py histogram-interval-sec pbench-fio)
if [ -z "$histogram_interval_sec" ] ;then
	histogram_interval_sec=10
fi
sysinfo="default"

function usage {
	printf "The following options are available:\n"
	printf "\n"
	printf -- "\t-t str[,str] --test-types=str[,str]\n"
	printf "\t\tone or more of %s\n" $supported_test_types
	printf "\n"
	printf -- "\t--direct=[0/1]\n"
	printf "\t\t1 = O_DIRECT enabled (default), 0 = O_DIRECT disabled\n"
	printf "\n"
	printf -- "\t--sync=[0/1]\n"
	printf "\t\t1 = O_SYNC enabled, 0 = O_SYNC disabled (default)\n"
	printf "\n"
	printf -- "\t--rate-iops=int\n"
	printf "\t\tdo not exceeed this IOP rate (per job, per client)\n"
	printf "\n"
	printf -- "\t-r int --runtime=int\n"
	printf "\t\truntime in seconds (default is $runtime)\n"
	printf "\n"
	printf -- "\t--ramptime=int\n"
	printf "\t\ttime in seconds to warm up test before taking measurements (default is $ramptime)\n"
	printf "\n"
	printf -- "\t-b int[,int] --block-sizes=str[,str] (default is $block_sizes)\n"
	printf "\t\tone or more block sizes in KiB\n"
	printf "\n"
	printf -- "\t-s int[,int] --file-size=str[,str] (no default)\n"
	printf "\t\tfile sizes in MiB\n"
	printf "\n"
	printf -- "\t-d str[,str] --targets=str[,str]\n"
	printf "\t\tone or more directories or block devices (default is $targets)\n"
	printf "\t\t(persistent names for devices highly recommended)\n"
	printf "\n"
	printf -- "\t-j str --job-mode=str    str=[serial|concurrent]  (default is '${job_mode_def}')\n"
	printf "\t\tdirects how --targets parameter(s) is/are used; with 'serial' mode all combinations\n"
	printf "\t\tof fio job parameters are run against each target one at a time, while with 'concurrent'\n"
	printf "\t\tmode all target devices are used at the same time.\n"
	printf "\n"
	printf -- "\t--ioengine=str           str= any ioengine fio supports (default is $ioengine)\n"
	printf "\n"
	printf -- "\t--iodepth=<int>"
	printf "\t\tSet the iodepth config variable in the fio job file\n"
	printf "\n"
	printf -- "\t-c str[,str] --clients=str[,str]      str= a list of one or more host names (hosta,hostb,hostc) where you want fio to run\n"
	printf "\t\tIf no clients are specified, fio is run locally\n"
	printf "\t\tNote: the pbench-agent must be installed on each of the client systems already.\n"
	printf "\n"
	printf -- "\t--client-file=str        str= file (with absolute path) which contains 1 client per line\n"
	printf "\n"
	printf -- "\t--config=str\n"
	printf "\t\tname of the test configuration\n"
	printf "\n"
	printf -- "\t--tool-group=str\n"
	printf "\n"
	printf -- "\t--postprocess-only=[y|n]\n"
	printf "\t\tuse this only if you want to postprocess an existing result again\n"
	printf "\t\tyou must use --run-dir option with this\n"
	printf "\n"
	printf -- "\t--run-dir=<path>\n"
	printf "\t\tprovide the path of an existig result (typically somewhere in $pbench_run\n"
	printf "\n"
	printf -- "\t--numjobs=<int>\n"
	printf "\t\tnumber of jobs to run, if not given then fio default of numjobs=1 will be used\n"
	printf "\n"
	printf -- "\t--job-file=<path>\n"
	printf "\t\tprovide the path of a fio job config file, (default is $job_file)\n"
	printf "\n"
	printf -- "\t--pre-iteration-script=str\n"
	printf "\t\tuse executable script/program to prepare the system for test iteration\n"
	printf "\t\texample: --pre-iteration-script=\$HOME/drop-cache.sh\n"
	printf "\n"
	printf -- "\t--samples=<int>\n"
	printf "\t\tnumber of samples to use per test iteration (default is $nr_samples)\n"
	printf "\n"
	printf -- "\t--max-stddev=<int>\n"
	printf "\t\tthe maximum percent stddev allowed to pass\n"
	printf "\n"
	printf -- "\t--max-failures=<int>\n"
	printf "\t\tthe maximum number of failures to get below stddev\n"
	printf "\n"
	printf -- "\t--install\n"
	printf "\t\tinstall only (default is False)\n"
	printf "\n"
	printf -- "\t--histogram-interval-sec=<int>\n"
	printf "\t\tset the histogram logging interval in seconds (default $histogram_interval_sec)\n"
	printf "\n"
	printf -- "\t--sysinfo=str            str= comma separated values of sysinfo to be collected\n"
	printf "\t\tavailable: $(pbench-collect-sysinfo --options)\n"
}

function fio_process_options() {
	opts=$(getopt -q -o b:c:d:hj:r:s:t: --longoptions "help,max-stddev:,max-failures:,samples:,direct:,sync:,install,remote-only,clients:,client-file:,iodepth:,ioengine:,config:,jobs-per-dev:,job-mode:,rate-iops:,ramptime:,runtime:,test-types:,block-sizes:,file-size:,targets:,tool-group:,postprocess-only:,run-dir:,numjobs:,job-file:,sysinfo:,pre-iteration-script:,histogram-interval-sec:,histogram-interval-msec:" -n "getopt.sh" -- "$@");
	if [ $? -ne 0 ]; then
		printf -- "${script_name} $*\n"
		printf -- "\n"
		printf -- "\tunrecognized option specified\n\n"
		usage
		exit 1
	fi
	eval set -- "$opts";
	while true; do
		case "$1" in
		-h|--help)
			usage
			exit
			;;
		--install)
			shift;
			if [[ "$postprocess_only" != "n" ]]; then
				printf -- "${script_name} $*\n"
				printf -- "\n"
				printf -- "\t--install not compatible with specified \"--postprocess-only ${postprocess_only}\" option\n\n"
				usage
				exit 1
			fi
			install_only="y"
			;;
		--remote-only)
			shift;
			warn_log "--remote-only is deprecated, ignoring (was non-operational previously)"
			;;
		--max-stddev)
			shift;
			if [ -n "$1" ]; then
				maxstddevpct="$1"
				shift;
			fi
			;;
		--max-failures)
			shift;
			if [ -n "$1" ]; then
				max_failures="$1"
				shift;
			fi
			;;
		--samples)
			shift;
			if [ -n "$1" ]; then
				nr_samples="$1"
				shift;
			fi
			;;
		--direct)
			shift;
			if [ -n "$1" ]; then
				direct=$1
				shift;
			fi
			;;
		--sync)
			shift;
			if [ -n "$1" ]; then
				sync=$1
				shift;
			fi
			;;
		-t|--test-types)
			shift;
			if [ -n "$1" ]; then
				test_types="$1"
				shift;
			fi
			;;
		-b|--block-sizes)
			shift;
			if [ -n "$1" ]; then
				block_sizes="$1"
				shift;
			fi
			;;
		-s|--file-size)
			shift;
			if [ -n "$1" ]; then
				file_size="$1"
				shift;
			fi
			;;
		--ramptime)
			shift;
			if [ -n "$1" ]; then
				ramptime="$1"
				shift;
			fi
			;;
		--rate-iops)
			shift;
			if [ -n "$1" ]; then
				rate_iops="$1"
				shift;
			fi
			;;
		-r|--runtime)
			shift;
			if [ -n "$1" ]; then
				runtime="$1"
				shift;
			fi
			;;
		-c|--clients)
			shift;
			if [ ! -z "$client_file" ] ;then
				printf -- "${script_name} $*\n"
				printf -- "\n"
				printf -- "\t-c|--clients and --client-file are mutually exclusive\n\n"
				usage
				exit 1
			fi
			if [ -n "$1" ]; then
				clients="$1"
				shift;
			fi
			;;
		--client-file)
			shift;
			if [ ! -z "$clients" ] ;then
				printf -- "${script_name} $*\n"
				printf -- "\n"
				printf -- "\t--client-file and -c|--clients are mutually exclusive\n\n"
				usage
				exit 1
			fi
			client_file=$1
			shift;
			if [ ! -e "${client_file}" ]; then
				printf -- "${script_name} $*\n"
				printf -- "\n"
				printf -- "\tSpecified client file, ${client_file}, does not exist\n\n"
				usage
				exit 1
			fi
			if [[ "$client_file" != /* ]] ;then
				# make it absolute
				client_file=$PWD/$client_file
			fi
			while read line; do
				clients="$clients,$line"
			done < ${client_file}
			# Remove the leading comma since $clients starts off empty
			clients=`echo $clients | sed -e 's/^,//'`
			;;
		-d|--targets)
			shift;
			if [ -n "$1" ]; then
				targets="$1"
				shift;
			fi
			;;
		-j|--job-mode)
			shift;
			if [ -n "$1" ]; then
				job_mode="$1"
				shift;
			fi
			;;
		--config)
			shift;
			if [ -n "$1" ]; then
				config="$1"
				shift;
			fi
			;;
		--ioengine)
			shift;
			if [ -n "$1" ]; then
				ioengine="$1"
				shift;
			fi
			;;
		--iodepth)
			shift;
			if [ -n "$1" ]; then
				iodepth="$1"
				shift;
			fi
			;;
		--tool-group)
			shift;
			if [ -n "$1" ]; then
				tool_group="$1"
				shift;
			fi
			;;
		--postprocess-only)
			shift;
			if [ -n "$1" ]; then
				postprocess_only="$1"
				shift;
				if [[ "$postprocess_only" != "n" && "$install_only" == "y" ]]; then
					printf -- "${script_name} $*\n"
					printf -- "\n"
					printf -- "\t\"--postprocess-only ${postprocess_only}\" not compatible with specified --install option\n\n"
					usage
					exit 1
				fi
			fi
			;;
		--run-dir)
			shift;
			if [ -n "$1" ]; then
				run_dir="$1"
				shift;
			fi
			;;
		--numjobs)
			shift;
			if [ -n "$1" ]; then
				numjobs="$1"
				shift;
			fi
			;;
		--job-file)
			shift;
			if [ -n "$1" ]; then
				job_file="$1"
				shift;

			fi
			;;
		--pre-iteration-script)
			shift;
			if [ -n "$1" ]; then
				pre_iteration_script="$1"
				if [ ! -x $pre_iteration_script ]; then
					printf "ERROR: $pre_iteration_script must be executable\n"
					exit 1
				fi
				shift;
			fi
			;;
		--histogram-interval-sec)
			shift;
			histogram_interval_sec="$1"
			shift
			;;
			# histogram-interval-msec is only kept for backwards compatibility
			# it is deprecated and is not documented in the --help output
		--histogram-interval-msec)
			shift;
			(( histogram_interval_sec = $1 / 1000 ))
			shift
			;;
		--sysinfo)
			shift;
			if [ -n "$1" ]; then
				sysinfo="$1"
				shift;
			fi
			;;
		--)
			shift;
			break;
			;;
		*)
			echo "what's this? [$1]"
			shift;
			break;
			;;
		esac
	done
	verify_common_bench_script_options $tool_group $sysinfo

	if [[ ${job_mode} != "serial" && ${job_mode} != "concurrent" ]]; then
		error_log "Unsupported --job-mode=${job_mode} encountered"
		usage
		exit 1
	fi

	local tt=""
	space_sep_vtt=$(echo $supported_test_types | sed -e s/,/" "/g)
	for tt in $(echo "$test_types" | sed -e s/,/" "/g); do
		local found=0
		for vtt in ${space_sep_vtt}; do
			if [[ "${tt}" == "${vtt}" ]]; then
				found=1
				break;
			fi
		done
		if [[ ${found} -ne 1 ]]; then
			error_log "Unsupported --test-types=${test_types} encountered"
			usage
			exit 1
		fi
	done

	if [ "$postprocess_only" = "n" ]; then
		benchmark_fullname="${benchmark}_${config}_${date_suffix}"
		benchmark_run_dir="$pbench_run/${benchmark_fullname}"
	else
		if [ -z "$run_dir" ]; then
			error_log "Please specify a directory if postprocessing an existing result (--run-dir=)"
			exit 1
		fi
		benchmark_fullname="$(basename $run_dir)"
		benchmark_run_dir="$run_dir"
	fi
}

function record_iteration {
	local count=${1}
	local test_type=${2}
	local block_size=${3}
	local dev=${4}
	local iteration=${5}

	local mdlog=${benchmark_run_dir}/metadata.log
	echo ${iteration} >> ${benchmark_iterations}
	echo ${count} | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_number
	echo ${test_type} | pbench-add-metalog-option ${mdlog} iterations/${iteration} test_type
	echo ${block_size} | pbench-add-metalog-option ${mdlog} iterations/${iteration} block_size_KiB
	echo ${dev} | pbench-add-metalog-option ${mdlog} iterations/${iteration} dev
	echo ${iteration} | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_name
}

# Ensure the right version of the benchmark is installed
function fio_install() {
	if check_install_rpm ${benchmark_rpm} ${ver}; then
		debug_log "[${script_name}] ${benchmark_rpm}-${ver} is installed"
	else
		debug_log "[${script_name}] ${benchmark_rpm}-${ver} is not installed, exiting"
		exit 1
	fi
	if [ ! -z "$clients" ] ; then
		debug_log "verifying clients have fio installed"
		echo "verifying clients have fio installed"
		for client in `echo $clients | sed -e s/,/" "/g`; do
			ssh $ssh_opts $client ${pbench_install_dir}/bench-scripts/$script_name --install &
		done
		# FIXME - save the pid and wait for each pid to ensure it returns with success
		wait
	fi
}

# Make sure this devices exists
function fio_device_check() {
	local devs=$1
	local clients=$2
	local dev=""
	local client=""
	local rc=0

	debug_log "fio_device_check(devs=\"${devs}\", clients=\"${clients}\")"
	for dev in `echo $devs | sed -e s/,/" "/g`; do
		if echo $dev | grep -q "^/dev/"; then
			if [ ! -z "$clients" ]; then
				for client in `echo $clients | sed -e s/,/" "/g`; do
					debug_log "checking to see if $dev exists on client $client"
					ssh $ssh_opts $client "if [ -L $dev ]; then dev=`dirname $dev`/`readlink $dev`; fi; test -b $dev" || rc=1
				done
			else
				if [ -L $dev ]; then dev=`dirname $dev`/`readlink $dev`; fi; test -b $dev || rc=1
			fi
			if [ $rc -eq 1 ]; then
				error_log "At least one client did not have block device $dev, exiting"
				exit 1
			fi
		fi
	done
}

function fio_create_jobfile() {
	local fio_job_file="${13}"
	local fio_job_file_path=$(dirname "${fio_job_file}")
	mkdir -p "${fio_job_file_path}"
	if [ ${?} -ne 0 ]; then
		error_log "Failed to create path to job file \"${fio_job_file}\""
		exit 1
	fi
	"${script_path}/templates/make-fio-jobfile.py" -j ${job_file} \
		-rw="${1}" \
		-ioengine="${2}" \
		-bs="$(printf %sk ${3})" \
		-iodepth="${4}" \
		-direct="${5}" \
		-sync="${6}" \
		-runtime="${7}" \
		-ramptime="${8}" \
		-size="${9}" \
		-rate_iops="${10}" \
		-log_hist_msec="${11}" \
		-targets $(echo "${12}" | sed -e 's/,/ /g') \
		-numjobs="${14}" \
		| sed -e 's/ = /=/g' > ${fio_job_file}

	if [ ${?} -ne 0 ]; then
		error_log "Failed to create job file \"${fio_job_file}\""
		exit 1
	fi
}

function does_wildcard_exist() {
	local fn_pat=$1
	local found=1
	for fn in ${fn_pat}; do
		if [[ -e ${fn} ]]; then
			found=0
			break
		fi
	done
	return ${found}
}

function fio_run_job() {
	local iteration="$1"
	local benchmark_results_dir="$2"
	local fio_job_file="$3"
	local bench_opts="--output-format=json $fio_job_file"
	local tgts="$4"
	local clients="$5"

	mkdir -p $benchmark_results_dir/clients
	if [[ ! -d $benchmark_results_dir/clients ]]; then
		error_log "Failed to create results directory hierarchy: $benchmark_results_dir/clients"
		exit 1
	fi

	echo "running fio job: $fio_job_file ($(basename $benchmark_results_dir))"

	if [ ! -z "$clients" ]; then
		debug_log "creating directories on the clients"
		for client in `echo $clients | sed -e s/,/" "/g`; do
			ssh $ssh_opts $client mkdir -p $benchmark_results_dir &
		done
		wait
		debug_log "opening port 8765 on firewall on the clients"
		for client in `echo $clients | sed -e s/,/" "/g`; do
			ssh $ssh_opts $client "firewall-cmd --add-port=$fio_server_port/tcp >/dev/null" &
		done
		wait
		debug_log "killing any old fio process on the clients"
		for client in `echo $clients | sed -e s/,/" "/g`; do
			ssh $ssh_opts $client "killall fio >/dev/null 2>&1" &
		done
		wait
		debug_log "starting new fio process on the clients"
		for client in `echo $clients | sed -e s/,/" "/g`; do
			ssh $ssh_opts $client "pushd $benchmark_results_dir >/dev/null; screen -dmS fio-server bash -c ''$benchmark_bin' --server 2>&1 >client-result.txt'"
		done
		wait
		debug_log "waiting for fio process(server) to start on clients"
		for client in `echo $clients | sed -e s/,/" "/g`; do
			timeout --kill-after=1 60 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' $client $fio_server_port
			if [[ $? -eq 124 ]]; then
				error_log "fio process took more than 60 seconds to start in client $client, failing"
				exit 1
			fi
		done
	fi

	local client_opts=""

	if [ ! -z "$clients" ]; then
		# If a client-file parameter was passed, then clients is
		# already initialized to its contents.
		local max_jobs=0
		local nclients=0

		if [ ! -z $numjobs ]; then
			# so that a separate filename will be generated
			# for each (host, job-number) pair
			let max_jobs=$numjobs
		else
			for t in `echo $tgts | sed -e s/,/" "/g` ; do
				let max_jobs=$max_jobs+1
			done
		fi

		let nclients=`wc -l < $_client_file`
		let max_jobs=$max_jobs*$nclients
		client_opts="$client_opts --client=$_client_file --max-jobs=$max_jobs"
	fi

	# Certain test preparation steps such as cache dropping can be a bit
	# hard on the system, give it a few seconds before actually starting
	# test by putting this before pbench-start-tools.
	if [ -n "$pre_iteration_script" ] ; then
		printf "running pre-iteration-script command: $pre_iteration_script\n"
		eval "$pre_iteration_script"
	fi
	pbench-start-tools --group=$tool_group --dir=$benchmark_results_dir

	# create a command file and keep it with the results for debugging later, or user can run outside of pbench
	echo "$benchmark_bin $client_opts $bench_opts" >$benchmark_results_dir/fio.cmd
	chmod +x $benchmark_results_dir/fio.cmd
	debug_log "$benchmark: Going to run [$benchmark_bin $bench_opts $client_opts]"
	pushd $benchmark_results_dir >/dev/null
	$benchmark_results_dir/fio.cmd >$benchmark_results_dir/fio-result.txt
	popd >/dev/null

	pbench-stop-tools --group=$tool_group --dir=$benchmark_results_dir
	pbench-postprocess-tools --group=$tool_group --dir=$benchmark_results_dir
 
	local histogram_params="--time-quantum ${histogram_interval_sec} --percentiles 0 50 90 95 99 99.5 100"
	local histogram_cmd="fio-histo-log-pctiles.py ${histogram_params} -- "

	# First move all fio log files to the appropriate client directory.
	if [ ! -z "${clients}" ]; then
		local client
		for client in `echo ${clients} | sed -e s/,/" "/g`; do
			mkdir -p ${benchmark_results_dir}/clients/${client}
			/bin/mv ${benchmark_results_dir}/*.log.${client} ${benchmark_results_dir}/clients/${client}/ 2> /dev/null
 			# remove the trailing client name in the log file names
			pushd ${benchmark_results_dir}/clients/${client} >/dev/null
			for i in $(/bin/ls | grep log); do
				mv ${i} $(echo ${i} | sed -e s/\.${client}//)
 			done
 			popd >/dev/null
		done
	else
		mkdir -p ${benchmark_results_dir}/clients/localhost/
		/bin/mv ${benchmark_results_dir}/*.log ${benchmark_results_dir}/clients/localhost/ 2> /dev/null
	fi

	if grep -q "^log_hist_msec" ${fio_job_file}; then
		local client
		local _clients_list
		if [ ! -z "${clients}" ]; then
			_clients_list="$(echo ${clients} | sed -e s/,/" "/g)"
		else
			_clients_list="localhost"
		fi
		for client in ${_clients_list}; do
 			# Process the histograms
 			# Determine histogram interval based on value used in job file.
 			#my $log_interval = `cat $dir/fio.job | grep "^log_hist_msec"`;
 			#if [ ! $log_interval =~ /^\s*$/ ]; then
			local log_dir="${benchmark_results_dir}/clients/${client}"
			does_wildcard_exist ${log_dir}/'fio_clat_hist.*.log*'
			if [[ ${?} -eq 0 ]]; then
				local hist_dir="${log_dir}/hist"
				mkdir -p ${hist_dir}
				# compute per-client latency percentiles
				${histogram_cmd} ${log_dir}/fio_clat_hist.*.log* >${hist_dir}/hist.csv
				${script_path}/postprocess/${benchmark}-postprocess-viz.py --job-file=${fio_job_file} ${histogram_params} -- ${hist_dir}
 			else
				warn_log "log_hist_msec specified in job file but failed to find any log files (.../clients/${client}/fio_clat_hist.*.log*) to process for histograms"
 			fi
		done
		does_wildcard_exist ${benchmark_results_dir}/clients/'*/fio_clat_hist.*.log*'
		if [[ ${?} -eq 0 ]]; then
			local hist_dir="${benchmark_results_dir}/hist"
			mkdir -p ${hist_dir}
			# compute cluster-wide latency percentiles
			${histogram_cmd} ${benchmark_results_dir}/clients/*/fio_clat_hist.*.log* >${hist_dir}/hist.csv
			${script_path}/postprocess/${benchmark}-postprocess-viz.py --job-file=${fio_job_file} ${histogram_params} -- ${benchmark_results_dir}/hist
		else
			warn_log "log_hist_msec specified in job file but failed to find any log files (.../clients/*/fio_clat_hist.*.log*) to process for histograms"
 		fi
 	fi
 
	echo "fio job complete"
}



# Run the benchmark and start/stop perf analysis tools
function fio_run_benchmark() {
	(( histogram_interval_msec = histogram_interval_sec * 1000 ))

	if [[ ! -z "$client_file" ]] ;then
		# Copy the clients file given on the command line into the run
		# directory to keep it with the run.
		cp $client_file $benchmark_run_dir/fio-client.file
		_client_file=$benchmark_run_dir/fio-client.file
	else
		if [[ ! -z "$clients" ]]; then
			# Record the -c|--clients list into the
			# fio-client.file.  This makes the fio command shorter
			# by always using a client file regardless of how
			# clients were specified.  We only need to save the
			# client list once in the benchmark run directory.
			_client_file=$benchmark_run_dir/fio-client.file
			for client in `echo $clients | sed -e s/,/" "/g`; do
				echo $client
			done > $_client_file
		fi
	fi

	local count=1
	if [ "$job_mode" = "serial" ]; then
		# if each target is separated by a space, there will be one job for each in next for loop
		targets=`echo $targets | sed -e s/,/" "/g`
	fi
	typeset -i ntargets=$(echo $targets | wc -w)
	typeset -i ntesttypes=$(echo $test_types | sed -e 's/,/ /g' | wc -w)
	typeset -i nblocksizes=$(echo $block_sizes | sed -e 's/,/ /g' | wc -w)
	typeset -i total_iterations=$ntargets*$ntesttypes*$nblocksizes
	for dev in $targets; do
		for test_type in `echo $test_types | sed -e s/,/" "/g`; do
			for block_size in `echo $block_sizes | sed -e s/,/" "/g`; do
				job_num=1
				iteration="${count}-${test_type}-${block_size}KiB"
				if [ "$job_mode" = "serial" ]; then
					dev_short_name="`basename $dev`"
					# easier to identify what job used what device when having 1 job per device
					iteration="$iteration-${dev_short_name}"
				fi
				record_iteration ${count} ${test_type} ${block_size} ${dev} ${iteration}
				iteration_dir=$benchmark_run_dir/$iteration
				result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough
				failures=0
				while [[ $(echo "if (${result_stddevpct} >= ${maxstddevpct}) 1 else 0" | bc) -eq 1 ]]; do
					if [[ $failures -gt 0 ]]; then
						echo "Restarting iteration $iteration ($count of $total_iterations)"
						log "Restarting iteration $iteration ($count of $total_iterations)"
					fi
					if [ "$postprocess_only" = "n" ]; then
						mkdir -p $iteration_dir
					else
						if [ ! -e $iteration_dir ]; then
							# if the iteration dir does not exist, look for a failed result directory or archive
							fail_nr=$failures
							((fail_nr++))
							fail_tag="-fail$fail_nr"
							failed_iteration_dir="$iteration_dir$fail_tag"
							if [ -e $failed_iteration_dir ]; then
								mv $failed_iteration_dir $iteration_dir || exit 1
							else
								failed_iteration_archive="$iteration_dir$fail_tag.tar.xz"
								if [ -e $failed_iteration_archive ]; then
									echo "using $failed_iteration_archive as $iteration_dir"
									tar -C $benchmark_run_dir -J -x -f $failed_iteration_archive || exit 1
									mv $failed_iteration_dir $iteration_dir || exit 1
								else
									echo "Could not find $iteration_dir, $failed_iteration_dir, or $failed_iteration_archive"
								fi
							fi
						fi
					fi
					if [ -e $iteration_dir ]; then
						fio_job_file="$iteration_dir/fio.job"
						local _action=""
						if [ "$postprocess_only" = "n" ]; then
							_action="Created"
							#             ARG   1            2           3             4          5         6       7          8           9            10           11                         12     13              14
							fio_create_jobfile "$test_type" "$ioengine" "$block_size" "$iodepth" "$direct" "$sync" "$runtime" "$ramptime" "$file_size" "$rate_iops" "$histogram_interval_msec" "$dev" "$fio_job_file" "$numjobs"
						else
							_action="Found"
						fi
						echo "${_action} the following job file (${fio_job_file}):"
						cat ${fio_job_file}

						iteration_failed=0
						sample_failed=0
						for sample in `seq 1 $nr_samples`; do
							# each attempt at a test config requires multiple samples to get stddev
							benchmark_results_dir="$iteration_dir/sample$sample"
							if [ "$postprocess_only" = "n" ]; then
								if [ -e $benchmark_results_dir ]; then
									# FIXME - Shouldn't we error and exit out instead of removing data?
									/bin/rm -rf $benchmark_results_dir
								fi
								mkdir -p $benchmark_results_dir
								fio_run_job "$iteration" "$benchmark_results_dir" "$fio_job_file" "$dev" "$clients"
							else
								# we are only postprocessing, so we might have to untar an existing result
								if [ ! -e $benchmark_results_dir ]; then
									if [ -e ${iteration_dir}/sample${sample}.tar.xz ]; then
										pushd $iteration_dir >/dev/null
										tar Jxf sample$sample.tar.xz && /bin/rm sample$sample.tar.xz
										popd >/dev/null
									else
										echo "sample $sample missing.  There should be $nr_samples samples"
										break 2
									fi
								fi
							fi
							debug_log "post-processing fio result"
							echo "$script_path/postprocess/$benchmark-postprocess \"$benchmark_results_dir\" \"$tool_label_pattern\" \"$tool_group\"" >"$benchmark_results_dir/$benchmark-postprocess.cmd"
							chmod +x "$benchmark_results_dir/$benchmark-postprocess.cmd"
							$benchmark_results_dir/$benchmark-postprocess.cmd
							rc=$?
							# if for any reason the benchmark postprocessing script fails, consider this a failure to get a sample
							if [ $rc -ne 0 ]; then
								debug_log "failed: $script_path/postprocess/$benchmark-postprocess $benchmark_results_dir $iteration $tool_group"
								sample_failed=1
							fi
							if [ $sample_failed -eq 1 ]; then
								# we need all samples to be good, so bust out of testing samples now
								break 2
							fi
						done
					fi
					echo "$script_path/postprocess/process-iteration-samples \"$iteration_dir\" \"$primary_metric\" \"$maxstddevpct\" \"$failures\" \"$max_failures\" \"$tar_nonref_data\" \"$keep_failed_tool_data\"" >"$iteration_dir/process-iteration-samples.cmd"
					chmod +x "$iteration_dir/process-iteration-samples.cmd"
					$iteration_dir/process-iteration-samples.cmd
					fail=$?
					if [ $fail -ne  0 ]; then
						((failures++))
					fi
					if [ $fail -eq 0 -o $failures -ge $max_failures ]; then
						break 1
					fi
				done
			let count=$count+1 # now we can move to the next iteration
			done
		done
	done
}

fio_process_options "$@"
if [ "$postprocess_only" = "n" ]; then
	fio_install
	if [ "$install_only" = "y" ]; then
		exit 0
	fi
	fio_device_check "$targets" "$clients"
	mkdir -p $benchmark_run_dir/.running
	if [[ $? -ne 0 ]]; then
		error_log "Unable to create $benchmark_run_dir path and/or .running marker directory"
		exit 1
	fi
fi

benchmark_iterations="${benchmark_run_dir}/.iterations"
> ${benchmark_iterations}

export benchmark config
pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg
pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg
# on abnormal exit, make sure that the metadata log exists and is complete.
trap "pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir int" INT QUIT

fio_run_benchmark
echo "$script_path/postprocess/generate-benchmark-summary \"$benchmark\" \"$orig_cmd\" \"$benchmark_run_dir\"" >"$benchmark_run_dir/generate-benchmark-summary.cmd"
chmod +x "$benchmark_run_dir/generate-benchmark-summary.cmd"
$benchmark_run_dir/generate-benchmark-summary.cmd
pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end
pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir  --sysinfo=$sysinfo end

rmdir $benchmark_run_dir/.running
