#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; tab-width: 8 -*-

# This is a script to run the uperf benchmark
# Author: Andrew Theurer

# This script attempts to automate potentially a very large number of tests for uperf
# By default, is runs 96 different tests and can take several hours.
# To limit the number of tests, use the --protocols --test-types --instances options to reduce the number of tests

# This script will take multiple samples of the same test type and try to achieve a standard deviation of <3%
#
# This script will repeat a test type 6 times in order to try to achieve target stddev.
# If a run (with several samples) fails the stddev, its directory is appended with -fail
#
# This script will also generate a "summary-results.txt" with a table of all
# results, efficiency, and other stats.
pbench_cmd="$@"

script_path=`dirname $0`
script_name=`basename $0`
pbench_bin="`cd ${script_path}/..; /bin/pwd`"

# source the base script
. "$pbench_bin"/base

export benchmark="uperf"

# Every bench-script follows a similar sequence:
# 1) process bench script arguments
# 2) ensure the right version of the benchmark is installed
# 3) gather pre-run state
# 4) run the benchmark and start/stop perf analysis tools
# 5) gather post-run state
# 6) postprocess benchmark data
# 7) postprocess analysis tool data

# Defaults

export benchmark_run_dir=""
protocols="tcp,udp"
test_types="stream,maerts,bidirec,rr"
message_sizes="1,64,1024,16384" # in bytes
export config=""
instances="1,8,64"
nr_samples=5
maxstddevpct=5 # maximum allowable standard deviation in percent
max_failures=6 # after N failed attempts to hit below $maxstddevpct, move on to the nest test
runtime=60
mode="loopback"
clients_def=127.0.0.1
servers_def=${clients_def}
server_base_port=20000
port_number_gap=10
postprocess_only=n
log_response_times=n
start_iteration_num=1
tar_nonref_data="n"
keep_failed_tool_data="y"
orig_cmd="$*"
tool_group=default
tool_label_pattern="uperf-"
server_nodes=""
client_nodes=""
sysinfo="default"
do_pre_check_only=0

function usage {
	printf "\tThe following options are available:\n\n"
	printf -- "\t--tool-group=str\n"
	printf -- "\t-c str       --config=str               name of the test config (e.g. jumbo_frames_and_network_throughput)\n"
	printf -- "\t-t str[,str] --test-types=str[,str]     can be stream, maerts, bidirec, and/or rr (default $test_types)\n"
	printf -- "\t-r int       --runtime=int              test measurement period in seconds (default is $runtime)\n"
	printf -- "\t-m int[,int] --message-sizes=str[,str]  list of message sizes in bytes (default is $message_sizes)\n"
	printf -- "\t-p str[,str] --protocols=str[,str]      tcp and/or udp (default is $protocols)\n"
	printf -- "\t-i int[,int] --instances=int[,int]      list of number of uperf instances to run (default is $instances)\n"
	printf -- "\t-C str[,str] --client[s]=str[,str]      a list of one or more hostnames/IPs.  These systems will run the\n"
	printf    "\t\t\t\t   uperf client (drive the test).\n"
	printf    "\t\t\t\t   If this is omitted, the local system is the client.\n"
	printf    "\t\t\t\t   Note: the number of clients and servers must be the same!\n"
	printf    "\t\t\t\t   Clients and servers are paired according to the order in the list (first\n"
	printf    "\t\t\t\t   client pairs with first server, etc)\n"
	printf -- "\t-S str[,str] --server[s]=str[,str]      a list of one or more hostnames/IPs.  These systems will run the uperf\n"
	printf    "\t\t\t\t   server (listening for connections).\n"
	printf    "\t\t\t\t   If this is omitted, the server will listen on the local system\n"
	printf    "\t\t\t\t   loopback interface.\n"
	printf -- "\t--vsock-server[s]=str[,str]             a list of one or more AF_VSOCK hostnames, used instead of the server\n"
	printf -- "\t\t\t\t   hostnames, and directs uperf to use the AF_VSOCK protocol\n"
	printf -- "\t\t\t\t   The number of --vsock-servers provided must match the number of --servers\n"
	printf -- "\t--server-node[s]=str[,str]              An ordered list of server NUMA nodes which should be used for CPU binding\n"
	printf -- "\t--client-node[s]=str[,str]              An ordered list of client NUMA nodes which should be used for CPU binding\n"
	printf -- "\t\t\t\t   For both options above, the order must correspond with the --clients/--servers list\n"
	printf -- "\t\t\t\t   To omit a specific client/server from binding, use a value of -1.\n"
	printf -- "\t--samples=int              the number of times each different test is run (to compute average &\n"
	printf    "\t\t\t\t   standard deviations).\n"
	printf -- "\t--max-failures=int         the maximum number of failures to get below stddev.\n"
	printf -- "\t--max-stddev=int           the maximum percent stddev allowed to pass.\n"
	printf -- "\t--postprocess-only=y|n     don't run the benchmark, but postprocess data from previous test.\n"
	printf -- "\t--run-dir=str              optionally specify what directory should be used (usually only used\n"
	printf    "\t\t\t\t   if postprocess-only=y).\n"
	printf -- "\t--start-iteration-num=int  optionally skip the first (n-1) tests.\n"
	printf -- "\t--log-response-times=y|n   record the response time of every single operation.\n"
	printf -- "\t--tool-label-pattern=str   uperf will provide CPU and efficiency information for any tool directory\n"
	printf    "\t\t\t\t   with a \"^<pattern>\" in the name, provided \"sar\" is one of the\n"
	printf    "\t\t\t\t   registered tools.\n"
	printf    "\t\t\t\t   a default pattern, \"uperf-\" is used if none is provided.\n"
	printf    "\t\t\t\t   simply register your tools with \"--label=uperf-\$X\", and this script\n"
	printf    "\t\t\t\t   will generate CPU_uperf-\$X and Gbps/CPU_uperf-\$X or\n"
	printf    "\t\t\t\t   trans_sec/CPU-uperf-\$X for all tools which have that pattern as a\n"
	printf    "\t\t\t\t   prefix.  If you don't want to register your tools with \"uperf-\" as\n"
	printf    "\t\t\t\t   part of the label, just use --tool-label-pattern= to tell this script\n"
	printf    "\t\t\t\t   the prefix pattern to use for CPU information.\n"
	printf -- "\t--sysinfo=str,             str= comma separated values of sysinfo to be collected\n"
	printf -- "\t                                available: $(pbench-display-sysinfo-options)\n"
}

function gen_xml {
	local h=$1
	local proto=$2
	local runtime=$3
	local size=$4
	local nthr=$5
	local test_type=$6

	echo "<?xml version=\"1.0\"?>"
	echo "<profile name=\"$proto-$test_type-${size}B-${nthr}i\">"
	case $test_type in
	rr)
		echo "  <group nthreads=\"$nthr\">"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"connect\" options=\"remotehost=$h protocol=$proto\"/>"
		echo "    </transaction>"
		echo "    <transaction duration=\"$runtime\">"
		echo "      <flowop type=\"write\" options=\"size=$size\"/>"
		echo "      <flowop type=\"read\"  options=\"size=$size\"/>"
		echo "    </transaction>"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"disconnect\" />"
		echo "    </transaction>"
		echo "  </group>"
		;;
	stream|bidirec)
		echo "  <group nthreads=\"$nthr\">"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"connect\" options=\"remotehost=$h protocol=$proto\"/>"
		echo "    </transaction>"
		echo "    <transaction duration=\"$runtime\">"
		echo "      <flowop type=\"write\" options=\"count=16 size=$size\"/>"
		echo "    </transaction>"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"disconnect\" />"
		echo "    </transaction>"
		echo "  </group>"
		;;&
	maerts|bidirec)
		echo "  <group nthreads=\"$nthr\">"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"accept\" options=\"remotehost=$h protocol=$proto\"/>"
		echo "    </transaction>"
		echo "    <transaction duration=\"$runtime\">"
		echo "      <flowop type=\"read\" options=\"count=16 size=$size\"/>"
		echo "    </transaction>"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"disconnect\" />"
		echo "    </transaction>"
		echo "  </group>"
		;;
	esac
	echo "</profile>"
}

function stop_server {
	local server="${1}"
	local server_port="${2}"
	local message="${3}"

	local ss_output
	if pbench-is-local ${server}; then
		ss_output=$(ss -tlnp --vsock)
	else
		ss_output=$(ssh ${ssh_opts} ${server} ss -tlnp --vsock)
	fi
	local uperf_pids=$(sed -En -e "/:${server_port} / s/.*pid=([^,]+),.*/\1/p" <<< "${ss_output}" | sort -u)
	if [[ -n "${uperf_pids}" ]]; then
		if [[ ${message} -eq 1 ]]; then
			echo "found uperf pid(s) ${uperf_pids} on ${server}:${server_port}, killing"
		fi
		if pbench-is-local ${server}; then
			for pid in ${uperf_pids}; do kill ${pid}; done
		else
			ssh ${ssh_opts} ${server} "for pid in ${uperf_pids}; do kill \${pid}; done"
		fi
	fi
}

function local_pre_check {
	local _ver="${1}"
	local _match="${2}"

	local _stdout
	_stdout=$(${benchmark} -V 2>&1)
	if [[ ${?} -ne 0 ]]; then
		error_log "${script_name}: ${benchmark} failed reporting version: '${_stdout}'"
		return 1
	fi
	local ver_installed=$(awk 'NR==1 {print $3}' <<< ${_stdout})
	vercmp "${benchmark}" "${_match}" "${_ver}" "${ver_installed}"
	return ${?}
}

# Process options and arguments
opts=$(getopt -q -o C:c:hi:m:p:r:S:t: --longoptions "help,server-node:,server-nodes:,client-node:,client-nodes:,tool-label-pattern:,pre-check,start-iteration-num:,config:,instances:,test-types:,runtime:,message-sizes:,protocols:,samples:,client:,clients:,server:,servers:,max-stddev:,max-failures:,log-response-times:,postprocess-only:,run-dir:,tool-group:,sysinfo:,vsock-server:,vsock-servers:" -n "getopt.sh" -- "$@")
if [ $? -ne 0 ]; then
	printf -- "${script_name} $*\n"
	printf -- "\n"
	printf -- "\tunrecognized option specified\n\n"
	usage >&2
	exit 1
fi
eval set -- "$opts"
while true; do
	arg="${1}"
	shift
	case "${arg}" in
		--pre-check)
			if [[ ${postprocess_only} == "y" ]]; then
				error_log "[${script_name}] '--pre-check not compatible with previously specified '--postprocess-only=y' option"
				usage >&2
				exit 1
			fi
			do_pre_check_only=1
		;;
		-h|--help)
			usage
			exit 0
		;;
		--tool-label-pattern)
			if [ -n "$1" ]; then
				tool_label_pattern="$1"
				shift
			fi
		;;
		--postprocess-only)
			if [ -n "$1" ]; then
				postprocess_only="$1"
				if [[ ${do_pre_check_only} -eq 1 ]]; then
					error_log "[${script_name}] '--postprocess-only=y' not compatible with previously specified '--pre-check' option"
					usage >&2
					exit 1
				fi
				shift
			fi
		;;
		--run-dir)
			if [ -n "$1" ]; then
				benchmark_run_dir="$1"
				shift
			fi
		;;
		--max-stddev)
			if [ -n "$1" ]; then
				maxstddevpct="$1"
				shift
			fi
		;;
		--max-failures)
			if [ -n "$1" ]; then
				max_failures="$1"
				shift
			fi
		;;
		--samples)
			if [ -n "$1" ]; then
				nr_samples="$1"
				shift
			fi
		;;
		-i|--instances)
			if [ -n "$1" ]; then
				instances="$1"
				shift
			fi
		;;
		-t|--test-types)
			if [ -n "$1" ]; then
				test_types="$1"
				shift
			fi
		;;
		--tool-group)
			if [ -n "$1" ]; then
				tool_group="$1"
				shift
			fi
		;;
		-m|--message-sizes)
			if [ -n "$1" ]; then
				message_sizes="$1"
				shift
			fi
		;;
		-r|--runtime)
			if [ -n "$1" ]; then
				runtime="$1"
				shift
			fi
		;;
		--log-response-times)
			if [ -n "$1" ]; then
				log_response_times="$1"
				shift
			fi
		;;
		-p|--protocols)
			if [ -n "$1" ]; then
				protocols="$1"
				shift
			fi
		;;
		-C|--client|--clients)
			if [ -n "$1" ]; then
				clients="$1"
				shift
			fi
		;;
		-S|--server|--servers)
			if [ -n "$1" ]; then
				servers="$1"
				shift
			fi
		;;
		--vsock-server|--vsock-servers)
			if [ -n "$1" ]; then
				vsock_servers="$1"
				shift
			fi
		;;
		-c|--config)
			if [ -n "$1" ]; then
				config="$1"
				shift
			fi
		;;
		--start-iteration-num)
			if [ -n "$1" ]; then
				start_iteration_num=$1
				shift
			fi
		;;
		--client-node|--client-nodes)
			if [ -n "$1" ]; then
				client_nodes="$1"
				shift
			fi
		;;
		--server-node|--server-nodes)
			if [ -n "$1" ]; then
				server_nodes="$1"
				shift
			fi
		;;
		--sysinfo)
			if [ -n "$1" ]; then
				sysinfo="$1"
				shift
			fi
		;;
		--)
			break
		;;
		*)
			error_log "[$script_name] bad option, \"${arg} ${1}\""
			usage >&2
			exit 1
		;;
	esac
done

if [[ ${do_pre_check_only} -eq 1 ]]; then
	local_pre_check ${@}
	exit ${?}
fi

verify_common_bench_script_options $tool_group $sysinfo

clients=${clients:-${clients_def}}
servers=${servers:-${servers_def}}

if [[ "${postprocess_only}" != "y" ]]; then
	ver="$(pbench-config version ${benchmark})"
	if [[ -z "${ver}" ]]; then
		error_log "${script_name}: 'version' is missing in the [${benchmark}] section of the config file"
		exit 1
	fi
	match="$(pbench-config match ${benchmark})"
	if [[ -z "${match}" ]]; then
		error_log "${script_name}: 'match' is missing in the [${benchmark}] section of the config file"
		exit 1
	fi

	# Boolean value recording if local clients or servers are specified.
	let check_local=0

	# Verify that the number of clients, servers, and optional vsock servers
	# match.
	let c_cnt=0
	for client in ${clients//,/ }; do
		(( c_cnt++ ))
		if pbench-is-local "${client}"; then
			check_local=1
		fi
	done
	if [[ ${c_cnt} -eq 0 ]]; then
		error_log "[${script_name}] INTERNAL ERROR - no clients found"
		exit 1
	fi
	let s_cnt=0
	for server in ${servers//,/ }; do
		(( s_cnt++ ))
		if pbench-is-local "${server}"; then
			check_local=1
		fi
	done
	if [[ ${s_cnt} -eq 0 ]]; then
		error_log "[${script_name}] INTERNAL ERROR - no servers found"
		exit 1
	fi
	if [ ${c_cnt} -ne ${s_cnt} ]; then
		error_log "Number of clients and servers specified on command line must match"
		error_log "    clients(${c_cnt}): ${clients}"
		error_log "    servers(${s_cnt}): ${servers}"
		exit 1
	fi
	let has_vsock=0
	for protocol in ${protocols//,/ }; do
		if [[ ${protocol} == "vsock" ]]; then
			has_vsock=1
		fi
	done
	let v_cnt=0
	for vserver in ${vsock_servers//,/ }; do
		(( v_cnt++ ))
	done
	if [[ ${has_vsock} -eq 1 ]]; then
		if [[ ${v_cnt} -ne ${s_cnt} ]]; then
			error_log "When using the 'vsock' protocol, servers and vsock-servers must match"
			error_log "    servers(${s_cnt}): ${servers}"
			error_log "    vsock-servers(${v_cnt}): ${vsock_servers}"
			exit 1
		fi
	elif [[ ${v_cnt} -gt 0 ]]; then
		error_log "Do not provide vsock-servers without using the vsock protocol"
		exit 1
	fi

	if [[ ${check_local} -eq 1 ]]; then
		# At least one client or server is a local host, so perform the
		# pre-check.
		local_pre_check "${ver}" "${match}"
		res=${?}
		if [[ ${res} -ne 0 ]]; then
			error_log "[${script_name}] The local pre-check failed"
			exit ${res}
		fi
	fi

	let err_cnt=0
	err_clients=""
	for client in ${clients//,/ }; do
		if ! pbench-is-local "${client}"; then
			debug_log "performing uperf pre-checks on client ${client}"
			ssh ${ssh_opts} ${client} pbench-uperf --pre-check "${ver}" "${match}"
			if [ ${?} -ne 0 ]; then
				(( err_cnt++ ))
				err_clients+=" ${client}"
			fi
		fi
	done
	err_servers=""
	for server in ${servers//,/ }; do
		if ! pbench-is-local "${server}"; then
			debug_log "performing uperf pre-checks on server ${server}"
			ssh ${ssh_opts} ${server} pbench-uperf --pre-check "${ver}" "${match}"
			if [ ${?} -ne 0 ]; then
				let err_cnt=err_cnt+1
				err_servers+=" ${server}"
			fi
		fi
	done
	if [[ ${err_cnt} -gt 0 ]]; then
		error_log "[${script_name}] The following pre-checks on clients and servers failed:"
		error_log "    clients: ${err_clients}"
		error_log "    servers: ${err_servers}"
		exit 1
	fi
fi

# runtime is padded by 6 seconds to allow 3 seconds ramp up and ramp down
runtime_padded=$((${runtime}+6))

###
# At this point all parameter checking has been done.
###

if [[ -z "$benchmark_run_dir" ]]; then
	# We don't have an explicit run directory, construct one
	benchmark_fullname="${benchmark}_${config}_${date_suffix}"
	benchmark_run_dir="$pbench_run/${benchmark_fullname}"
else
	# We have an explicit run directory provided by --run-dir, so warn
	# the user if they also used --config
	if [[ ! -z "$config" ]]; then
		warn_log "[$script_name] ignoring --config=\"$config\" in favor of --rundir=\"$benchmark_run_dir\""
	fi
	benchmark_fullname=$(basename $benchmark_run_dir)
fi
# we'll record the iterations in this file
benchmark_iterations="${benchmark_run_dir}/.iterations"
mdlog=${benchmark_run_dir}/metadata.log

function record_iteration {
	local count=$1
	local protocol=$2
	local test_type=$3
	local message_size=$4
	local instance=$5
	local iteration=$6

	echo ${iteration} >> ${benchmark_iterations}
	echo $count | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_number
	echo $protocol | pbench-add-metalog-option ${mdlog} iterations/${iteration} protocol
	echo $test_type | pbench-add-metalog-option ${mdlog} iterations/${iteration} test_type
	echo $message_size | pbench-add-metalog-option ${mdlog} iterations/${iteration} message_size_bytes
	echo $instance | pbench-add-metalog-option ${mdlog} iterations/${iteration} instances
	echo $iteration | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_name
}

mkdir -p $benchmark_run_dir/.running

# now that the benchmark_run_dir directory exists, we can initialize the iterations file
> ${benchmark_iterations}

# save a copy of the command, in case the test needs to be reproduced or post-processed again
echo "$script_name $pbench_cmd" >$benchmark_run_dir/$script_name.cmd
chmod +x $benchmark_run_dir/$script_name.cmd

## Run the benchmark and start/stop perf analysis tools

total_iterations=0
for protocol in ${protocols//,/ }; do
	for test_type in ${test_types//,/ }; do
		for message_size in ${message_sizes//,/ }; do
			for instance in ${instances//,/ }; do
				((total_iterations++))
			done
		done
	done
done

if [[ $postprocess_only == "n" ]]; then
	# Start the tool meisters on each registered local/remote host
	pbench-tool-meister-start --sysinfo="${sysinfo}" "${tool_group}"
	if [[ ${?} != 0 ]]; then
		error_log "[${script_name}]: failed to start the tool meisters."
		exit 1
	fi

	trap "interrupt" INT QUIT TERM
fi

# start the server processes
count=1
for protocol in ${protocols//,/ }; do
	for test_type in ${test_types//,/ }; do
		case $test_type in
			rr)
			primary_metric="trans_sec"
			;;
			stream|maerts|bidirec)
			primary_metric="Gb_sec"
			;;
			*)
			error_log "$script_name: test type \"$test_type\" is not suported"
			continue
			;;
		esac
		for message_size in ${message_sizes//,/ }; do
			for instance in ${instances//,/ }; do
				if [ $count -ge $start_iteration_num ]; then
					echo "Starting iteration $iteration ($count of $total_iterations)"
					log "Starting iteration $iteration ($count of $total_iterations)"
					iteration="${count}-${protocol}_${test_type}-${message_size}B-${instance}i"
					record_iteration ${count} ${protocol} ${test_type} ${message_size} ${instance} ${iteration}
					iteration_dir="$benchmark_run_dir/$iteration"
					result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough
					failures=0
					while [[ $(echo "if (${result_stddevpct} >= ${maxstddevpct}) 1 else 0" | bc) -eq 1 ]]; do
						if [[ $failures -gt 0 ]]; then
							echo "Restarting iteration $iteration ($count of $total_iterations)"
							log "Restarting iteration $iteration ($count of $total_iterations)"
						fi
						if [ $postprocess_only == "n" ]; then
							mkdir -p $iteration_dir
						fi
						# each attempt at a test config requires multiple samples to get stddev
						for sample in `seq 1 $nr_samples`; do
							sample_name="sample${sample}"
							benchmark_results_dir="${iteration_dir}/${sample_name}"
							if [ "$postprocess_only" != "y" ]; then
								mkdir -p $benchmark_results_dir
								server_nr=1
								# the following loop does all of the pre-benchmark execution work
								for server in ${servers//,/ }; do
									client=`echo ${clients} | cut -d, -f$server_nr`
									vsock_server=`echo $vsock_servers | cut -d, -f$server_nr`
									uperf_server="${vsock_server:-${server}}"
									server_port=`echo "$port_number_gap * $server_nr + $server_base_port" | bc`
									uperf_identifier="client::$client-server::$server:$server_port"

									benchmark_client_cmd_file="$iteration_dir/$uperf_identifier--client_start.sh"
									benchmark_server_cmd_file="$iteration_dir/$uperf_identifier--server_start.sh"
									result_file="$benchmark_results_dir/$uperf_identifier--client_output.txt"
									server_log="$benchmark_results_dir/$uperf_identifier--server.log"

									xml_file="$iteration_dir/$uperf_identifier--test_config.xml"
									gen_xml $uperf_server $protocol ${runtime_padded}s $message_size $instance $test_type >$xml_file

									# construct the server command
									if [ "$protocol" == "vsock" ]; then
										# vsock protocol requires socket-type to be specified
										vsock_extra_opts=" -S vsock"
									fi
									benchmark_server_cmd="${benchmark} -v -s -P ${server_port}${vsock_extra_opts} > ${server_log} 2>&1"
									# adjust server command for NUMA binding
									server_node=`echo "$server_nodes," | cut -d, -f$server_nr`
									if [ ! -z "$server_node" ]; then
										if [ $server_node -ge 0 ]; then
											benchmark_server_cmd="numactl --cpunodebind=$server_node bash -c \"$benchmark_server_cmd\""
										fi
									fi

									# create a server command file, to be used for debugging purposes or to run this uperf command later [without pbench]
									echo "$benchmark_server_cmd" >$benchmark_server_cmd_file
									chmod +x $benchmark_server_cmd_file

									# construct the client command
									if [ "$log_response_times" == "y" ]; then
										resp_opt=" -X $benchmark_results_dir/$uperf_identifier--response-times.txt"
									fi
									benchmark_client_cmd="${benchmark} -v -m $xml_file -R -a -i 1 ${resp_opt}${vsock_extra_opts} -P $server_port >$result_file 2>&1"
									# adjust client command for NUMA binding
									client_node=`echo "$client_nodes," | cut -d, -f$server_nr`
									if [ ! -z "$client_node" ]; then
										if [ $client_node -ge 0 ]; then
											benchmark_client_cmd="numactl --cpunodebind=$client_node bash -c \"$benchmark_client_cmd\""
										fi
									fi

									# create the client command file, to be used for debugging purposes or to run this uperf command later [without pbench]
									echo "$benchmark_client_cmd" >$benchmark_client_cmd_file
									chmod +x $benchmark_client_cmd_file

									# prepare test files and dirs if using remote clients
									if ! pbench-is-local "${client}"; then
										ssh $ssh_opts $client mkdir -p $benchmark_results_dir
										scp $scp_opts $xml_file $client:$xml_file >/dev/null
										scp $scp_opts $benchmark_client_cmd_file $client:$benchmark_client_cmd_file >/dev/null
									fi

									# prepare test files and dirs on servers
									if ! pbench-is-local "${server}"; then
										ssh $ssh_opts $server mkdir -p $benchmark_results_dir
										scp $scp_opts $benchmark_server_cmd_file $server:$benchmark_server_cmd_file >/dev/null
									fi

									# start the uperf server(s)
									stop_server $server $server_port 1

									if pbench-is-local "${server}"; then
										screen -dmS uperf-server $benchmark_server_cmd_file
									else
										ssh $ssh_opts $server "screen -dmS uperf-server $benchmark_server_cmd_file"
									fi

									((server_nr++))
								done

								pbench-start-tools --group=$tool_group --dir=$benchmark_results_dir

								# start the uperf clients
								echo "test sample $sample of $nr_samples"
								log "test sample $sample of $nr_samples"
								server_nr=1
								for server in ${servers//,/ }; do
									client=`echo ${clients} | cut -d, -f$server_nr`
									server_port=`echo "$port_number_gap * $server_nr + $server_base_port" | bc`
									uperf_identifier="client::$client-server::$server:$server_port"

									xml_file="$iteration_dir/$uperf_identifier--test_config.xml"
									benchmark_client_cmd_file="$iteration_dir/$uperf_identifier--client_start.sh"
									result_file="$benchmark_results_dir/$uperf_identifier--client_output.txt"

									client_node=`echo "$client_nodes," | cut -d, -f$server_nr`
									client_nodeinfo=""
									if [ ! -z "$client_node" ]; then
										if [ $client_node -ge 0 ]; then
											client_nodeinfo="node[$client_node]"
										fi
									fi
									server_node=`echo "$server_nodes," | cut -d, -f$server_nr`
									server_nodeinfo=""
									if [ ! -z "$server_node" ]; then
										if [ $server_node -ge 0 ]; then
											server_nodeinfo="node[$server_node]"
										fi
									fi
									debug_log "client[$client]${client_nodeinfo}protocol[$protocol]test[$test_type]instances[$instance]size[$message_size] <-> server[$server]${server_nodeinfo}"
									echo "client[$client]${client_nodeinfo}protocol[$protocol]test[$test_type]instances[$instance]size[$message_size] <-> server[$server]${server_nodeinfo}"

									if pbench-is-local "${client}"; then
										# using local client
										debug_log "screen -dmS uperf-client $benchmark_client_cmd_file"
										screen -dmS uperf-client $benchmark_client_cmd_file
									else
										# using remote client
										debug_log "ssh $ssh_opts $client screen -dmS uperf-client $benchmark_client_cmd_file"
										ssh $ssh_opts $client "screen -dmS uperf-client $benchmark_client_cmd_file"
									fi
									((server_nr++))
								done

								sleep $runtime_padded

								# stop tools and clean up
								pbench-stop-tools --group=$tool_group --dir=$benchmark_results_dir
								pbench-send-tools --group=$tool_group --dir=$benchmark_results_dir

								server_nr=1
								for server in ${servers//,/ }; do
									client=`echo ${clients} | cut -d, -f$server_nr`
									server_port=`echo "$port_number_gap * $server_nr + $server_base_port" | bc`
									uperf_identifier="client::$client-server::$server:$server_port"

									stop_server $server $server_port 0

									if ! pbench-is-local "${server}"; then
										server_log="$benchmark_results_dir/$uperf_identifier--server.log"
										scp $scp_opts $server:$server_log $server_log >/dev/null
									fi
									if ! pbench-is-local "${client}"; then
										result_file="$benchmark_results_dir/$uperf_identifier--client_output.txt"
										scp $scp_opts $client:$result_file $result_file >/dev/null
									fi
									((server_nr++))
								done
							else # only postprocessing
								let fail_num=$failures+1
								set -x
								if [ ! -e $iteration_dir -a -e $iteration_dir-fail$fail_num ]; then
									mv $iteration_dir-fail$fail_num $iteration_dir
								fi
								set +x
								pushd $iteration_dir >/dev/null
								if [ ! -e ${sample_name} -a -e ${sample_name}.tar.xz ]; then
									tar Jxf ${sample_name}.tar.xz && /bin/rm -rf ${sample_name}.tar.xz
								fi
								popd >/dev/null
								if [[ ! -d $benchmark_results_dir ]]; then
									error_log "Results directory $benchmark_results_dir does not exist, skipping post-processing"
									continue
								fi
								echo "Not going to run uperf.  Only postprocesing existing data"
								log "Not going to run uperf.  Only postprocesing existing data"
							fi
							pbench-postprocess-tools --group=$tool_group --dir=$benchmark_results_dir
							echo "$script_path/postprocess/$benchmark-postprocess \"$benchmark_results_dir\" \"$protocol\" \"$message_size\" \"$instance\" \"$test_type\" \"${clients}\" \"${servers}\" \"$tool_label_pattern\" \"$tool_group\" \"${ver}\"" >"$benchmark_results_dir/$benchmark-postprocess.cmd"
							chmod +x "$benchmark_results_dir/$benchmark-postprocess.cmd"
							$benchmark_results_dir/$benchmark-postprocess.cmd
						done
						echo "$script_path/postprocess/process-iteration-samples \"$iteration_dir\" \"$primary_metric\" \"$maxstddevpct\" \"$failures\" \"$max_failures\" \"$tar_nonref_data\" \"$keep_failed_tool_data\"" >"$iteration_dir/process-iteration-samples.cmd"
						chmod +x "$iteration_dir/process-iteration-samples.cmd"
						$iteration_dir/process-iteration-samples.cmd
						fail=$?
						if [ $fail -ne 0 ]; then
							((failures++))
						fi
						if [ $fail -eq 0 -o $failures -ge $max_failures ]; then
							break
						fi
					done # break out of this loop only if the $result_stddevpct & $eff_stddevpct lower than $maxstddevpct
					echo "Iteration $iteration complete ($count of $total_iterations), with 1 pass and $failures failures"
					log "Iteration $iteration complete ($count of $total_iterations), with 1 pass and $failures failures"
				else
					echo "Skipping iteration $iteration ($count of $total_iterations)"
					log "Skipping iteration $iteration ($count of $total_iterations)"
				fi
				last_test_type="$test_type"
				let count=$count+1 # now we can move to the next iteration
			done
		done
	done
done
echo "$script_path/postprocess/generate-benchmark-summary \"$benchmark\" \"$orig_cmd\" \"$benchmark_run_dir\"" >"$benchmark_run_dir/generate-benchmark-summary.cmd"
chmod +x "$benchmark_run_dir/generate-benchmark-summary.cmd"
$benchmark_run_dir/generate-benchmark-summary.cmd

if [[ $postprocess_only == "n" ]]; then
	# Stop the tool meisters on each registered local/remote host
	pbench-tool-meister-stop --sysinfo="${sysinfo}" "${tool_group}"
	if [[ ${?} != 0 ]]; then
		error_log "[${script_name}]: failed to stop the tool meisters."
	fi
fi

rmdir $benchmark_run_dir/.running
