#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; tab-width: 8 -*-

# This is a script to run the uperf benchmark
# Author: Andrew Theurer

# This script attempts to automate potentially a very large number of tests for uperf
# By default, is runs 96 different tests and can take several hours.
# To limit the number of tests, use the --protocols --test-types --instances options to reduce the number of tests

# This script will take multiple samples of the same test type and try to achieve a standard deviation of <3%
#
# This script will repeat a test type 6 times in order to try to achieve target stddev.
# If a run (with several samples) fails the stddev, its directory is appended with -fail
#
# This script will also generate a "summary-results.txt" with a table of all
# results, efficiency, and other stats.
pbench_cmd="$@"

script_path=`dirname $0`
script_name=`basename $0`
pbench_bin="`cd ${script_path}/..; /bin/pwd`"

# source the base script
. "$pbench_bin"/base

benchmark="uperf"
benchmark_rpm=${benchmark}
if [[ -z "${benchmark_bin}" ]]; then
	benchmark_bin="$(resolve_benchmark_bin "${benchmark}")"
	if [[ -z "${benchmark_bin}" ]]; then
		error_log "${script_name}: ${benchmark} executable not found on PATH=${PATH}"
		exit 1
	fi
fi
ver="$(getconf.py version ${benchmark})"
if [[ -z "${ver}" ]]; then
        error_log "${script_name}: package version is missing in config file"
        exit 1
fi

# Every bench-script follows a similar sequence:
# 1) process bench script arguments
# 2) ensure the right version of the benchmark is installed
# 3) gather pre-run state
# 4) run the benchmark and start/stop perf analysis tools
# 5) gather post-run state
# 6) postprocess benchmark data
# 7) postprocess analysis tool data

# Defaults

export benchmark_run_dir=""
protocols="tcp,udp"
test_types="stream,maerts,bidirec,rr"
message_sizes="1,64,1024,16384" # in bytes
config=""
instances="1,8,64"
nr_samples=5
maxstddevpct=5 # maximum allowable standard deviation in percent
max_failures=6 # after N failed attempts to hit below $maxstddevpct, move on to the nest test
runtime=60
mode="loopback"
servers=127.0.0.1
server_base_port=20000
port_number_gap=10
postprocess_only=n
log_response_times=n
start_iteration_num=1
tar_nonref_data="n"
keep_failed_tool_data="y"
orig_cmd="$*"
tool_group=default
tool_label_pattern="uperf-"
server_nodes=""
client_nodes=""
sysinfo="default"

function usage {
	printf "\tThe following options are available:\n\n"
	printf -- "\t--tool-group=str\n"
	printf -- "\t-c str       --config=str               name of the test config (e.g. jumbo_frames_and_network_throughput)\n"
	printf -- "\t-t str[,str] --test-types=str[,str]     can be stream, maerts, bidirec, and/or rr (default $test_types)\n"
	printf -- "\t-r int       --runtime=int              test measurement period in seconds (default is $runtime)\n"
	printf -- "\t-m int[,int] --message-sizes=str[,str]  list of message sizes in bytes (default is $message_sizes)\n"
	printf -- "\t-p str[,str] --protocols=str[,str]      tcp and/or udp (default is $protocols)\n"
	printf -- "\t-i int[,int] --instances=int[,int]      list of number of uperf instances to run (default is $instances)\n"
	printf -- "\t-C str[,str] --client[s]=str[,str]      a list of one or more hostnames/IPs.  These systems will run the\n"
	printf    "\t\t\t\t   uperf client (drive the test).\n"
	printf    "\t\t\t\t   If this is omitted, the local system is the client.\n"
	printf    "\t\t\t\t   Note: the number of clients and servers must be the same!\n"
	printf    "\t\t\t\t   Clients and servers are paired according to the order in the list (first\n"
	printf    "\t\t\t\t   client pairs with first server, etc)\n"
	printf -- "\t-S str[,str] --server[s]=str[,str]      a list of one or more hostnames/IPs.  These systems will run the uperf\n"
	printf    "\t\t\t\t   server (listening for connections).\n"
	printf    "\t\t\t\t   If this is omitted, the server will listen on the local system\n"
	printf    "\t\t\t\t   loopback interface.\n"
	printf -- "\t--server-node[s]=str[,str]              An ordered list of server NUMA nodes which should be used for CPU binding\n"
	printf -- "\t--client-node[s]=str[,str]              An ordered list of client NUMA nodes which should be used for CPU binding\n"
	printf -- "\t\t\t\t   For both options above, the order must correspond with the --clients/--servers list\n"
	printf -- "\t\t\t\t   To omit a specific client/server from binding, use a value of -1.\n"
	printf -- "\t--samples=int              the number of times each different test is run (to compute average &\n"
	printf    "\t\t\t\t   standard deviations).\n"
	printf -- "\t--max-failures=int         the maximum number of failures to get below stddev.\n"
	printf -- "\t--max-stddev=int           the maximum percent stddev allowed to pass.\n"
	printf -- "\t--postprocess-only=y|n     don't run the benchmark, but postprocess data from previous test.\n"
	printf -- "\t--run-dir=str              optionally specify what directory should be used (usually only used\n"
	printf    "\t\t\t\t   if postprocess-only=y).\n"
	printf -- "\t--start-iteration-num=int  optionally skip the first (n-1) tests.\n"
	printf -- "\t--log-response-times=y|n   record the response time of every single operation.\n"
	printf -- "\t--tool-label-pattern=str   uperf will provide CPU and efficiency information for any tool directory\n"
	printf    "\t\t\t\t   with a \"^<pattern>\" in the name, provided \"sar\" is one of the\n"
	printf    "\t\t\t\t   registered tools.\n"
	printf    "\t\t\t\t   a default pattern, \"uperf-\" is used if none is provided.\n"
	printf    "\t\t\t\t   simply register your tools with \"--label=uperf-\$X\", and this script\n"
	printf    "\t\t\t\t   will generate CPU_uperf-\$X and Gbps/CPU_uperf-\$X or\n"
	printf    "\t\t\t\t   trans_sec/CPU-uperf-\$X for all tools which have that pattern as a\n"
	printf    "\t\t\t\t   prefix.  If you don't want to register your tools with \"uperf-\" as\n"
	printf    "\t\t\t\t   part of the label, just use --tool-label-pattern= to tell this script\n"
	printf    "\t\t\t\t   the prefix pattern to use for CPU information.\n"
	printf -- "\t--sysinfo=str,             str= comma separated values of sysinfo to be collected\n"
	printf -- "\t                                available: $(pbench-collect-sysinfo --options)\n"
}

function gen_xml {
	local h=$1
	local proto=$2
	local runtime=$3
	local size=$4
	local nthr=$5
	local test_type=$6

	echo "<?xml version=\"1.0\"?>"
	echo "<profile name=\"$proto-$test_type-${size}B-${nthr}i\">"
	case $test_type in
	rr)
		echo "  <group nthreads=\"$nthr\">"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"connect\" options=\"remotehost=$h protocol=$proto\"/>"
		echo "    </transaction>"
		echo "    <transaction duration=\"$runtime\">"
		echo "      <flowop type=\"write\" options=\"size=$size\"/>"
		echo "      <flowop type=\"read\"  options=\"size=$size\"/>"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"disconnect\" />"
		echo "    </transaction>"
		echo "  </group>"
		;;
	stream|bidirec)
		echo "  <group nthreads=\"$nthr\">"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"connect\" options=\"remotehost=$h protocol=$proto\"/>"
		echo "    </transaction>"
		echo "    <transaction duration=\"$runtime\">"
		echo "      <flowop type=\"write\" options=\"count=16 size=$size\"/>"
		echo "    </transaction>"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"disconnect\" />"
		echo "    </transaction>"
		echo "  </group>"
		;;&
	maerts|bidirec)
		echo "  <group nthreads=\"$nthr\">"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"accept\" options=\"remotehost=$h protocol=$proto\"/>"
		echo "    </transaction>"
		echo "    <transaction duration=\"$runtime\">"
		echo "      <flowop type=\"read\" options=\"count=16 size=$size\"/>"
		echo "    </transaction>"
		echo "    <transaction iterations=\"1\">"
		echo "      <flowop type=\"disconnect\" />"
		echo "    </transaction>"
		echo "  </group>"
		;;
	esac
	echo "</profile>"
}

function stop_server {
	local server=$1
	local server_port=$2
	local message=$3
	local uperf_pid=`ssh $ssh_opts $server netstat -tlnp | grep ":$server_port " | awk '{print $7}' | awk -F/ '{print $1}'`
	if [ ! -z "$uperf_pid" ]; then
		if [ $message -eq 1 ]; then
			echo found uperf pid $uperf_pid on $server:$server_port, killing
		fi
		ssh $ssh_opts $server kill $uperf_pid
	fi
}

function install_uperf {
	if check_install_rpm ${benchmark_rpm} ${ver}; then
		debug_log "[$script_name] ${benchmark_rpm}-${ver} is installed"
	else
		error_log "[$script_name] ${benchmark_rpm}-${ver} is not installed, exiting"
		exit 1
	fi
}

# Process options and arguments
opts=$(getopt -q -o C:c:hi:m:p:r:S:t: --longoptions "help,server-node:,server-nodes:,client-node:,client-nodes:,client-label:,server-label:,tool-label-pattern:,install,start-iteration-num:,config:,instances:,test-types:,runtime:,message-sizes:,protocols:,samples:,client:,clients:,servers:,server:,max-stddev:,max-failures:,log-response-times:,postprocess-only:,run-dir:,tool-group:,sysinfo:" -n "getopt.sh" -- "$@")
if [ $? -ne 0 ]; then
	printf -- "${script_name} $*\n"
	printf -- "\n"
	printf -- "\tunrecognized option specified\n\n"
	usage
	exit 1
fi
eval set -- "$opts"
while true; do
	case "$1" in
		--install)
		shift
		install_uperf
		exit
		;;
		-h|--help)
		usage
		exit 0
		;;
		--client-label)
		shift
		if [ -n "$1" ]; then
			shift
		fi
		warn_log "The --client-label option is deprecated, please use --tool-label-pattern"
		;;
		--server-label)
		shift
		if [ -n "$1" ]; then
			shift
		fi
		warn_log "The --server-label option is deprecated, please use --tool-label-pattern"
		;;
		--tool-label-pattern)
		shift
		if [ -n "$1" ]; then
			tool_label_pattern="$1"
			shift
		fi
		;;
		--postprocess-only)
		shift
		if [ -n "$1" ]; then
			postprocess_only="$1"
			shift
		fi
		;;
		--run-dir)
		shift
		if [ -n "$1" ]; then
			benchmark_run_dir="$1"
			shift
		fi
		;;
		--max-stddev)
		shift
		if [ -n "$1" ]; then
			maxstddevpct="$1"
			shift
		fi
		;;
		--max-failures)
		shift
		if [ -n "$1" ]; then
			max_failures="$1"
			shift
		fi
		;;
		--samples)
		shift
		if [ -n "$1" ]; then
			nr_samples="$1"
			shift
		fi
		;;
		-i|--instances)
		shift
		if [ -n "$1" ]; then
			instances="$1"
			shift
		fi
		;;
		-t|--test-types)
		shift
		if [ -n "$1" ]; then
			test_types="$1"
			shift
		fi
		;;
		--tool-group)
		shift
		if [ -n "$1" ]; then
			tool_group="$1"
			shift
		fi
		;;
		-m|--message-sizes)
		shift
		if [ -n "$1" ]; then
			message_sizes="$1"
			shift
		fi
		;;
		-r|--runtime)
		shift
		if [ -n "$1" ]; then
			runtime="$1"
			shift
		fi
		;;
		--log-response-times)
		shift
		if [ -n "$1" ]; then
			log_response_times="$1"
			shift
		fi
		;;
		-p|--protocols)
		shift
		if [ -n "$1" ]; then
			protocols="$1"
			shift
		fi
		;;
		-C|--client|--clients)
		shift
		if [ -n "$1" ]; then
			clients="$1"
			shift
		fi
		;;
		-S|--server|--servers)
		shift
		if [ -n "$1" ]; then
			servers="$1"
			shift
		fi
		;;
		-c|--config)
		shift
		if [ -n "$1" ]; then
			config="$1"
			shift
		fi
		;;
		--start-iteration-num)
		shift
		if [ -n "$1" ]; then
			start_iteration_num=$1
			shift
		fi
		;;
		--client-node|--client-nodes)
		shift
		if [ -n "$1" ]; then
			client_nodes="$1"
			shift
		fi
		;;
		--server-node|--server-nodes)
		shift
		if [ -n "$1" ]; then
			server_nodes="$1"
			shift
		fi
		;;
		--sysinfo)
		shift;
		if [ -n "$1" ]; then
			sysinfo="$1"
			shift;
		fi
		;;
		--)
		shift
		break
		;;
		*)
		error_log "[$script_name] bad option, \"$1 $2\""
		exit 1
		break
		;;
	esac
done
verify_common_bench_script_options $tool_group $sysinfo

# Verify the number of clients and servers match.
if [ ! -z "$clients" -a "$postprocess_only" != "y" ]; then
	let c_cnt=0
	for client in `echo $clients | sed -e s/,/" "/g`; do
		let c_cnt=c_cnt+1
	done
	let s_cnt=0
	for server in `echo $servers | sed -e s/,/" "/g`; do
		let s_cnt=s_cnt+1
	done
	if [ $c_cnt -ne $s_cnt ]; then
		error_log "Number of clients and servers specified on command line must match"
		error_log "    clients($c_cnt): $clients"
		error_log "    servers($s_cnt): $servers"
		exit 1
	fi
fi
# before we run a test, verify clients and servers are accessible and install
# uperf if necessary
if [ "$postprocess_only" != "y" ]; then
	# Always install locally
	# FIXME: why is this necessary?
	install_uperf
	let err_cnt=0
	err_clients=""
	for client in `echo $clients | sed -e s/,/" "/g`; do
		debug_log "checking for uperf on client $client"
		ssh $ssh_opts $client pbench-uperf --install
		if [ $? -ne 0 ]; then
			let err_cnt=err_cnt+1
			err_clients="$err_clients $client"
		fi
	done
	err_servers=""
	for server in `echo $servers | sed -e s/,/" "/g`; do
		debug_log "checking for uperf on server $server"
		ssh $ssh_opts $server pbench-uperf --install
		if [ $? -ne 0 ]; then
			let err_cnt=err_cnt+1
			err_servers="$err_servers $server"
		fi
	done
	if [ $err_cnt -gt 0 ]; then
		error_log "Unable to verify connectivity and uperf installation on the following clients and servers:"
		error_log "    clients: $err_clients"
		error_log "    servers: $err_servers"
		exit 1
	fi
fi

# runtime is padded by 6 seconds to allow 3 seconds ramp up and ramp down
runtime_padded=$((${runtime}+6))

###
# At this point all parameter checking has been done.
###

if [[ -z "$benchmark_run_dir" ]]; then
	# We don't have an explicit run directory, construct one
	benchmark_fullname="${benchmark}_${config}_${date_suffix}"
	benchmark_run_dir="$pbench_run/${benchmark_fullname}"
else
	# We have an explicit run directory provided by --run-dir, so warn
	# the user if they also used --config
	if [[ ! -z "$config" ]]; then
		warn_log "[$script_name] ignoring --config=\"$config\" in favor of --rundir=\"$benchmark_run_dir\""
	fi
	benchmark_fullname=$(basename $benchmark_run_dir)
fi
# we'll record the iterations in this file
benchmark_iterations="${benchmark_run_dir}/.iterations"
mdlog=${benchmark_run_dir}/metadata.log

function record_iteration {
	local count=$1
	local protocol=$2
	local test_type=$3
	local message_size=$4
	local instance=$5
	local iteration=$6

	echo ${iteration} >> ${benchmark_iterations}
	echo $count | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_number
	echo $protocol | pbench-add-metalog-option ${mdlog} iterations/${iteration} protocol
	echo $test_type | pbench-add-metalog-option ${mdlog} iterations/${iteration} test_type
	echo $message_size | pbench-add-metalog-option ${mdlog} iterations/${iteration} message_size_bytes
	echo $instance | pbench-add-metalog-option ${mdlog} iterations/${iteration} instances
	echo $iteration | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_name
}

mkdir -p $benchmark_run_dir/.running

# now that the benchmark_run_dir directory exists, we can initialize the iterations file
> ${benchmark_iterations}

# save a copy of the command, in case the test needs to be reproduced or post-processed again
echo "$script_name $pbench_cmd" >$benchmark_run_dir/$script_name.cmd
chmod +x $benchmark_run_dir/$script_name.cmd

## Run the benchmark and start/stop perf analysis tools

total_iterations=0
for protocol in `echo $protocols | sed -e s/,/" "/g`; do
	for test_type in `echo $test_types | sed -e s/,/" "/g`; do
		for message_size in `echo $message_sizes | sed -e s/,/" "/g`; do
			for instance in `echo $instances | sed -e s/,/" "/g`; do
				((total_iterations++))
			done
		done
	done
done

export benchmark config
pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg
pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir beg
# on abnormal exit, make sure that the metadata log exists and is complete.
trap "pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir int" INT QUIT

# start the server processes
count=1
for protocol in `echo $protocols | sed -e s/,/" "/g`; do
	for test_type in `echo $test_types | sed -e s/,/" "/g`; do
		case $test_type in
			rr)
			primary_metric="trans_sec"
			;;
			stream|maerts|bidirec)
			primary_metric="Gb_sec"
			;;
			*)
			error_log "$script_name: test type \"$test_type\" is not suported"
			continue
			;;
		esac
		for message_size in `echo $message_sizes | sed -e s/,/" "/g`; do
			for instance in `echo $instances | sed -e s/,/" "/g`; do
				if [ $count -ge $start_iteration_num ]; then
					echo "Starting iteration $iteration ($count of $total_iterations)"
					log "Starting iteration $iteration ($count of $total_iterations)"
					iteration="${count}-${protocol}_${test_type}-${message_size}B-${instance}i"
					record_iteration ${count} ${protocol} ${test_type} ${message_size} ${instance} ${iteration}
					iteration_dir="$benchmark_run_dir/$iteration"
					result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough
					failures=0
					while [[ $(echo "if (${result_stddevpct} >= ${maxstddevpct}) 1 else 0" | bc) -eq 1 ]]; do
						if [[ $failures -gt 0 ]]; then
							echo "Restarting iteration $iteration ($count of $total_iterations)"
							log "Restarting iteration $iteration ($count of $total_iterations)"
						fi
						if [ $postprocess_only == "n" ]; then
							mkdir -p $iteration_dir
						fi
						# each attempt at a test config requires multiple samples to get stddev
						for sample in `seq 1 $nr_samples`; do
							benchmark_results_dir="$iteration_dir/sample$sample"
							if [ "$postprocess_only" != "y" ]; then
								mkdir -p $benchmark_results_dir
								server_nr=1
								# the following loop does all of the pre-benchmark execution work
								for server in `echo $servers | sed -e s/,/" "/g`; do
									client=`echo $clients | cut -d, -f$server_nr`
									if [ -z "$client" ]; then
										client="127.0.0.1"
									fi
									server_port=`echo "$port_number_gap * $server_nr + $server_base_port" | bc`
									uperf_identifier="client::$client-server::$server:$server_port"

									benchmark_client_cmd_file="$iteration_dir/$uperf_identifier--client_start.sh"
									benchmark_server_cmd_file="$iteration_dir/$uperf_identifier--server_start.sh"
									result_file="$benchmark_results_dir/$uperf_identifier--client_output.txt"
									server_log="$benchmark_results_dir/$uperf_identifier--server.log"

									xml_file="$iteration_dir/$uperf_identifier--test_config.xml"
									gen_xml $server $protocol ${runtime_padded}s $message_size $instance $test_type >$xml_file

									# construct the server command
									benchmark_server_cmd="${benchmark_bin} -v -s -P $server_port > ${server_log} 2>&1"
									# adjust server command for NUMA binding
									server_node=`echo "$server_nodes," | cut -d, -f$server_nr`
									if [ ! -z "$server_node" ]; then
										if [ $server_node -ge 0 ]; then
											benchmark_server_cmd="numactl --cpunodebind=$server_node bash -c \"$benchmark_server_cmd\""
										fi
									fi

									# create a server command file, to be used for debugging purposes or to run this uperf command later [without pbench]
									echo "$benchmark_server_cmd" >$benchmark_server_cmd_file
									chmod +x $benchmark_server_cmd_file

									# construct the client command
									if [ "$log_response_times" == "y" ]; then
										resp_opt=" -X $benchmark_results_dir/$uperf_identifier--response-times.txt"
									fi
									benchmark_client_cmd="${benchmark_bin} -v -m $xml_file -R -a -i 1 $resp_opt -P $server_port >$result_file 2>&1"
									# adjust client command for NUMA binding
									client_node=`echo "$client_nodes," | cut -d, -f$server_nr`
									if [ ! -z "$client_node" ]; then
										if [ $client_node -ge 0 ]; then
											benchmark_client_cmd="numactl --cpunodebind=$client_node bash -c \"$benchmark_client_cmd\""
										fi
									fi

									# create the client command file, to be used for debugging purposes or to run this uperf command later [without pbench]
									echo "$benchmark_client_cmd" >$benchmark_client_cmd_file
									chmod +x $benchmark_client_cmd_file

									# prepare test files and dirs if using remote clients
									if [ ! -z "$clients" ]; then
										ssh $ssh_opts $client mkdir -p $benchmark_results_dir
										scp $scp_opts $xml_file $client:$xml_file >/dev/null
										scp $scp_opts $benchmark_client_cmd_file $client:$benchmark_client_cmd_file >/dev/null
										ssh $ssh_opts $client "systemctl stop firewalld"
									fi

									# prepare test files and dirs on servers
									ssh $ssh_opts $server mkdir -p $benchmark_results_dir
									scp $scp_opts $benchmark_server_cmd_file $server:$benchmark_server_cmd_file >/dev/null

									# start the uperf server(s)
									stop_server $server $server_port 1
									ssh $ssh_opts $server "systemctl stop firewalld"
									ssh $ssh_opts $server "screen -dmS uperf-server $benchmark_server_cmd_file"

									((server_nr++))
								done

								pbench-start-tools --group=$tool_group --dir=$benchmark_results_dir

								# start the uperf clients
								echo "test sample $sample of $nr_samples"
								log "test sample $sample of $nr_samples"
								server_nr=1
								for server in `echo $servers | sed -e s/","/" "/g`; do
									client=`echo $clients | cut -d, -f$server_nr`
									if [ -z "$client" ]; then
										client="127.0.0.1"
									fi
									server_port=`echo "$port_number_gap * $server_nr + $server_base_port" | bc`
									uperf_identifier="client::$client-server::$server:$server_port"

									xml_file="$iteration_dir/$uperf_identifier--test_config.xml"
									benchmark_client_cmd_file="$iteration_dir/$uperf_identifier--client_start.sh"
									result_file="$benchmark_results_dir/$uperf_identifier--client_output.txt"

									client_node=`echo "$client_nodes," | cut -d, -f$server_nr`
									client_nodeinfo=""
									if [ ! -z "$client_node" ]; then
										if [ $client_node -ge 0 ]; then
											client_nodeinfo="node[$client_node]"
										fi
									fi
									server_node=`echo "$server_nodes," | cut -d, -f$server_nr`
									server_nodeinfo=""
									if [ ! -z "$server_node" ]; then
										if [ $server_node -ge 0 ]; then
											server_nodeinfo="node[$server_node]"
										fi
									fi
									debug_log "client[$client]${client_nodeinfo}protocol[$protocol]test[$test_type]instances[$instance]size[$message_size] <-> server[$server]${server_nodeinfo}"
									echo "client[$client]${client_nodeinfo}protocol[$protocol]test[$test_type]instances[$instance]size[$message_size] <-> server[$server]${server_nodeinfo}"

									if [ -z "$clients" ]; then
										# using local clients
										debug_log "screen -dmS uperf-client $benchmark_client_cmd_file"
										screen -dmS uperf-client $benchmark_client_cmd_file
									else
										# using remote clients
										debug_log "ssh $ssh_opts $client screen -dmS uperf-client $benchmark_client_cmd_file"
										ssh $ssh_opts $client "screen -dmS uperf-client $benchmark_client_cmd_file"
									fi
									((server_nr++))
								done

								sleep $runtime_padded

								# stop tools and clean up
								pbench-stop-tools --group=$tool_group --dir=$benchmark_results_dir
								pbench-postprocess-tools --group=$tool_group --dir=$benchmark_results_dir

								server_nr=1
								for server in `echo $servers | sed -e s/,/" "/g`; do
									client=`echo $clients | cut -d, -f$server_nr`
									if [ -z "$client" ]; then
										client="127.0.0.1"
									fi
									server_port=`echo "$port_number_gap * $server_nr + $server_base_port" | bc`
									uperf_identifier="client::$client-server::$server:$server_port"

									stop_server $server $server_port 0
									server_log="$benchmark_results_dir/$uperf_identifier--server.log"
									scp $scp_opts $server:$server_log $server_log >/dev/null
									if [ ! -z "$clients" ]; then
										result_file="$benchmark_results_dir/$uperf_identifier--client_output.txt"
										scp $scp_opts $client:$result_file $result_file >/dev/null
									fi
									((server_nr++))
								done
							else # only postprocessing
								let fail_num=$failures+1
								set -x
								if [ ! -e $iteration_dir -a -e $iteration_dir-fail$fail_num ]; then
									mv $iteration_dir-fail$fail_num $iteration_dir
								fi
								set +x
								pushd $iteration_dir >/dev/null
								if [ ! -e sample$sample -a -e sample$sample.tar.xz ]; then
									tar Jxf sample$sample.tar.xz && /bin/rm -rf sample$sample.tar.xz
								fi
								popd >/dev/null
								if [[ ! -d $benchmark_results_dir ]]; then
									error_log "Results directory $benchmark_results_dir does not exist, skipping post-processing"
									continue
								fi
								echo "Not going to run uperf.  Only postprocesing existing data"
								log "Not going to run uperf.  Only postprocesing existing data"
							fi
							echo "$script_path/postprocess/$benchmark-postprocess \"$benchmark_results_dir\" \"$protocol\" \"$message_size\" \"$instance\" \"$test_type\" \"$clients\" \"$servers\" \"$tool_label_pattern\" \"$tool_group\" \"${ver}\"" >"$benchmark_results_dir/$benchmark-postprocess.cmd"
							chmod +x "$benchmark_results_dir/$benchmark-postprocess.cmd"
							$benchmark_results_dir/$benchmark-postprocess.cmd
						done
						echo "$script_path/postprocess/process-iteration-samples \"$iteration_dir\" \"$primary_metric\" \"$maxstddevpct\" \"$failures\" \"$max_failures\" \"$tar_nonref_data\" \"$keep_failed_tool_data\"" >"$iteration_dir/process-iteration-samples.cmd"
						chmod +x "$iteration_dir/process-iteration-samples.cmd"
						$iteration_dir/process-iteration-samples.cmd
						fail=$?
						if [ $fail -ne 0 ]; then
							((failures++))
						fi
						if [ $fail -eq 0 -o $failures -ge $max_failures ]; then
							break
						fi
					done # break out of this loop only if the $result_stddevpct & $eff_stddevpct lower than $maxstddevpct
					echo "Iteration $iteration complete ($count of $total_iterations), with 1 pass and $failures failures"
					log "Iteration $iteration complete ($count of $total_iterations), with 1 pass and $failures failures"
				else
					echo "Skipping iteration $iteration ($count of $total_iterations)"
					log "Skipping iteration $iteration ($count of $total_iterations)"
				fi
				last_test_type="$test_type"
				let count=$count+1 # now we can move to the next iteration
			done
		done
	done
done
echo "$script_path/postprocess/generate-benchmark-summary \"$benchmark\" \"$orig_cmd\" \"$benchmark_run_dir\"" >"$benchmark_run_dir/generate-benchmark-summary.cmd"
chmod +x "$benchmark_run_dir/generate-benchmark-summary.cmd"
$benchmark_run_dir/generate-benchmark-summary.cmd
pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end
pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo end

rmdir $benchmark_run_dir/.running
