#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*-

# Author: Andrew Theurer
#
# This is a script to run a network traffic generator like MoonGen or TRex.
# It is intended to test a network function, like a bridge or router.
#
# Hardware prerequisites:
# - A Linux host with 1 to 2 network adapters.  These adapters must be either Intel XL710 or Intel 82599.
#   Ideally this host should have x86-64 processors (current or previous generation) 1GB HW pages, and PCI-E v2 or v3.
# - A "device-under-test" (DUT) which network traffic will pass through.
#   The DUT will need to be configured to pass traffic before running this benchmark script.
#

# This script attempts to automate potentially a very large number of tests.
# This script will take multiple samples of the same test type and try to achieve a standard deviation of <3%.
# This script will repeat a test type 6 times in order to try to achieve target stddev.
# If a run (with several samples) fails the stddev, its directory is appended with -fail.
# This script will also generate a "results.txt" with a table of all results, efficiency, and other stats.

script_path="$(dirname ${0})"
script_name="$(basename ${0})"
pbench_bin="$(realpath -e ${script_path}/..)"

# Save the original command line invocation
orig_cmd="${*}"

# source the base script
. "${pbench_bin}"/base

# Every bench-script follows a similar sequence:
# 1) process bench script arguments
# 2) ensure the right version of the benchmark is installed
# 3) gather pre-run state
# 4) run the benchmark and start/stop perf analysis tools
# 5) gather post-run state
# 6) postprocess benchmark data
# 7) postprocess analysis tool data

benchmark_name="trafficgen"
export benchmark=${benchmark_name}

# General pbench defaults
export benchmark_run_dir=""
export config=""
postprocess_only="n"
tool_group="default"
sysinfo="none"

# Trafficgen specific defaults
active_devices="" # the active network devices as PCI location IDs, like "0000:04:00.0,0000:04:00.1"
all_possible_flow_mods="src-ip,dst-ip,src-mac,dst-mac,src-port,dst-port,encap-src-ip,encap-dst-ip,encap-src-mac,encap-dst-mac,protocol,none"
devices="" # the network devices as PCI location IDs, like "0000:04:00.0,0000:04:00.1"
df_search_runtime=120
df_sniff_runtime=30
df_validation_runtime=300
df_zl_search_runtime=1200
df_zl_sniff_runtime=30
df_zl_validation_runtime=7200
dst_ips=""
dst_macs=""
encap_dst_ips=""
encap_dst_macs=""
encap_src_ips=""
encap_src_macs=""
flow_mods_def="src-ip,dst-ip,src-mac,dst-mac"
flow_mods="${flow_mods_def}"
frame_sizes_def="64" # size in bytes of the Ethernet frame including CRC
frame_sizes="${frame_sizes_def}"
keep_failed_tool_data="y"
max_failures=1 # after N failed attempts to hit below $maxstddevpct, move on to the nest test
max_loss_pcts_def="0.002"
max_loss_pcts="${max_loss_pcts_def}"
maxstddevpct=5 # maximum allowable standard deviation in percent
num_flows_def="1024"
num_flows="${num_flows_def}"
num_samples=1
one_shot="n"
packet_protocol_def="UDP"
packet_protocol="${packet_protocol_def}"
passthrough_arguments=""
pre_sample_cmd=""
rate_tolerance_pct=3 # by percent, how much the TX rate can vary from desired rate
rate_unit="mpps"
rates="none"
skip_git_pull="n"
skip_trex_install="n"
skip_trex_server="n"
src_ips=""
src_macs=""
start_iteration_num=1
tar_nonref_data="n"
tool_period="binary-search" # can also be "repeat-final-validation"
traffic_directions_def="bidirectional"
traffic_directions="${traffic_directions_def}"
traffic_generator_def="trex-txrx" # can be either trex-txrx or moongen-txrx
traffic_generator="${traffic_generator_def}"
traffic_profiles=""
if [[ -z "${trafficgen_dir}" ]]; then
	trafficgen_dir="/opt/bench-trafficgen/trafficgen"
fi
trafficgen_version="unknown"
trex_use_ht="n"
trex_use_l2="n"

cmd="./binary-search.py"

function kill_trex() {
	if [[ "${_PBENCH_UNIT_TESTS}" == 1 ]]; then
		return 0
	fi

	# kill any existing trex server process
	local trex_pids=$(netstat -tanp | grep -E :4500\|:4501 | awk '{print $7}' | awk -F/ '{print $1}' | grep -v -- "-" | sort | uniq)
	if [ ! -z "$trex_pids" ]; then
		echo "killing existing trex server"
		kill ${trex_pids}
		sleep 5
	fi
}

function start_trex() {
	if [[ "${_PBENCH_UNIT_TESTS}" == 1 ]]; then
		return 0
	fi

	local _skip_trex_install=${1}
	local _trex_use_ht=${2}
	local _trex_use_l2=${3}
	local _devices=${4}

	kill_trex

	local trex_base_dir="/opt/trex"
	local trex_dir="${trex_base_dir}/current"
	if [ "${_skip_trex_install}" == "n" ]; then
		if ! ${trafficgen_dir}/install-trex.sh --tmp-dir=${pbench_tmp} --base-dir=${trex_base_dir}; then
			error_log "installation of trex failed."
			exit 1
		fi
	fi

	if ! ${trafficgen_dir}/launch-trex.sh --tmp-dir=${pbench_tmp} --trex-dir=${trex_dir} --use-ht="${_trex_use_ht}" --use-l2="${_trex_use_l2}" --devices="${_devices}"; then
		error_log "failed to start trex."
		exit 1
	fi
}

function load_module() {
	if [[ "${_PBENCH_UNIT_TESTS}" == 1 ]]; then
		return 0
	fi

	if [ ! -e /sys/module/$module ]; then
		echo "attempting to load $module"
		modprobe $module
		if [ ! -e /sys/module/$module ]; then
			error_log "could not load $module module, exiting"
			exit 1
		fi
	fi
}

function configure_devices() {
	if [[ "${_PBENCH_UNIT_TESTS}" == 1 ]]; then
		return 0
	fi

	local devices="${1}"

	for dev in `echo $devices | sed -e s/,/" "/g`; do
		echo "configuring device $dev"
		if /bin/ls /sys/module/vfio_pci/drivers/pci\:vfio-pci/ | grep -q "$dev"; then
			echo "device $dev already bound to vfio-pci"
		else
			if [ -e /sys/bus/pci/devices/$dev/driver ]; then
				if dpdk-devbind -u $dev; then
					dpdk-devbind -b vfio-pci $dev
				else
					error_log "could not unbind device, exiting"
					exit 1
				fi
			else
				dpdk-devbind -b vfio-pci $dev
			fi
		fi
	done
	for dev in `echo $devices | sed -e s/,/" "/g`; do
		if /bin/ls /sys/module/vfio_pci/drivers/pci\:vfio-pci/ | grep -q "$dev"; then
			echo "device $dev bound to vfio-pci"
		else
			echo "device $dev not bound to vfio-pci, exiting"
		fi
	done
}

function convert_number_range() {
	# converts a range of cpus, like "1-3,5" to a list, like "1,2,3,5"
	local cpu_range=$1
	local cpus_list=""
	local cpus=""
	for cpus in `echo "$cpu_range" | sed -e 's/,/ /g'`; do
		if echo "$cpus" | grep -q -- "-"; then
			cpus=`echo $cpus | sed -e 's/-/ /'`
			cpus=`seq $cpus | sed -e 's/ /,/g'`
		fi
		for cpu in $cpus; do
			cpus_list="$cpus_list,$cpu"
		done
	done
	cpus_list=`echo $cpus_list | sed -e 's/^,//'`
	echo "$cpus_list"
}

function usage {
	printf "\tThe following options are available:\n\n"
	#                   1         2         3         4         5         6         7         8
	#          12345678901234567890123456789012345678901234567890123456789012345678901234567890
	printf -- "\n"
	printf -- "trafficgen general test options\n"
	printf -- "-------------------------------\n"
	printf -- "\n"
	printf -- "--traffic-generator=str\n"
	printf -- "  The traffic generation engine to use (trex-txrx or trex-txrx-profile or moongen-txrx)\n"
	printf -- "  Default is ${traffic_generator_def}\n"
	printf -- "\n"
	printf -- "--devices=str,str\n"
	printf -- "  List of 2 (or more) DPDK devices to use, by PCI location ID (0000:04:00.0,0000:04:00.1)\n"
	printf -- "  This script will attempt to bind the device to vfio-pci kernel module.\n"
	printf -- "  You must ensure your system has IOMMU enabled.  Devices must be listed in increments of 2.\n"
	printf -- "\n"
	printf -- "--active-devices=str,str\n"
	printf -- "  List of 2 (ore more) DPDK devices to use as active participants in the test.  These devices\n"
	printf -- "  must be a subset of what is supplied for --devices.  Devices must be listed in increments of 2.\n"
	printf -- "\n"
	printf -- "--one-shot\n"
	printf -- "  Run one test for throughput at specified rate\n"
	printf -- "\n"
	printf -- "--sniff-runtime=int\n"
	printf -- "  Measurement period in seconds for a \"sniff\" trial, right before the search trial\n"
	printf -- "  if the sniff test fails, the search trial is skipped and moves on to the next trial\n"
	printf -- "  Default is ${df_sniff_runtime}\n"
	printf -- "\n"
	printf -- "--search-runtime=int\n"
	printf -- "  Measurement period in seconds when searching for max throughput\n"
	printf -- "  Default is ${df_search_runtime}\n"
	printf -- "\n"
	printf -- "--validation-runtime=int\n"
	printf -- "  Measurement period in seconds when running final validation or a latency test\n"
	printf -- "  Default is ${df_validation_runtime}\n"
	printf -- "\n\n"
	printf -- "      The following options can take 1 or more values and will control how many\n"
	printf -- "      tests will be conducted.  For example, the following options:\n"
	printf -- "\n"
	printf -- "        --rates=1,2 --traffic-directions=unidirec,bidirec --max-loss-pcts=0,1\n"
	printf -- "        --frame-sizes=64,256 --nr-flows=1024,65536\n"
	printf -- "\n"
	printf -- "      would produce 32 different test permutations\n"
	printf -- "\n"
	printf -- "--traffic-profile[s]=FILE[,FILE]\n"
	printf -- "  A list of traffic profiles to load when --traffic-generator=trex-txrx-profile.  The\n"
	printf -- "  profile contains a list of streams, each with fine grained options defined on a per\n"
	printf -- "  stream basis.  See README-trex-txrx-profile.md in the trafficgen distribution for\n"
	printf -- "  documentation on the file format.\n"
	printf -- "\n"
	printf -- "--rate[s]=value[,value]]\n"
	printf -- "  A list of packet rates, in millions of packets per second (mpps), unless\n"
	printf -- "  --rate-unit=%% is used, where the rate value is interpreted as a percentage\n"
	printf -- "  of line rate for the transmitting device.\n"
	printf -- "\n"
	printf -- "  Only use this option when:\n"
	printf -- "    1) Using the --one-shot option (skip binary search) and only 1 frame size\n"
	printf -- "      For example, you may want to use this when measuring latency for\n"
	printf -- "      10%%, 50%%, 90%% of line rate with 64-byte frames:\n"
	printf -- "      --rates=10%%,50%%,90%% --rate-unit=%% --frame-size=64 --one-shot\n"
	printf -- "    2) Performing a binary search, but you want the search to start with a rate\n"
	printf -- "       lower than 100%%.  When doing this, it is recommended that only 1 frame\n"
	printf -- "       size is used:\n"
	printf -- "       --rate=50%% --rate-unit=%% --frame-size=64\n"
	printf -- "\n"
	printf -- "--traffic-direction[s]=str[,str]\n"
	printf -- "  A list of one or more: unidirectional, revunidirectional, or bidirectional\n"
	printf -- "  Default is ${traffic_directions_def}\n"
	printf -- "  unidirectional: packets will Tx out the 1st device and Rx in the 2nd device\n"
	printf -- "  revunidirectional: packets will Tx out the 2nd device and RX in the 1st device\n"
	printf -- "  bidirectional: packets will Tx out the both devices and Rx in both devices\n"
	printf -- "\n"
	printf -- "--max-loss-pct[s]=fl,[fl]\n"
	printf -- "  A list of maximum allowed percentage of dropped frames, used for binary search\n"
	printf -- "  Default is ${max_loss_pcts_def}\n"
	printf -- "\n"
	printf -- "--frame-sizes=str[,str]\n"
	printf -- "  A list of Ethernet frame sizes (including CRC) in bytes\n"
	printf -- "  Default is ${frame_sizes_def}\n"
	printf -- "\n"
	printf -- "--num-flows=int\n"
	printf -- "  Number of packet flows to run\n"
	printf -- "  Default is ${num_flows_def}\n"
	printf -- "\n\n"
	#                   1         2         3         4         5         6         7         8
	#          12345678901234567890123456789012345678901234567890123456789012345678901234567890
	printf -- "trafficgen options that control packet contents\n"
	printf -- "-----------------------------------------------\n"
	printf -- "\n"
	printf -- "    The values provided for the following options correspond to the devices\n"
	printf -- "    device list (--devices).  For example\n"
	printf -- "\n"
	printf -- "      --devices=0000:04:00.0,0000:04:00.1 --src-ips=10.0.0.1,8.0.0.1\n"
	printf -- "\n"
	printf -- "    In the example above, device 0000:04:00.0 would use source IP address 10.0.0.1\n"
	printf -- "    and device 0000:04:00.1 would use source IP adress 8.0.0.1\n"
	printf -- "\n"
	printf -- "--src-ports=PORT,PORT\n"
	printf -- "  A list of source ports\n"
	printf -- "\n"
	printf -- "--dst-ports=PORT,PORT\n"
	printf -- "  A list of destination ports\n"
	printf -- "\n"
	printf -- "--src-macs=MAC,MAC\n"
	printf -- "  A list of two source MAC addresses\n"
	printf -- "\n"
	printf -- "--src-ips=IP,IP\n"
	printf -- "  A list of two source IP addresses\n"
	printf -- "\n"
	printf -- "--dst-macs=MAC,MAC\n"
	printf -- "  A list of two destination MAC addresses\n"
	printf -- "\n"
	printf -- "--dst-ips=IP,IP\n"
	printf -- "  A list of two destination IP addresses\n"
	printf -- "\n"
	printf -- "--encap-src-macs=MAC,MAC\n"
	printf -- "  A list of two source MAC addresses for the encapsulated network. Only used in\n"
	printf -- "  conjuction with an overlay network like VxLAN\n"
	printf -- "\n"
	printf -- "--encap-src-ips=IP,IP\n"
	printf -- "  A list of two source IP addresses for the encapsulated network.  Only used in\n"
	printf -- "  conjuction with an overlay network like VxLAN.\n"
	printf -- "\n"
	printf -- "--encap-dst-macs=MAC,MAC\n"
	printf -- "  A list of two destination MAC addresses for the encapsulated network.  Only\n"
	printf -- "  used on conjuction with an overlay network like VxLAN.\n"
	printf -- "\n"
	printf -- "--encap-dst-ips=IP,IP\n"
	printf -- "  A list of two destination IP addresses for the encapsulated network.  Only\n"
	printf -- "  used on conjuction with an overlay network like VxLAN.\n"
	printf -- "\n"
	printf -- "--vlan-ids=[int][,int]\n"
	printf -- "  If a value is provided, the corresponding device will use a VLAN tag when\n"
	printf -- "  transmitting and expect received packets to also have the VLAN tag\n"
	printf -- "\n"
	printf -- "--overlay-ids=[int][,int]\n"
	printf -- "  If a value is provided, the corresponding device will encapsulate the packet\n"
	printf -- "  and use VNI = value provided when transmitting packets.  Received packets\n"
	printf -- "  for the this device are expected to be encapsulated and use same VNI.  This\n"
	printf -- "  option must be used with --overlay-type option\n"
	printf -- "\n"
	#                   1         2         3         4         5         6         7         8
	#          12345678901234567890123456789012345678901234567890123456789012345678901234567890
	printf -- "--overlay-types\n"
	printf -- "  Per device specification of overlay network type, can be \"none\", or skipped,\n"
	printf -- "  or \"VxLAN\".  If \"VxLAN\" is used, --overlay-ids must also be used as well\n"
	printf -- "  as --encap-[src|dst]-[macs|ips] options.  When using an overlay, the\n"
	printf -- "  device-under-test must have the matching configuration.  For example, if\n"
	printf -- "  --overlay-types=none,vxlan is used, the device-under-test would receive\n"
	printf -- "  non-overlay packets on its first device, encapsulate the packets, and send\n"
	printf -- "  the packet out the second device.  Packets received by the second device are\n"
	printf -- "  expected to already be encapsulated, and the device-under-test would\n"
	printf -- "  decapsulate them and send them out the first device.\n"
	printf -- "\n"
	printf -- "--flow-mods=str[,str] (default is ${flow_mods_def})\n"
	printf -- "  A list of IP packet header fields which are used to implement unique flows.\n"
	printf -- "  All fields listed here are changed in unison, for example if \"src-ip,src-mac\"\n"
	printf -- "  are used, the first packet will have a source IP of 192.168.0.X and source MAC\n"
	printf -- "  of 01:02:03:04:X:06.  The second packet will have a source IP of\n"
	printf -- "  192.168.0.(X+1) and source MAC of 01:02:03:04:(X+1):06. The list of available\n"
	printf -- "  flow mods are:\n\n"
	printf -- "    src-ip,dst-ip,src-mac,dst-mac,src-port,dst-port,encap-src-mac,encap-dst-mac,\n"
	printf -- "    encap-src-ip,encap-dst-ip,protocol,none\n\n"
	printf -- "  Note that none is a special flow mod option.  It's presence will override and\n"
	printf -- "  disable all other specified flow mods\n"
	printf -- "  Note that the encap-* flow mods are only viable for overlay network tests like\n"
	printf -- "  VxLAN, and alter the inner packet header.\n"
	printf -- "  The maximum number of unique flows may be limited by the traffic-generator used.\n"
	printf -- "  For trex-txrx, it is 16.7 million.  Trex-txrx modifies the 2nd, 3rd, and 4th\n"
	printf -- "  octet in the IP addresses and the 3rd, 4th, and 5th octet in the MAC addresses\n"
	printf -- "  When using source or destination ports numbers, the range is limited to 1-32768\n"
	printf -- "  and higher flow counts will \"roll\" over back to 1\n"
	printf -- "\n"
	printf -- "--packet-protocol=str (default is ${packet_protocol_def})\n"
	printf -- "  The protocol to use when generating IP packets.  Available options are TCP or\n"
	printf -- "  UDP.\n"
	printf -- "\n"
	printf -- "--trex-use-ht\n"
	printf -- "  Should TRex be allowed to use Hyperthreading siblings.  Defaults to disabled.\n"
	printf -- "--trex-use-l2\n"
	printf -- "  Should TRex be configured in L2 mode instead of L3.  Defaults to disabled.\n"
	printf -- "\n\n"
	printf -- "trafficgen debug options\n"
	printf -- "-----------------------------------------------\n"
	printf -- "\n"
	printf -- "--skip-trex-install\n"
	printf -- "  Do no install TRex server process (assumes you have one) installed already\n"
	printf -- "\n"
	printf -- "--skip-trex-server\n"
	printf -- "  Do no kill existing or start a new TRex server process (assumes you have one)\n"
	printf -- "  running already)\n"
	printf -- "\n"
	printf -- "--skip-git-pull\n"
	printf -- "  Do not call git pull on the trafficgen repo if it already exists\n"
	printf -- "\n\n"
	printf -- "options common in most pbench benchmark scripts\n"
	printf -- "-----------------------------------------------\n"
	printf -- "\n"
	printf -- "--config=str\n"
	printf -- "  Name of the test config (i.e. jumbo_frames_and_network_throughput)\n"
	printf -- "\n"
	printf -- "--samples=int\n"
	printf -- "  The number of times each different test is run (to compute average)\n"
	printf -- "\n"
	printf -- "--max-failures=int\n"
	printf -- "  The maximm number of failures to get below stddev\n"
	printf -- "\n"
	printf -- "--max-stddev=int\n"
	printf -- "  The maximm percent stddev allowed to pass\n"
	printf -- "\n"
	printf -- "--postprocess-only=y|n\n"
	printf -- "  Don't run the benchmark, but postprocess data from previous test\n"
	printf -- "\n"
	printf -- "--run-dir=str\n"
	printf -- "  Optionally specify what directory should be used (usually only used if\n"
	printf -- "  postprocess-only=y)\n"
	printf -- "\n"
	printf -- "--start-iteration-num=int\n"
	printf -- "  Optionally skip the first (n-1) tests\n"
	printf -- "\n"
	printf -- "--tool-group=str\n"
	printf -- "  Start/stop/post-process tools using this group\n"
	printf -- "\n"
	printf -- "--pre-sample-cmd=str\n"
	printf -- "  run this command before executing each test sample\n"
	printf -- "\n"
	printf -- "--sysinfo=str,\n"
	printf -- "  comma separated values of sysinfo to be collected, default=\"none\"\n"
	printf -- "    available: $(pbench-collect-sysinfo --options)\n"
	printf -- "\n"
	printf -- "--tool-period=str\n"
	printf -- "  str = [binary-search|repeat-final-validation]\n"
	printf -- "  binary-search (default): tools are collected during binary-search\n"
	printf -- "  repeat-final-validation: tools are collected during a repeat-run of the final validation trial\n"
}

# Process options and arguments
opts=$(getopt -q -o h --longoptions "pre-benchmark-cmd:,skip-trex-install,skip-trex-server,skip-git-pull,traffic-generator:,devices:,active-devices:,rate-unit:,rate:,rates:,rate-tolerance-pct:,max-loss-pct:,max-loss-pcts:,start-iteration-num:,config:,num-flows:,test-type:,traffic-direction:,traffic-directions:,sniff-runtime:,search-runtime:,validation-runtime:,frame-size:,frame-sizes:,samples:,max-stddev:,max-failures:,postprocess-only:,run-dir:,tool-group:,one-shot,src-ports:,dst-ports:,src-macs:,src-ips:,dst-macs:,dst-ips:,encap-src-macs:,encap-src-ips:,encap-dst-macs:,encapdst-ips:,vlan-ids:,overlay-ids:,overlay-types:,flow-mods:,packet-protocol:,trex-use-ht,trex-use-l2,traffic-profile:,traffic-profiles:,sysinfo:,tool-period:,help" -n "getopt.sh" -- "$@")
if [ $? -ne 0 ]; then
	printf -- "%s %s\n\n\tunrecognized option specified\n\n" "${script_name}" "${*}" >&2
	usage >&2
	exit 1
fi
eval set -- "$opts"
while true; do
	case "$1" in
	--tool-label-pattern)
		shift
		if [ -n "$1" ]; then
			tool_label_pattern="$1"
			shift
		fi
		;;
	--postprocess-only)
		shift
		if [ -n "$1" ]; then
			postprocess_only="$1"
			shift
		fi
		;;
	--run-dir)
		shift
		if [ -n "$1" ]; then
			benchmark_run_dir="$1"
			shift
		fi
		;;
	--max-stddev)
		shift
		if [ -n "$1" ]; then
			maxstddevpct="$1"
			shift
		fi
		;;
	--max-failures)
		shift
		if [ -n "$1" ]; then
			max_failures="$1"
			shift
		fi
		;;
	--samples)
		shift
		if [ -n "$1" ]; then
			num_samples="$1"
			shift
		fi
		;;
	--traffic-generator)
		shift
		if [ -n "$1" ]; then
			traffic_generator="$1"
			shift
		fi
		;;
	--num-flows)
		shift
		if [ -n "$1" ]; then
			num_flows="$1"
			shift
		fi
		;;
	--rate-unit)
		shift
		if [ -n "$1" ]; then
			rate_unit="$1"
			shift
		fi
		;;
	--rate|--rates)
		shift
		if [ -n "$1" ]; then
			rates="$1"
			shift
		fi
		;;
	--rate-tolerance-pct)
		shift
		if [ -n "$1" ]; then
			rate_tolerance_pct="$1"
			shift
		fi
		;;
	--max-loss-pct|--max-loss-pcts)
		shift
		if [ -n "$1" ]; then
			max_loss_pcts="$1"
			shift
		fi
		;;
	--traffic-direction|--traffic-directions)
		shift
		if [ -n "$1" ]; then
			traffic_directions="$1"
			shift
		fi
		;;
	--tool-group)
		shift
		if [ -n "$1" ]; then
			tool_group="$1"
			shift
		fi
		;;
	--frame-size|--frame-sizes)
		shift
		if [ -n "$1" ]; then
			frame_sizes="$1"
			shift
		fi
		;;
	--devices)
		shift
		if [ -n "$1" ]; then
			devices="$1"
			shift
		fi
		;;
	--active-devices)
		shift
		if [ -n "${1}" ]; then
			active_devices="${1}"
			shift
		fi
		;;
	--config)
		shift
		if [ -n "$1" ]; then
			config="$1"
			shift
		fi
		;;
	--start-iteration-num)
		shift
		if [ -n "$1" ]; then
			start_iteration_num=$1
			shift
		fi
		;;
	--pre-benchmark-cmd)
		shift
		if [ -n "$1" ]; then
			pre_benchmark_cmd=$1
			shift
		fi
		;;
	--tool-period)
		shift
		if [ -n "$1" ]; then
			tool_period=$1
			shift
		fi
		;;
	--one-shot)
		shift
		one_shot="y"
		;;
	--trex-use-ht)
		shift
		trex_use_ht="y"
		;;
	--trex-use-l2)
		shift
		trex_use_l2="y"
		;;
	--flow-mods)
		shift
		if [ -n "$1" ]; then
			flow_mods="$1"
			shift
		fi
		;;
	--validation-runtime|--sniff-runtime|--search-runtime)
		arg="$1"
		shift
		if [ -n "$1" ]; then
			val=$1
			var=`echo $arg | sed -e s/^--// -e s/-/_/g`
			eval $var=$val
			shift
		fi
		;;
		# for options which if not used, do not affect the binary-search.py options (and binary-search should not use them by default)
		# these options must match the options to binary-search.py *exactly*
	--overlay-ids|--vlan-ids|--overlay-types|--src-ips|--src-macs|--dst-ips|--dst-macs|--encap-src-ips|--encap-src-macs|--encap-dst-ips|--encap-dst-macs|--src-ports|--dst-ports|--packet-protocol)
		arg="$1"
		shift
		if [ -n "$1" ]; then
			val=$1
			cmd="$cmd $arg=$val"
			var=`echo $arg | sed -e s/^--// -e s/-/_/g`
			eval $var=$val
			shift
		fi
		;;
	--skip-trex-install)
		shift
		skip_trex_install="y"
		;;
	--skip-trex-server)
		shift
		skip_trex_server="y"
		;;
	--skip-git-pull)
		shift
		skip_git_pull="y"
		;;
	--traffic-profile|--traffic-profiles)
		shift
		if [ -n "$1" ]; then
			traffic_profiles="$1"
			shift
		fi
		;;
	--sysinfo)
		shift;
		if [ -n "$1" ]; then
			sysinfo="$1"
			shift;
		fi
		;;
	-h|--help)
		usage
		exit 0
		;;
	--)
		shift
		while true; do
			arg="$1"
			shift
			if [ -n "$arg" ]; then
				passthrough_arguments="${passthrough_arguments} ${arg}"
			else
				break
			fi
		done
		break
		;;
	*)
		error_log "[$script_name] bad option, \"$1 $2\""
		usage >&2
		exit 1
		;;
	esac
done
verify_common_bench_script_options $tool_group $sysinfo

let errors=0
for traffic_profile in `echo $traffic_profiles | sed -e s/,/" "/g`; do
	if [[ ! -e "${traffic_profile}" ]]; then
		error_log "Traffic profile, '${traffic_profile}', does not exist"
		(( errors++ ))
	fi
done
if [[ ${errors} -ne 0 ]]; then
	exit 1
fi

if [ "$traffic_generator" == "trex-txrx" -o "$traffic_generator" == "moongen-txrx" -o "$traffic_generator" == "trex-txrx-profile" ]; then
	cmd="$cmd --traffic-generator=$traffic_generator --rate-tolerance=$rate_tolerance_pct"
else
	error_log "The traffic engine is not valid: $traffic_generator"
	exit 1
fi

for num_flow in `echo $num_flows | sed -e 's/,/ /g'`; do
	if ! [ $num_flow -gt 0 -a $num_flow -le 16777216 ]; then
		error_log "This number of flows is invalid: $num_flow"
		exit 1
	fi
done

not_flow_mods="$all_possible_flow_mods"
profile_flow_mods=""
if ! echo "$flow_mods" | grep -e "^none$" -e ",none$" -e "^none," -e ",none,"; then
	for flow_mod in `echo $flow_mods | sed -e 's/,/ /g'`; do
		found_flow_mod=0
		for this_flow_mod in `echo $all_possible_flow_mods | sed -e 's/,/ /g'`; do
			if [ $flow_mod = $this_flow_mod ]; then
				if [ "$traffic_generator" == "trex-txrx-profile" ]; then
					profile_flow_mods="${profile_flow_mods} --use-${flow_mod}-flows"
				else
					cmd="$cmd --use-${flow_mod}-flows=1"
				fi
				found_flow_mod=1
				not_flow_mods=`echo $not_flow_mods | sed -e s/^$flow_mod,// -e s/,$flow_mod,/,/ -e s/,$flow_mod//`
			fi
		done
		if [ $found_flow_mod -eq 0 ]; then
			error_log "This flow_mod is invalid: $flow_mod"
			exit 1
		fi
	done
fi
if [ "$traffic_generator" != "trex-txrx-profile" ]; then
	# ensure the other flows are forced off (avoiding any defaults from binary-search.py)
	for flow_mod in `echo $not_flow_mods | sed -e 's/,/ /g'`; do
		if [ "${flow_mod}" == "none" ]; then
			continue
		else
			cmd="$cmd --use-${flow_mod}-flows=0"
		fi
	done
fi

if [ $one_shot == "y" ]; then
	cmd="$cmd --one-shot=1"
fi

if [[ "$tool_period" == "binary-search" ]]; then
	start_trigger="Starting binary-search"
	stop_trigger="Finished binary-search"
elif [[ "$tool_period" == "repeat-final-validation" ]]; then
	cmd="$cmd --repeat-final-validation"
	start_trigger="Starting repeat final validation for debug collection"
	stop_trigger="Finished repeat final validation for debug collection"
else
	error_log "invalid --tool-period argument, '${tool_period}'"
	exit 1
fi

if [ "${postprocess_only}" == "n" ]; then
	# Before running any tests, the following must be satisfied:
	# - IOMMU must be enabled
	if cat /proc/cmdline | grep -q "intel_iommu=on"; then
		echo "found IOMMU option, continuing"
	else
		error_log "could not find IOMMU option, intel_iommu=on, in /proc/cmdline, exiting"
		exit 1
	fi
	# - dpdk-tools package must be installed
	if check_install_rpm dpdk-tools; then
		echo "found dpdk-tools, continuing"
	else
		error_log "could not find dpdk-tools, exiting"
		exit 1
	fi
	# - PCI devices bound to vfio-pci
	if [ -z "$devices" ]; then
		error_log "no devices were provided (--devices), exiting"
		exit 1
	fi
	# - vfio and vfio_pci kernel modules must be loaded
	for module in vfio vfio_pci; do
		load_module ${module}
	done

	total_devices=$(echo ${devices} | sed -e "s/,/ /g" | wc -w)
	if [ $(echo "${total_devices} % 2" | bc) != 0 ]; then
		error_log "devices must be supplied in quantities of 2, exiting"
		exit 1
	fi
	total_device_pairs=$(echo "${total_devices} / 2" | bc)
	if [ $traffic_generator == "trex-txrx" -o $traffic_generator == "trex-txrx-profile" ]; then
		device_pair_index=0
		cmd+=" --device-pairs="
		for device in $(seq 1 2 ${total_devices}); do
			device_a=$(( device - 1))
			device_b=${device}
			cmd+="${device_a}:${device_b}"
			(( device_pair_index++ ))
			if [ ${device_pair_index} -lt ${total_device_pairs} ]; then
				cmd+=","
			fi
		done
	else
		if [ "${total_device_pairs}" -gt 1 ]; then
			error_log "only the trex-txrx* traffic generators support more than 1 pair of devices, exiting"
			exit 1
		fi
	fi

	if [ -z "${active_devices}" ]; then
		active_devices=${devices}
	fi

	total_active_devices=$(echo ${active_devices} | sed -e "s/,/ /g" | wc -w)
	if [ $(echo "${total_active_devices} % 2" | bc) -ne 0 ]; then
		error_log "active devices must be supplied in quantities of 2, exiting"
		exit 1
	fi
	if $(echo $traffic_generator | grep -Eq "^trex-txrx") && [ "${devices}" != "${active_devices}" ]; then
		error_log "only the trex-txrx* traffic generators support --devices != --active-devices, exiting"
		exit 1
	fi
	if [ $traffic_generator == "trex-txrx" -o $traffic_generator == "trex-txrx-profile" ]; then
		active_device_pair_index=0
		cmd+=" --active-device-pairs="
		active_device_state=0
		active_devices_processed=0
		for active_device in `echo $active_devices | sed -e s/,/" "/g`; do
			device_index=0
			active_device_index=-1
			for device in `echo $devices | sed -e s/,/" "/g`; do
				if [ "${device}" == "${active_device}" ]; then
					active_device_index=${device_index}
				fi
				(( device_index++ ))
			done
			(( active_devices_processed++ ))
			if [ ${active_device_index} -eq -1 ]; then
				error_log "couldn't find an active device [${active_device}] in the list of devices [${devices}], exiting"
				exit 1
			fi
			if [ ${active_device_state} -eq 0 ]; then
				cmd+="${active_device_index}:"
				(( active_device_state++ ))
			elif [ ${active_device_state} -eq 1 ]; then
				cmd+="${active_device_index}"
				active_device_state=0
				if [ ${active_devices_processed} -lt ${total_active_devices} ]; then
					cmd+=","
				fi
			fi
		done
	fi

	# Ensure all specified devices are properly configured via dpdk.
	configure_devices "${devices}"

	if [ -d $trafficgen_dir ]; then
		if [ $skip_git_pull == "n" ]; then
			if pushd $trafficgen_dir >/dev/null && git pull && git checkout master; then
				echo "trafficgen is up to date"
				trafficgen_version=$(git log --max-count=1 --pretty=%h)
				popd >/dev/null
			else
				error_log "could not pull latest trafficgen"
				exit 1
			fi
		fi
	else
		if pushd /opt >/dev/null && git clone https://github.com/perftool-incubator/bench-trafficgen.git; then
			trafficgen_version=$(git log --max-count=1 --pretty=%h)
			echo "trafficgen cloned"
			popd >/dev/null
		else
			error_log "could not clone latest trafficgen"
			exit 1
		fi
	fi

	# Moongen is built under the trafficgen repo
	if [ $traffic_generator == "moongen-txrx" ]; then
		if ! [ -x $trafficgen_dir/MoonGen/build/MoonGen ]; then
			pushd >/dev/null $trafficgen_dir
			echo "building MoonGen"
			if ./setup.sh; then
				echo "MoonGen build complete"
			else
				error_log "could not build MoonGen"
				exit 1
			fi
		fi
	fi

	if [ $traffic_generator == "trex-txrx" -o $traffic_generator == "trex-txrx-profile" ]; then
		if [ $skip_trex_server == "n" ]; then
			start_trex "${skip_trex_install}" "${trex_use_ht}" "${trex_use_l2}" "${devices}"
		fi
	fi
fi

###
# At this point all parameter checking has been done.
###

# Build up a result directory identifier from parameters
tg_config="tg:$(echo ${traffic_generator} | sed -e 's/-txrx//')"
if [ $traffic_generator == "trex-txrx-profile" -a "${traffic_profiles}" != "" ]; then
	traffic_profiles_list=""
	for traffic_profile in $(echo ${traffic_profiles} | sed -e s/,/" "/g); do
		traffic_profile=$(basename ${traffic_profile})
		if [ -n "${traffic_profiles_list}" ]; then
			traffic_profiles_list+=","
		fi
		traffic_profiles_list+="${traffic_profile}"
	done
	tg_config="${tg_config}_pf:${traffic_profiles_list}"
else
	tg_config="${tg_config}_r:${rates}"
	tg_config="${tg_config}_fs:${frame_sizes}"
	tg_config="${tg_config}_nf:${num_flows}"
	tg_config="${tg_config}_fm:$(echo ${flow_mods} | sed -e 's/src-/s/g' -e 's/dst-/d/g' -e 's/ip/i/g' -e 's/mac/m/g' -e 's/port/p/g')"
	tg_config="${tg_config}_td:$(echo ${traffic_directions} | sed -e 's/directional//g')"
fi
tg_config="${tg_config}_ml:${max_loss_pcts}"
if [ "$one_shot" == "y" ]; then
	tg_config="${tg_config}_tt:os"
else
	tg_config="${tg_config}_tt:bs"
fi
# add test options to $config, so this test result directory is easily identifiable
if [[ -z "${config}" ]]; then
	config="${tg_config}"
else
	config="${config}_${tg_config}"
fi

if [[ -z "$benchmark_run_dir" ]]; then
	# We don't have an explicit run directory, construct one
	benchmark_fullname="${benchmark_name}_${config}_${date}"
	benchmark_run_dir="$pbench_run/${benchmark_fullname}"
else
	# We have an explicit run directory provided by --run-dir, so warn
	# the user if they also used --config
	if [[ ! -z "$config" ]]; then
		warn_log "[$script_name] ignoring --config=\"$config\" in favor of --rundir=\"$benchmark_run_dir\""
	fi
	benchmark_fullname=$(basename $benchmark_run_dir)
fi

# we'll record the iterations in this file
benchmark_iterations="${benchmark_run_dir}/.iterations"
mdlog=${benchmark_run_dir}/metadata.log

function record_default_iteration {
	local count=$1
	local rate=$2
	local direction=$3
	local frame_size=$4
	local num_flow=$5
	local loss=$6
	local iteration=$7

	echo ${iteration} >> ${benchmark_iterations}
	echo $count | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_number
	echo $rate | pbench-add-metalog-option ${mdlog} iterations/${iteration} rate
	echo $direction | pbench-add-metalog-option ${mdlog} iterations/${iteration} traffic_direction
	echo $frame_size | pbench-add-metalog-option ${mdlog} iterations/${iteration} frame_size
	echo $num_flow | pbench-add-metalog-option ${mdlog} iterations/${iteration} num_flow
	echo $loss | pbench-add-metalog-option ${mdlog} iterations/${iteration} max_loss_pct
	echo $iteration | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_name
}

function record_profile_iteration {
	local count=$1
	local profile=$2
	local loss=$3
	local iteration=$4

	echo ${iteration} >> ${benchmark_iterations}
	echo ${count} | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_number
	echo ${profile} | pbench-add-metalog-option ${mdlog} iterations/${iteration} profile
	echo ${loss} | pbench-add-metalog-option ${mdlog} iterations/${iteration} max_loss_pct
	echo ${iteration} | pbench-add-metalog-option ${mdlog} iterations/${iteration} iteration_name
}

if [ "${postprocess_only}" == "n" ]; then
	mkdir -p $benchmark_run_dir/.running

	# now that the benchmark_run_dir directory exists, we can initialize the iterations file
	> ${benchmark_iterations}

	# save a copy of the command, in case the test needs to be reproduced or post-processed again
	echo "${script_name} ${orig_cmd}" > ${benchmark_run_dir}/${script_name}.cmd
	chmod +x ${benchmark_run_dir}/${script_name}.cmd
fi

## Run the benchmark and start/stop perf analysis tools

total_iterations=0
if [ $traffic_generator == "trex-txrx-profile" -a "${traffic_profiles}" != "" ]; then
	for traffic_profile in `echo $traffic_profiles | sed -e s/,/" "/g`; do
		for max_loss_pct in `echo $max_loss_pcts | sed -e s/,/" "/g`; do
			((total_iterations++))
		done
	done
else
	for rate in `echo $rates | sed -e s/,/" "/g`; do
		for traffic_direction in `echo $traffic_directions | sed -e s/,/" "/g`; do
			for frame_size in `echo $frame_sizes | sed -e s/,/" "/g`; do
				for num_flow in `echo $num_flows | sed -e s/,/" "/g`; do
					for max_loss_pct in `echo $max_loss_pcts | sed -e s/,/" "/g`; do
						((total_iterations++))
					done
				done
			done
		done
	done
fi
echo "Total number of benchmark iterations: $total_iterations"

if [ "${postprocess_only}" == "n" ]; then
	# Register the tool triggers depending on the tool period preference
	# FIXME - tool triggers should only be registered for the duration of the benchmark run.
	pbench-register-tool-trigger --group="${tool_group}" --start-trigger="${start_trigger}" --stop-trigger="${stop_trigger}"

	pbench-collect-sysinfo --group=${tool_group} --dir=${benchmark_run_dir} --sysinfo=${sysinfo} beg
	pbench-metadata-log --group=${tool_group} --dir=${benchmark_run_dir} beg
	# on abnormal exit, make sure that the metadata log exists and is complete.
	trap "pbench-metadata-log --group=${tool_group} --dir=${benchmark_run_dir} int" INT QUIT

	if [ $traffic_generator == "trex-txrx" -o $traffic_generator == "trex-txrx-profile" ]; then
		if [ $skip_trex_server == "n" ]; then
			cp ${pbench_tmp}/trex_cfg.yaml ${benchmark_run_dir}
		fi
	fi
fi

function echo_log {
	echo "${1}"
	log "${1}"
}

pre_loop_cmd=${cmd}

function inner_most_loop() {
	local loop_type=${1}
	local traffic_profile_arg=${2}
	local rate_arg=${3}
	local traffic_direction_arg=${4}
	local frame_size_arg=${5}
	local num_flow_arg=${6}

	if [[ "${loop_type}" == "default" ]]; then
		local fs_unit="B"
		unset traffic_profile_arg
	elif [[ "${loop_type}" == "profile" ]]; then
		local traffic_profile_name=$(basename ${traffic_profile_arg})
		unset rate_arg
		unset traffic_direction_arg
		unset frame_size_arg
		unset num_flow_arg
	else
		error_log "INTERNAL ERROR - unexpected loop type, '${loop_type}'"
		exit 1
	fi

	for max_loss_pct in `echo $max_loss_pcts | sed -e s/,/" "/g`; do
		if [[ ${rc} -ne 0 ]]; then
			break
		fi

		let count=${count}+1 # now we can move to the next iteration

		if [ "${loop_type}" == "default" ]; then
			iteration="${count}-${rate_arg}-${traffic_direction_arg}-${frame_size_arg}${fs_unit}-${num_flow_arg}flows-${max_loss_pct}pct_drop"
		else
			iteration="${count}-${traffic_profile_name}-${max_loss_pct}pct_drop"
		fi

		if [[ ${count} -lt ${start_iteration_num} ]]; then
			echo_log "Skipping iteration ${iteration} (${count} of ${total_iterations})"
			continue
		fi

		if [ "${postprocess_only}" == "n" ]; then
			if [ "${loop_type}" == "default" ]; then
				record_default_iteration "${count}" "${rate_arg}" "${traffic_direction_arg}" "${frame_size_arg}${fs_unit}" "${num_flow_arg}flows" "${max_loss_pct}pct_drop" "${iteration}"
			else
				record_profile_iteration "${count}" "${traffic_profile_name}" "${max_loss_pct}pct_drop" "${iteration}"
			fi
		fi

		# reset the command to run
		cmd=${pre_loop_cmd}

		# if any of the runtimes are provided by the user, then use those
		unset iteration_sniff_runtime
		unset iteration_search_runtime
		unset iteration_validation_runtime
		if [ ! -z "$sniff_runtime" ]; then
			iteration_sniff_runtime=$sniff_runtime
		fi
		if [ ! -z "$search_runtime" ]; then
			iteration_search_runtime=$search_runtime
		fi
		if [ ! -z "$validation_runtime" ]; then
			iteration_validation_runtime=$validation_runtime
		fi
		# for any runtimes not provided by user, assign a default value
		# there are different defaults for 0-loss and non-zero-loss
		if [ -z "$iteration_sniff_runtime" ]; then
			if [[ $(echo "if (${max_loss_pct} == 0) 1 else 0" | bc) -eq 1 ]]; then
				iteration_sniff_runtime=$df_zl_sniff_runtime
			else
				iteration_sniff_runtime=$df_sniff_runtime
			fi
		fi
		if [ -z "$iteration_search_runtime" ]; then
			if [[ $(echo "if (${max_loss_pct} == 0) 1 else 0" | bc) -eq 1 ]]; then
				iteration_search_runtime=$df_zl_search_runtime
			else
				iteration_search_runtime=$df_search_runtime
			fi
		fi
		if [ -z "$iteration_validation_runtime" ]; then
			if [[ $(echo "if (${max_loss_pct} == 0) 1 else 0" | bc) -eq 1 ]]; then
				iteration_validation_runtime=$df_zl_validation_runtime
			else
				iteration_validation_runtime=$df_validation_runtime
			fi
		fi
		cmd="$cmd --sniff-runtime=$iteration_sniff_runtime"
		cmd="$cmd --search-runtime=$iteration_search_runtime"
		cmd="$cmd --validation-runtime=$iteration_validation_runtime"

		tt_iteration_dir="${benchmark_run_dir}/${count}-default"
		iteration_dir="$benchmark_run_dir/$iteration"

		if [ "${postprocess_only}" == "n" ]; then
			mkdir -p $iteration_dir

			local iteration_options="--max-loss-pct=${max_loss_pct}"
			if [[ ${traffic_generator} == "trex-txrx-profile" ]]; then
				if [[ "${traffic_profile_arg}" == "" || "${rates}" == "none" ]]; then
					iteration_options="${iteration_options} --rate-unit=% --rate=100"
				else
					iteration_options="${iteration_options} --rate-unit=% --rate=${rates}"
				fi
				if [[ "${traffic_profile_arg}" == "" ]]; then
					profile_builder_cmd_file="${iteration_dir}/profile-builder.cmd"

					echo "pushd ${trafficgen_dir} > /dev/null" > ${profile_builder_cmd_file}
					echo "./profile-builder.py --frame-size=${frame_size_arg} --num-flows=${num_flow_arg} --rate=${rate_arg} ${profile_flow_mods} --traffic-direction=${traffic_direction_arg} --measure-latency" >> ${profile_builder_cmd_file}
					echo "cmd_rc=\${?}" >> ${profile_builder_cmd_file}
					echo "popd > /dev/null" >> ${profile_builder_cmd_file}
					echo "exit \${cmd_rc}" >> ${profile_builder_cmd_file}
					chmod +x ${profile_builder_cmd_file}

					traffic_profile_file="${iteration_dir}/traffic-profile.json"

					${profile_builder_cmd_file} > ${traffic_profile_file} 2>&1
				else
					cp ${traffic_profile_arg} ${iteration_dir}/
					traffic_profile_file="${iteration_dir}/${traffic_profile_name}"
				fi
				iteration_options="${iteration_options} --traffic-profile=${traffic_profile_file}"
			else
				if [[ ${rate_arg} == "none" ]]; then
					if [[ ${traffic_generator} == "trex-txrx" ]]; then
						iteration_options="${iteration_options} --rate-unit=% --rate=100"
					else
						# don't specify a rate and let binary-search.py + whatever traffic-generator figure it out
						iteration_options="${iteration_options}"
					fi
				else
					iteration_options="${iteration_options} --rate-unit=${rate_unit} --rate=${rate_arg}"
				fi
				iteration_options="${iteration_options} --traffic-direction=${traffic_direction_arg}"
				iteration_options="${iteration_options} --frame-size=${frame_size_arg}"
				iteration_options="${iteration_options} --num-flows=${num_flow_arg}"
			fi

			# save benchmark command in file for debugging or running manually
			benchmark_cmd_file="${iteration_dir}/${benchmark_name}.cmd"
			echo "pushd ${trafficgen_dir} > /dev/null" > ${benchmark_cmd_file}
			echo "${cmd} ${iteration_options} ${passthrough_arguments} \${@}" >> ${benchmark_cmd_file}
			echo "cmd_rc=\${?}" >> ${benchmark_cmd_file}
			echo "popd > /dev/null" >> ${benchmark_cmd_file}
			echo "exit \${cmd_rc}" >> ${benchmark_cmd_file}
			chmod +x ${benchmark_cmd_file}
		fi

		echo_log "Starting iteration[$iteration] ($count of $total_iterations)"

		result_stddevpct=$maxstddevpct # this test case will get a "do-over" if the stddev is not low enough
		failures=0
		while [[ $(echo "if (${result_stddevpct} >= ${maxstddevpct}) 1 else 0" | bc) -eq 1 ]]; do
			if [[ ${failures} -gt 0 ]]; then
				echo_log "Restarting iteration[$iteration] ($count of $total_iterations)"
			fi
			# each attempt at a test config requires multiple samples to get stddev
			for sample in `seq 1 $num_samples`; do
				if [ $rc -ne 0 ]; then
					break
				fi
				benchmark_results_dir="$iteration_dir/sample$sample"
				tt_benchmark_results_dir="${tt_iteration_dir}/sample${sample}"
				if [ "$postprocess_only" != "y" ]; then
					mkdir -p $benchmark_results_dir
					echo_log "test sample $sample of $num_samples"
					pre_benchmark_cmd_log=$benchmark_results_dir/pre-benchmark-cmd-log.txt
					result_file=$benchmark_results_dir/result.txt
					if [ "$pre_benchmark_cmd" != "" ]; then
						PBENCH_ITERATION=$iteration PBENCH_SAMPLE=$sample $pre_benchmark_cmd 2>&1 | tee -i $pre_benchmark_cmd_log
					fi
					${benchmark_cmd_file} --output-dir=${benchmark_results_dir} 2>&1 | pbench-tool-trigger "${count}" "${benchmark_run_dir}" ${tool_group} ${sample} | tee -i ${result_file}
					rc=$?
					if [ $rc -ne 0 ]; then
						error_log "iteration ${iteration} sample ${sample} returned non-zero exit code, ${rc}"
						pbench-stop-tools --group=${tool_group} --dir=${tt_benchmark_results_dir}
					fi

					if [[ -e "${tt_benchmark_results_dir}/tools-${tool_group}" ]]; then
						# Post-process the collected tool data now
						pbench-postprocess-tools --group=${tool_group} --dir=${tt_benchmark_results_dir}

						# Move the tool data collected for the tool-group to its final resting place.
						mv ${tt_benchmark_results_dir}/tools-${tool_group} ${benchmark_results_dir}/
						if [[ ${?} -ne 0 ]]; then
							error_log "INTERNAL ERROR: failed to move tool-trigger data, '${tt_benchmark_results_dir}/tools-${tool_group}' to '${benchmark_results_dir}'"
							exit 1
						fi
						# The "tool-trigger" sample directory should be empty now.
						rmdir ${tt_benchmark_results_dir}
						if [[ ${?} -ne 0 ]]; then
							error_log "INTERNAL ERROR: failed to remove tool-trigger results directory, '${tt_benchmark_results_dir}'"
							exit 1
						fi
					else
						echo_log "tool triggers did not fire for iteration/sample, '${iteration}/sample${sample}'"
					fi
				else
					if [[ ! -d $benchmark_results_dir ]]; then
						error_log "Results directory $benchmark_results_dir does not exist, skipping post-processing"
						continue
					fi
					echo_log "Not going to run $benchmark_name.  Only postprocesing existing data"
				fi
				if [ $rc -eq 0 ]; then
					echo "${script_path}/postprocess/${benchmark_name}-postprocess \"${benchmark_results_dir}\" \"${full_hostname}\" \"${rate_arg}\" \"${frame_size_arg}\" \"${num_flow_arg}\" \"${traffic_profile_name}\" \"${trafficgen_version}\"" > "${benchmark_results_dir}/${benchmark_name}-postprocess.cmd"
					chmod +x "${benchmark_results_dir}/${benchmark_name}-postprocess.cmd"
					${benchmark_results_dir}/${benchmark_name}-postprocess.cmd 2>&1 | tee ${benchmark_results_dir}/${benchmark_name}-postprocess.out || rc=1
				fi
			done
			if [ $rc -eq 0 ]; then
				echo "$script_path/postprocess/process-iteration-samples \"$iteration_dir\" Mframes_sec \"$maxstddevpct\" \"$failures\" \"$max_failures\" \"$tar_nonref_data\" \"$keep_failed_tool_data\"" >"$iteration_dir/process-iteration-samples.cmd"
				chmod +x "$iteration_dir/process-iteration-samples.cmd"
				$iteration_dir/process-iteration-samples.cmd 2>&1 | tee $iteration_dir/process-iteration-samples.out
				fail=$?
				if [ $fail -ne 0 -a $fail -ne 1 ]; then
					error_log "process-iteration-samples generated an unknown error, aborting"
					exit 1
				fi
				if [ $fail -eq 1 ]; then
					error_log "This test iteration failed"
					((failures++))
				fi
				if [ $fail -eq 0 -o $failures -ge $max_failures ]; then
					debug_log "Moving to the next iteration"
					break
				fi
			else
				error_log "Aborting benchmark"
				break
			fi
		done # break out of this loop only if the $result_stddevpct is lower than $maxstddevpct
		if [ $rc -eq 0 ]; then
			echo_log "Iteration $iteration complete ($count of $total_iterations), with 1 pass and $failures failures"
		fi
		if [[ "$postprocess_only" != "y" && -e "${tt_iteration_dir}" ]]; then
			# Remove this iteration's "tool-trigger" directory.
			rmdir ${tt_iteration_dir}
			if [[ ${?} -ne 0 ]]; then
				error_log "INTERNAL ERROR: failed to remove tool-trigger directory, '${tt_iteration_dir}'"
				exit 1
			fi
		fi
	done
}

# Inner loop now pre-increments
count=0
rc=0
if [[ ${traffic_generator} == "trex-txrx-profile" && "${traffic_profiles}" != "" ]]; then
	for traffic_profile in `echo ${traffic_profiles} | sed -e s/,/" "/g`; do
		if [[ ${rc} -ne 0 ]]; then
			break
		fi

		inner_most_loop "profile" "${traffic_profile}" "no-rate" "no-traffic-direction" "no-frame-size" "no-num-flow"
	done
else
	for rate in `echo ${rates} | sed -e s/,/" "/g`; do
		if [[ ${rc} -ne 0 ]]; then
			break
		fi
		for traffic_direction in `echo ${traffic_directions} | sed -e s/,/" "/g`; do
			if [[ ${rc} -ne 0 ]]; then
				break
			fi
			for frame_size in `echo ${frame_sizes} | sed -e s/,/" "/g`; do
				if [[ ${rc} -ne 0 ]]; then
					break
				fi
				for num_flow in `echo ${num_flows} | sed -e s/,/" "/g`; do
					if [[ ${rc} -ne 0 ]]; then
						break
					fi

					inner_most_loop "default" "no-traffic-profile" "${rate}" "${traffic_direction}" "${frame_size}" "${num_flow}"
				done
			done
		done
	done
fi
if [ $rc -eq 0 ]; then
	echo "$script_path/postprocess/generate-benchmark-summary \"$benchmark_name\" \"$orig_cmd\" \"$benchmark_run_dir\"" >"$benchmark_run_dir/generate-benchmark-summary.cmd"
	chmod +x "$benchmark_run_dir/generate-benchmark-summary.cmd"
	"$benchmark_run_dir/generate-benchmark-summary.cmd"
fi

if [ "${postprocess_only}" == "n" ]; then
	if [ $traffic_generator == "trex-txrx" -o $traffic_generator == "trex-txrx-profile" ]; then
		if [ $skip_trex_server == "n" ]; then
			kill_trex

			if [ "${trex_use_l2}" == "y" ]; then
				mv -v /etc/trex_cfg.yaml.pbench-backup /etc/trex_cfg.yaml
			fi
		fi
	fi

	pbench-metadata-log --group=$tool_group --dir=$benchmark_run_dir end
	pbench-collect-sysinfo --group=$tool_group --dir=$benchmark_run_dir --sysinfo=$sysinfo beg

	rmdir $benchmark_run_dir/.running
fi
