#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; sh-indent-for-case-alt: + -*-

# This is a script to run the specjbb2005 benchmark
# Author: Andrew Theurer
#
# Note: you must have a license and access to SPECjbb2005 benchmark in order to run this
#       pbench does not provide this
#
# TODO:
# 1) write results in pbench standard file names and formats
# 2) add support for multiple local or remote copies of benchmark running concurrently
# 3) add support for binding copies of benchmark to numa nodes [in addition to binding certain JVMs within each copy]

script_path=`dirname $0`
script_name=`basename $0`
pbench_bin="`cd ${script_path}/..; /bin/pwd`"

# source the base script
. "$pbench_bin"/base

benchmark_rpm=$script_name
benchmark="specjbb2005"

# Every bench-script follows a similar sequence:
# 1) process bench script arguments
# 2) ensure the right version of the benchmark is installed
# 3) gather pre-run state
# 4) run the benchmark and start/stop perf analysis tools
# 5) gather post-run state
# 6) postprocess benchmark data
# 7) postprocess analysis tool data

# Defaults

runtime=30
nr_jvms=1
start_warehouses=1
stop_warehouses="default"
inc_warehouses=1
heap_size="4096m"
nr_cpus=`cat /proc/cpuinfo | grep processor | wc -l`
specjbb2005_dir=/usr/local/share/specjbb2005
tool_group="default"
config=""
sysinfo="default"
java_opts=""
dir=""

function usage {
	printf "\tThe following options are available:\n\n"
	printf -- "\t\t-C str --config=<str>            name of the test config\n"
	printf -- "\t\t-j str --java-opts=<str>         options passed directly to the JVM\n"
	printf -- "\t\t       --nr-jvms=<int>|node      number of JVMs. if = node, number of JVMs = number of NUMA nodes\n"
	printf -- "\t\t       --start-warehouses=<int>  number of warehouses to start with (default is $start_warehouses)\n"
	printf -- "\t\t       --inc-warehouses=<int>    number of warehouses to increment by (default is $inc_warehouses)\n"
	printf -- "\t\t       --stop-warehouses=<int>   number of warehouses to stop with (default is nr_cpus * 2 / nr_jvms)\n"
	printf -- "\t\t       --heap-size=<str>         size of the heap, java size spec (default is $heap_size)\n"
	printf -- "\t\t       --runtime=<int>           measurement period in seconds (default is $runtime)\n"
	printf -- "\t\t-d str --dir=<str>               directory to run the test\n"
	printf -- "\t\t       --tool-group=<str>        tool group to use during test\n"
	printf -- "\t\t       --sysinfo=<str,>          comma separated values of sysinfo to be collected\n"
	printf -- "\t\t                                      available: $(pbench-display-sysinfo-options)\n"
	printf -- "\t\t       --specjbb2005-dir=<str>   the location of the install directory for SPECjbb2005\n"
	printf -- "\t\t                                      (default is $specjbb2005_dir)\n"
}

# Process options and arguments
opts=$(getopt -q -o C:c:d:hj: --longoptions "config:,dir:,nr-jvms:,heap-size:,java-opts:,start-warehouses:,inc-warehouses:,stop-warehouses:,runtime:,tool-group:,sysinfo:,specjbb2005-dir:,help" -n "getopt.sh" -- "$@");
if [ $? -ne 0 ]; then
	printf -- "${script_name} $*\n"
	printf -- "\n"
	printf -- "\tunrecognized option specified\n\n"
	usage
	exit 1
fi
eval set -- "$opts";
while true; do
	case "$1" in
		-j|--java-opts)
		shift;
		if [ -n "$1" ]; then
			java_opts="$1"
			shift;
		fi
		;;
		-c|-C|--config)
		shift;
		if [ -n "$1" ]; then
			config="$1"
			shift;
		fi
		;;
		--nr-jvms)
		shift;
		if [ -n "$1" ]; then
			nr_jvms="$1"
			shift;
		fi
		;;
		--heap-size)
		shift;
		if [ -n "$1" ]; then
			heap_size="$1"
			shift;
		fi
		;;
		--start-warehouses)
		shift;
		if [ -n "$1" ]; then
			start_warehouses="$1"
			shift;
		fi
		;;
		--inc-warehouses)
		shift;
		if [ -n "$1" ]; then
			inc_warehouses="$1"
			shift;
		fi
		;;
		--stop-warehouses)
		shift;
		if [ -n "$1" ]; then
			stop_warehouses="$1"
			shift;
		fi
		;;
		--runtime)
		shift;
		if [ -n "$1" ]; then
			runtime="$1"
			shift;
		fi
		;;
		-d|--dir)
		shift;
		if [ -n "$1" ]; then
			dir="$1"
			shift;
		fi
		;;
		--tool-group)
		shift;
		if [ -n "$1" ]; then
			tool_group="$1"
			shift;
		fi
		;;
		--sysinfo)
		shift;
		if [ -n "$1" ]; then
			sysinfo="$1"
			shift;
		fi
		;;
		--specjbb2005-dir)
		shift;
		if [ -n "$1" ]; then
			specjbb2005_dir="$1"
			shift;
		fi
		;;
		-h|--help)
		usage
		exit 0
		;;
		--)
		shift;
		break;
		;;
		*)
		echo "what happened? [$1]"
		break;
		;;
	esac
done
verify_common_bench_script_options $tool_group $sysinfo

if [ ! -d "${specjbb2005_dir}" ]; then
	printf "\t${script_name}: you specified an invalid --specjbb2005-dir option (\"$specjbb2005_dir\")\n\n"
	usage
	exit 1
fi

if [ "${nr_jvms}" == "node" ]; then
	# Find out number of numa nodes
	nr_jvms=$(ls -1d /sys/devices/system/node/node* | wc -l)
fi
if [ "${nr_jvms}" -eq "${nr_jvms}" ] 2>/dev/null ; then
	# Verify the value is an int
	:
else
	printf -- "$*\n\n"
	printf "\t${benchmark}: you specified an invalid --nr-jvms option (\"${nr_jvms}\")\n\n"
	usage
	exit 1
fi
if [ "${stop_warehouses}" == "default" ]; then
	stop_warehouses=`echo "${nr_cpus} *2 /${nr_jvms}" | bc`
fi
if [ "${stop_warehouses}" -eq "${stop_warehouses}" ] 2>/dev/null ; then
	# Verify the value is an int
	:
else
	printf -- "$*\n\n"
	printf "\t${benchmark}: you specified an invalid --stop-warehouses option (\"${stop_warehouses}\")\n\n"
	usage
	exit 1
fi
if [ "${start_warehouses}" -eq "${start_warehouses}" ] 2>/dev/null ; then
	# Verify the value is an int
	:
else
	printf -- "$*\n\n"
	printf "\t${benchmark}: you specified an invalid --start-warehouses option (\"${start_warehouses}\")\n\n"
	usage
	exit 1
fi
if [ "${inc_warehouses}" -eq "${inc_warehouses}" ] 2>/dev/null ; then
	# Verify the value is an int
	:
else
	printf -- "$*\n\n"
	printf "\t${benchmark}: you specified an invalid --inc-warehouses option (\"${inc_warehouses}\")\n\n"
	usage
	exit 1
fi
if [ "${runtime}" -eq "${runtime}" ] 2>/dev/null ; then
	# Verify the value is an int
	:
else
	printf -- "$*\n\n"
	printf "\t${benchmark}: you specified an invalid --runtime option (\"${runtime}\")\n\n"
	usage
	exit 1
fi
if [ -d "${dir}" ]; then
	if [ -d "${dir}/results" ]; then
		printf -- "$*\n\n"
		printf "\t${benchmark}: the --dir option has an unexpected results directory (\"${dir}/results\")\n\n"
		usage
		exit 1
	fi
	benchmark_run_dir="${dir}"
elif [ -n "${dir}" ]; then
	printf -- "$*\n\n"
	printf "\t${benchmark}: you specified an invalid --dir option (\"${dir}\")\n\n"
	usage
	exit 1
fi

benchmark_fullname="${benchmark}_${config}_${date_suffix}"
export benchmark_run_dir="$pbench_run/${benchmark_fullname}"
export benchmark config

mkdir -p $benchmark_run_dir/.running
if [ $? -ne 0 ]; then
	printf "\t${benchmark}: unable to setup run directory (\"${benchmark_run_dir}\")\n\n"
	exit 1
fi

# now that the benchmark_run_dir directory exists, we can initialize the iterations file
benchmark_iterations="${benchmark_run_dir}/.iterations"
> ${benchmark_iterations}

pbench-register-tool-trigger --group=$tool_group --start-trigger="Timing Measurement began" --stop-trigger="Timing Measurement ended"

# Start the tool meisters on each registered local/remote host
pbench-tool-meister-start --sysinfo="${sysinfo}" "${tool_group}"
if [[ ${?} != 0 ]]; then
	error_log "[${script_name}]: failed to start the tool meisters."
	exit 1
fi

trap "interrupt" INT QUIT TERM

# edit the jbb properties file to match user's options
cp ${specjbb2005_dir}/SPECjbb.props ${benchmark_run_dir}/SPECjbb.props
if [ $? -ne 0 ]; then
	printf "\t${benchmark}: unable to create the SPECjbb.props file (\"${benchmark_run_dir}/SPECjbb.props\")\n\n"
	exit 1
fi
printf "input.jvm_instances=$nr_jvms\n" >> ${benchmark_run_dir}/SPECjbb.props
printf "input.measurement_seconds=$runtime\n" >> ${benchmark_run_dir}/SPECjbb.props
printf "input.starting_number_warehouses=$start_warehouses\n" >> ${benchmark_run_dir}/SPECjbb.props
printf "input.increment_number_warehouses=$inc_warehouses\n" >> ${benchmark_run_dir}/SPECjbb.props
printf "input.ending_number_warehouses=$stop_warehouses\n" >> ${benchmark_run_dir}/SPECjbb.props
cp ${specjbb2005_dir}/SPECjbb_config.props ${benchmark_run_dir}/
if [ $? -ne 0 ]; then
	printf "\t${benchmark}: unable to copy SPECjbb_config.props file from \"${specjbb2005_dir}\"\n\n"
	exit 1
fi

mkdir ${benchmark_run_dir}/results
if [ $? -ne 0 ]; then
	printf "\t${benchmark}: unable to create the results directory (\"${benchmark_run_dir}/results\")\n\n"
	exit 1
fi

ln -s ${specjbb2005_dir}/xml ${benchmark_run_dir}/
if [ $? -ne 0 ]; then
	printf "\t${benchmark}: unable to create the xml directory link (\"ln -s ${specjbb2005_dir}/xml ${benchmark_run_dir}/\")\n\n"
	exit 1
fi

# We execute from the benchmark run directory instead of the SPECjbb2005 directory.
pushd ${benchmark_run_dir} > /dev/null 2>&1

specjbb_cp="${specjbb2005_dir}/jbb.jar:${specjbb2005_dir}/check.jar"

# start the controller
java -cp ${specjbb_cp} -Xms32m -Xmx32m ${java_opts} spec.jbb.Controller -propfile ${benchmark_run_dir}/SPECjbb.props &
controller_pid=$!
if [ "x$controller_pid" = "x" ]; then
	printf "\t${benchmark}: unable to start the java controller\n\n"
	exit 1
fi

sleep 5

specjbb_opts="-cp ${specjbb_cp} -Xms${heap_size} -Xmx${heap_size} ${java_opts} spec.jbb.JBBmain -propfile ${benchmark_run_dir}/SPECjbb.props"
for i in `seq $nr_jvms -1 2`; do
	java ${specjbb_opts} -id $i > ./jvm-$i-worker.txt &
done
# The last JVM's output is used to trigger the tools starting at iteration 1.
java ${specjbb_opts} -id 1 | tee ./jvm-1-worker.txt | pbench-tool-trigger 1 "${benchmark_run_dir}" "${tool_group}"

debug_log "worker JVMs have finished"
debug_log "waiting for controller JVM to finish"
kill $controller_pid > /dev/null 2>&1
wait $controller_pid

if [ $nr_jvms -eq 1 ]; then
	specjbb_results="results/SPECjbbSingleJVM"
else
	specjbb_results="results/SPECjbbMultiJVM.001"
fi

# Function to add data to metadata.log
function record_iteration {
	local iteration=$1
	local iteration_name=$2
	local benchmark_bin=$3
	local mdlog="${benchmark_run_dir}/metadata.log"

	echo ${iteration_name} >> ${benchmark_iterations}
	echo ${iteration}      | pbench-add-metalog-option ${mdlog} iterations/${iteration_name} iteration_number
	echo ${iteration_name} | pbench-add-metalog-option ${mdlog} iterations/${iteration_name} iteration_name
	echo ${benchmark_bin}  | pbench-add-metalog-option ${mdlog} iterations/${iteration_name} user_script
}

# rename the iterations with warehouse count and then postprocess the tools
> ${benchmark_iterations}
iteration=1
sample="sample1"
for i in `seq $start_warehouses $inc_warehouses $stop_warehouses`; do
	if [ ! -d ${iteration}-default/${sample} ]; then
		error_log "missing iteration directory ${benchmark_run_dir}/${iteration}-default/${sample}"
		let iteration=$iteration+1
		continue
	fi
	# for each of the test iterations (each test with a specific number of warehouses), copy the raw result data for that iteration in the iteration directory
	grep "test${i}" ${specjbb_results}/SPECjbb.001.raw > ${iteration}-default/${sample}/SPECjbb.raw
	if [ $? -ne 0 ]; then
		error_log "missing test${i} raw results in ${specjbb_results}/SPECjbb.raw"
	fi
	pbench-send-tools --group=${tool_group} --dir=${benchmark_run_dir}/${iteration}-default/${sample}
	new_iteration_name="${iteration}-warehouses:${i}"
	mv ${iteration}-default ${new_iteration_name}
	if [ $? -ne 0 ]; then
		error_log "unable to move ${iteration}-default directory hierarchy to ${new_iteration_name}, skipping tools post-processing"
	else
		record_iteration ${iteration} ${new_iteration_name} ${benchmark}
		pbench-postprocess-tools --group=${tool_group} --dir=${benchmark_run_dir}/${new_iteration_name}/${sample}
		ln -s ${sample} ${benchmark_run_dir}/${new_iteration_name}/reference-result
	fi
	let iteration=$iteration+1
done

popd > /dev/null 2>&1

# Stop the tool meisters on each registered local/remote host
pbench-tool-meister-stop --sysinfo="${sysinfo}" "${tool_group}"
if [[ ${?} != 0 ]]; then
	error_log "[${script_name}]: failed to stop the tool meisters."
fi

rmdir $benchmark_run_dir/.running
