#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: t; sh-basic-offset: 8; sh-indentation: 8; tab-width: 8 -*-
#
# Pbench "linpack" driver script
#

script_name="$(basename ${0})"

linpack_input_file_name="linpack.input"
linpack_metadata_file_name="linpack.meta"
linpack_output_file_name="linpack.out"
linpack_command_file_name="linpack.cmd"
linpack_pid_file_name="linpack.pid"

binary=""

# Input file defaults, only lines 4-7 (headers defined below).
header=""
subheader=""

# Line 4 default
problem_sizes_def="20000"
problem_sizes=""
# Line 5 default
leading_dimensions_def="20016"
leading_dimensions=""
# Line 6 default
run_trials_def="1"
run_trials=""
# Line 7 default
alignment_values_def="4"
alignment_values=""

# Environment input defaults
use_omp_def="y"
use_omp=""
threads_arg=""
kmp_affinity_args_def="nowarnings,compact,1,0,granularity=fine"
kmp_affinity_args=""
numactl_args=""

# Output directory, defaults to the current working directory if not provided.
output_dir=""

# The linpack driver always performs pre-checks. When requested, the driver will
# only run the pre-check and exit.
pre_check_only=0

function usage() {
	cat <<-__EOF__
Usage: ${script_name} [--problem-sizes=#[[,#]...]] [--leading-dimenions=#[[,#]...]]
        [--run-trials=#[[,#]]...] [--alignment-values=#[[,#]...]] [--use-omp={y|n}]
        [--threads=#] [--kmp-affinity=<options>] [--numactl-args=<args>]
        [--output-dir=<directory>] <linpack version number installed>

Optional output control:

    --output-dir   Location to store the generated input file, metadata file,
                   and LINPACK generated output (defaults to current working
                   directory)

Optional command line parameters to specify values for the LINPACK input file:

    --header               Line 1 header to use (default provided)
    --subheader            Line 2 sub-header to use (default provided)
    --problem-sizes        Values for line 4 (defaults to ${problem_sizes_def})
    --leading-dimensions   Values for line 5 (defaults to ${leading_dimensions_def})
    --run-trials           Values for line 6 (defaults to ${run_trials_def})
    --alignment-values     Values for line 7 (defaults to ${alignment_values_def})

Optional parameters to direct the use of OpenMP:

    --use-omp        Use OpenMP; either 'y' or 'n' (default is '${use_omp_def}')
    --threads        Number of threads to use under OpenMP (defaults to host CPU count)
    --kmp-affinity   KMP_AFFINITY setting for OpenMP (defaults to ${kmp_affinity_args_def})

Optional NUMA control parameter:

    --numactl-args   \`numactl\` command parameters (no default provided)
	__EOF__
}

function help() {
	cat <<-__EOF__
Pbench ${script_name} driver

${script_name} is responsible for transforming command line inputs into the
LINPACK invocation requirements.

The linpack binary expects an input file with 7 lines in it:

    Line 1: Header (discarded by LINPACK; max 254 bytes + new line)
    Line 2: Sub-Header (emitted as-is to stdout; max 254 bytes + new line)
    Line 3: Positive integer number of tests to run
    line 4: Number of equations to solve (problem size)
    line 5: Leading dimension of array
    line 6: Number of trials to run
    line 7: Data alignment value (in Kbytes)

For lines 4-7, if there are N tests to run, each line should have N values
describing the parameters for those tests. For example, runs three tests using
problem sizes of 10,000, 20,000, and 30,000:

    Line 1: Header
    Line 2: Sub-Header
    Line 3: 3
    line 4: 10000 20000 30000
    line 5: 10016 20016 30016
    line 6:     2     2     2
    line 7:     4     4     4

This script provides optional command line switches to override the default
values for lines 1, 2, 4-7; the script calculates the number of tests to be
run from those inputs to fill in line 3 of the LINPACK input file.

Additionally, it offers input parameters to control the use of OpenMP during
the LINPACK run, along with values for the OMP_NUM_THREADS and KMP_AFFINITY
environment variables used by OpenMP, and if any NUMA controls should be applied
(via \`numactl\`).  These parameters are displayed in the default "Sub-Header".

The script generates four files:

   {output-dir}/linpack.cmd    -  Command that was actually executed
   {output-dir}/linpack.input  -  Constructed seven line LINPACK input file
   {output-dir}/linpack.meta   -  All input parameters provided as key=val
   {output-dir}/linpack.out    -  The output generated by the LINPACK binary

The script exits with 0 on success, 1 for any operational failures, and 2 for
an invocation errors.

	__EOF__
	usage
}

# Process options and arguments
opts=$(getopt -q -o h --longoptions "header:,subheader:,problem-sizes:,leading-dimensions:,alignment-values:,run-trials:,threads:,use-omp:,kmp-affinity:,numactl-args:,output-dir:,pre-check-only,help" -n "${script_name}" -- "${@}")
if [[ ${?} -ne 0 ]]; then
	printf -- "%s %s\n\n\tunrecognized option specified\n\n" "${script_name}" "${*}" >&2
	usage >&2
	exit 2
fi
eval set -- "${opts}"
while true; do
	arg="${1}"
	shift
	case "${arg}" in
	--header)
		# Values for line 1 of the LINPACK input file
		if [[ -n "${1}" ]]; then
			header="${1}"
			shift
		fi
		;;
	--subheader)
		# Values for line 2 of the LINPACK input file
		if [[ -n "${1}" ]]; then
			subheader="${1}"
			shift
		fi
		;;
	--problem-sizes)
		# Values for line 4 of the LINPACK input file
		if [[ -n "${1}" ]]; then
			problem_sizes=${1//,/ }
			shift
		fi
		;;
	--leading-dimensions)
		# Values for line 5 of the LINPACK input file
		if [[ -n "${1}" ]]; then
			leading_dimensions=${1//,/ }
			shift
		fi
		;;
	--run-trials)
		# Values for line 6 of the LINPACK input file
		if [[ -n "${1}" ]]; then
			run_trials=${1//,/ }
			shift
		fi
		;;
	--alignment-values)
		# Values for line 7 of the LINPACK input file
		if [[ -n "${1}" ]]; then
			alignment_values=${1//,/ }
			shift
		fi
		;;
	--threads)
		if [[ -n "${1}" ]]; then
			threads_arg="${1}"
			shift
		fi
		;;
	--use-omp)
		if [[ -n "${1}" ]]; then
			use_omp="${1}"
			shift
		fi
		;;
	--kmp-affinity)
		if [[ -n "${1}" ]]; then
			kmp_affinity_args="${1}"
			shift
		fi
		;;
	--numactl-args)
		if [[ -n "${1}" ]]; then
			numactl_args="${1}"
			shift
		fi
		;;
	--output-dir)
		if [[ -n "${1}" ]]; then
			output_dir="${1}"
			shift
		fi
		;;
	--pre-check-only)
		pre_check_only=1
		;;
	-h|--help)
		help
		exit 0
		;;
	--)
		break
		;;
	*)
		printf -- "[%s] WARNING: unrecognized parameter encountered, '%s'\n" "${script_name}" "${arg}" >&2
		break
		;;
	esac
done

ver=${1}
if [[ -z "${ver}" ]]; then
	printf -- "[%s] ERROR: You must specify the version of the LINPACK binary\n\n" "${script_name}" >&2
	usage >&2
	exit 2
fi

# Installation directory, optional 2nd argument, defaults to /usr/local.
install_prefix_dir=${2:-"/usr/local"}

binary="${install_prefix_dir}/pbench-linpack-${ver}/benchmarks/linpack/xlinpack_xeon64"
if [[ ! -x "${binary}" ]]; then
	printf -- "[%s] ERROR: The --binary, '${binary}', must exist and be executable\n" "${script_name}" >&2
	exit 2
fi

if [[ ${pre_check_only} -ne 0 ]]; then
	exit 0
fi

output_dir=${output_dir:-$(pwd)}
if [[ ! -d "${output_dir}" ]]; then
	printf -- "[%s] ERROR: Specified --output-dir '${output_dir}' is not a directory\n" "${script_name}" >&2
	exit 2
fi
use_omp=${use_omp:-${use_omp_def}}
if [[ "${use_omp}" != "y" && "${use_omp}" != "n" ]]; then
	printf -- "[%s] ERROR: --use-omp must either be 'y' or 'n'\n" "${script_name}" >&2
	exit 2
fi
if [[ "${use_omp}" == "n" && (-n "${threads_arg}" || -n "${kmp_affinity_args}") ]]; then
	printf -- "[%s] ERROR: --threads and --kmp-affinity not allowed with --use-omp=n\n" "${script_name}" >&2
	exit 2
fi

problem_sizes=${problem_sizes:-${problem_sizes_def}}
ps_list=(${problem_sizes})
leading_dimensions=${leading_dimensions:-${leading_dimensions_def}}
ls_list=(${leading_dimensions})
run_trials=${run_trials:-${run_trials_def}}
rt_list=(${run_trials})
alignment_values=${alignment_values:-${alignment_values_def}}
av_list=(${alignment_values})
number_of_tests=${#ps_list[*]}
if [[ ${#ls_list[*]} -ne ${number_of_tests} || \
      ${#rt_list[*]} -ne ${number_of_tests} || \
      ${#av_list[*]} -ne ${number_of_tests} ]]; then
	printf -- "[%s] ERROR: You must have the same number of elements for --problem-sizes, --leading-dimensions, --run-trials, and --alignment-values\n" >&2
	exit 2
fi

linpack_input_file="${output_dir}/${linpack_input_file_name}"
linpack_metadata_file="${output_dir}/${linpack_metadata_file_name}"
linpack_output_file="${output_dir}/${linpack_output_file_name}"
linpack_command_file="${output_dir}/${linpack_command_file_name}"
linpack_pid_file="${output_dir}/${linpack_pid_file_name}"

# N.B. - Trailing whitespace is required.
numactl_cmd=${numactl_args:+"numactl ${numactl_args} "}

if [[ "${use_omp}" == "y" ]]; then
	threads_arg=${threads_arg:-$(grep -c processor /proc/cpuinfo)}
	kmp_affinity_args=${kmp_affinity_args:-${kmp_affinity_args_def}}

	# N.B. - Trailing whitespace is required.
	omp_num_threads="OMP_NUM_THREADS=${threads_arg} "
	# N.B. - Trailing whitespace is required.
	kmp_affinity="KMP_AFFINITY=${kmp_affinity_args} "
else
	# No environment variables to add to the command line.
	omp_num_threads=""
	kmp_affinity=""
fi
final_cmd="${omp_num_threads}${kmp_affinity}${numactl_cmd}${binary}"
display_cmd="${omp_num_threads}${kmp_affinity}${numactl_cmd}${binary}"
final_cmd+=" ${linpack_input_file} |"
final_cmd+=" stdbuf --input=0 tee -i ${linpack_output_file}"
# Save the actual command we'll use below
printf -- "${final_cmd}\n" > ${linpack_command_file}

# Line 1, providing a default header (truncated to 254 characters below)
header=${header:-"Pbench - LINPACK Benchmark data file (${linpack_input_file})"}
# Line 2, providing a default sub-header (truncated to 254 characters below)
subheader=${subheader:-"Pbench - LINPACK Command: '${display_cmd}'"}

# Create the LINPACK input file.  Note the comment lines are redundant but
# provided in a spirit of consistency with the example input files provided by
# linpack RPM.
cat > ${linpack_input_file} <<-__EOF__
	${header::254}
	${subheader::254}
	${number_of_tests} # number of tests
	${problem_sizes} # problem sizes
	${leading_dimensions} # leading dimensions
	${run_trials} # times to run a test
	${alignment_values} # alingment values (in KBytes)
	__EOF__

# Declare ourselves to anybody waiting.
echo "$$" > ${linpack_pid_file}

# Now we can execute the final LINPACK command.
source ${linpack_command_file}
exit_code=${?}

# Create the linpack.meta file containing all the metadata used for invoking
# LINPACK (does not include "header" or "subheader").  This serves as the marker
# file for its completion.
cat > ${linpack_metadata_file} <<-__EOF__
	binary=${binary}
	use_omp=${use_omp}
	threads=${threads_arg}
	kmp_affinity=${kmp_affinity_args}
	numactl_cmd=${numactl_cmd}
	problem_sizes=${problem_sizes}
	leading_dimensions=${leading_dimensions}
	alignment_values=${alignment_values}
	__EOF__

exit ${exit_code}
