#!/usr/bin/perl
#
# Author: Andrew Theurer
#
# The purpose of this script is to convert mpstat-stdout.txt
# into either (depending on how this script is invoked):
#
#   CSV files and statically generated graphs
#   with embedded javascript, using d3.js libraries
#
# or
#
#   CommonDataModel documents
#
# Usage:
#
#   mpstat-postprocess <dir> [<period-doc> <es-dir> <hostname>]
#
#   dir = directory where mpstat-stdout.txt can be found
#   period-doc = path to CDM period document for this iteration-sample
#   es-dir = path to base directory where CDM documents should be written
#   hostname = the host where mpstat was run
#
# Note:
#
#   arguments 2 - 4 are only used when generating CommonDataModel
#
#   mpstat-stdout.txt must be generated from "mpstat -P ALL <interval>"

use strict;
use warnings;

use lib $ENV{'pbench_lib_dir'};
no lib ".";
BEGIN {
    if (scalar @ARGV > 1) {
        require PbenchCDM;
        PbenchCDM->import( qw(log_cdm_metric_sample gen_cdm_metric_data) );
    }
}
use GenData qw(gen_data);
use SysStat qw(get_mpstat_cpumode_attributes build_cpu_topology get_cpu_socket_core);

my $dir = shift;

die "Non-default options, skipping post-processing for mpstat" if (-f $dir . "/mpstat.options");

my $date_ms;
my $prev_time = "";
my %online_cpus;
my $hostname; # for CDM

# We'll force mpstat into a LANG=C environment: two-digit year on the first line,
# 24-hour time with *no* AM/PM designation in the timestamp.
# Usage example: $timestamp_ms = calc_timestamp($time);
sub calc_timestamp {
	# Time format: HH:MM:SS
	# 24-hour format,*no* AM/PM designation
	my $hourminsec = $_[0];
	$hourminsec =~ /(\d+):(\d+):(\d+)/;
	my $hour = $1;
	my $minute = $2;
	my $second = $3;
	# Keep curr/prev time in milliseconds
	my $this_time = (($hour * 60 * 60) + ($minute * 60) + $second) * 1000;

	if ( $prev_time eq "" ) {
		# This is the first time stamp we have seen, so remember
		# it as the previous one to start.
		$prev_time = $this_time;
	} else {
		if ( $prev_time > $this_time ) {
			# We have detected a date roll-over, so add 24 hours to
			# the current date.
			$date_ms += (24 * 60 * 60 * 1000);
		}
		$prev_time = $this_time;
	}
	return $date_ms + $this_time;
}

sub cpu_is_online {
	my $cpu = shift;
	if (exists $online_cpus{$cpu}) {
		return $online_cpus{$cpu};
	} elsif (-e $dir . "/online-cpus.txt" and open(FH, $dir . "/online-cpus.txt")) {
		# Try a saved file of online status for this mpstat (this
		# relies on this file being generated at some point before
		# postprocessing, likely at stop-tools phase). This may not
		# exist depending on what version of pbench was used to get
		# the data.
		while (my $line = <FH>) {
			# Take the opportunity to read all the CPUs and populate
			# our hash
			if ($line =~ /^(\d+):\s*([0-1])$/) {
				$online_cpus{$1} = $2;
			}
		}
		close(FH);
		return $online_cpus{$cpu};
	} else {
		# If nothing can be found, just assume the cpu is online
		$online_cpus{$cpu} = 1;
		return $online_cpus{$cpu};
	}
}

my %mpstat; # For gen_data()
my %mpstat_cdm; # For log_cdm_metric_sample()
my $timestamp_ms;
my $es_dir; # for CDM
my $period_doc_path; # for CDM
my @cpu_topology; # for CDM
my $cdm = 0;
my @cpumode_attributes = get_mpstat_cpumode_attributes;
my @nonbusy = (qw(idle iowait steal));
my @busy = grep(!/^idle?|^iowait?|^steal?/, @cpumode_attributes);
my %cpu_modes = ( 'idle' => \@nonbusy, 'busy' => \@busy );

# When producing CDM docs, this is typically called from the pbench
# controller only, and the following arguments must be provided:
if (scalar @ARGV > 1) {
	die "Must provide <path-to-period.doc> <es-dir> <tool-hostname>" if (scalar @ARGV != 3);
	$cdm = 1;
	$period_doc_path = shift;
	$es_dir = shift;
	$hostname = shift;
	build_cpu_topology(\@cpu_topology, $dir);
}
open(MPSTAT_TXT, "$dir/mpstat-stdout.txt") || die "could not find $dir/mpstat-stdout.txt\n";
# All mpstat output processing assumes LANG=C environment when mpstat was run.
#
# The first line contains the date:
#    Linux 3.16.6-200.fc20.x86_64 (a.foo.example.com) 02/25/15 _x86_64_ (8 CPU)
# Subsequent lines are of the form:
#    23:59:56     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
#    00:00:06     all    0.01    0.00    0.03    0.00    0.00    0.00    0.00    0.00    0.00   99.96
#    00:00:06       0    0.10    0.00    0.10    0.00    0.00    0.00    0.00    0.00    0.00   99.80
#    00:00:06       1    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00
# containing only the time, so we need to combine the two to get a meaningful timestamp.
chomp(my $line = <MPSTAT_TXT>);
if ( $line =~ /\S+\s\S+\s\S+\s+(\d\d\/\d\d\/\d\d).*/ ) {
	# Linux 3.16.6-200.fc20.x86_64 (a.foo.example.com) 02/25/15 _x86_64_ (8 CPU)
	# Convert to milliseconds since the epoch
	$date_ms = 1000 * `date --date="$1" +%s.%N`;
} else {
	die("Bad mpstat-stdout.txt file format: first line should have the form: Linux 3.16.6-200.fc20.x86_64 (a.foo.example.com) 02/25/2015 _x86_64_ (8 CPU)");
}

while (my $line = <MPSTAT_TXT>) {
	chomp $line;
	if ( $line =~ /(\d\d:\d\d:\d\d)\s+(\S+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+).*/ ) {
		(my $time, my $cpuid, my @values) = split(/\s+/, $line);
		next if ($cpuid ne "all" and not cpu_is_online($cpuid));
		$timestamp_ms = calc_timestamp($time);
		my %cpumode_util;
		@cpumode_util{@cpumode_attributes} = @values;
		$timestamp_ms = calc_timestamp($time);
		if ($cdm) {
			if ($cpuid ne "all") {
				# Note that an "all" cpu metric is not logged
				# with CDM like with traditional pbench.  The
				# "all" metric represents all CPUs in a host
				# as a range of 0-100%.  We do not represent
				# CPU utilization this way in CDM.  Instead,
				# we calculate system CPU by aggregating per-
				# CPU stats into a single stat, dynamically,
				# via the Web UI.  A per-system CPU utiliza-
				# tion will be from 0 to (100 * nr_cpus)
				# instead of 0-100%.
				#
				# Addtionally, cpu busy and idle stats have
				# different name_format, so that only busy or
				# only idle stats are aggregated.  Allowing
				# aggregation of all the modes (idle and busy)
				# together makes little sense, since it will
				# just add up to 100.
				(my $socket, my $core) = get_cpu_socket_core($cpuid, \@cpu_topology);
				for my $util_type (keys %cpu_modes ) {
				my $type = "Processor" . ucfirst $util_type . "Util";
					for my $mode (@{ $cpu_modes{$util_type} }) {
						my %md = ( 'host' => $hostname, 'socket' => $socket, 'core' => $core, 'id' => $cpuid, 'mode' => $mode );
						log_cdm_metric_sample('mpstat', 'throughput', $type, '%host%-%socket%:%core%:%id%-%mode%',
									\%md, \%mpstat_cdm, $timestamp_ms, $cpumode_util{$mode});
					}
				}
			}
		} else {
			for my $mode (@cpumode_attributes) {
				next if ($mode eq "gnice");
				my $mode_label;
				if ($mode eq 'soft') { # Stay compatible with legacy label
					$mode_label = 'softirq';
				} else {
					$mode_label = $mode;
				}
				my $cpu_label = "cpu" . $cpuid;
				$mpstat{$cpu_label}{$cpu_label}{$mode_label}{$timestamp_ms} = $cpumode_util{$mode};
			}
		}
	} elsif ( $line =~ /(\d\d:\d\d:\d\d)\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+.*/ ) {
		# Run the timestamp calculation since the first head might
		# contain the last timestamp of the day.
		$timestamp_ms = calc_timestamp($1);
	}
}
close(MPSTAT_TXT);
if ($cdm) {
	print "cdm\n";
	gen_cdm_metric_data(\%mpstat_cdm, $period_doc_path, $es_dir, $hostname, "mpstat");
} else {
	my $file;
	my $graph;
	my %graph_threshold;
	my %graph_type;
	foreach $file (keys %mpstat) {
		foreach $graph (keys %{ $mpstat{$file} }) {
			$graph_type{$file}{$graph} = "stackedAreaChart";
		}
	}
	gen_data(\%mpstat, \%graph_type, \%graph_threshold, $dir);
}
