#!/usr/bin/perl
#
# Author: Andrew Theurer
#
# The purpose of this script is to convert mpstat-stdout.txt
# into either (depending on how this script is invoked):
#
#   CSV files and statically generated graphs
#   with embedded javascript, using d3.js libraries
#
# Usage:
#
#   mpstat-postprocess <dir>
#
#   dir = directory where mpstat-stdout.txt can be found
#
# Note:
#
#   mpstat-stdout.txt must be generated from "mpstat -P ALL <interval>"

use strict;
use warnings;

use lib $ENV{'pbench_lib_dir'};
no lib ".";
use GenData qw(gen_data);
use SysStat qw(get_mpstat_cpumode_attributes build_cpu_topology get_cpu_socket_core);

my $dir = shift;

die "Non-default options, skipping post-processing for mpstat" if (-f $dir . "/mpstat.options");

my $date_ms;
my $prev_time = "";
my %online_cpus;

# We'll force mpstat into a LANG=C environment: two-digit year on the first line,
# 24-hour time with *no* AM/PM designation in the timestamp.
# Usage example: $timestamp_ms = calc_timestamp($time);
sub calc_timestamp {
	# Time format: HH:MM:SS
	# 24-hour format,*no* AM/PM designation
	my $hourminsec = $_[0];
	$hourminsec =~ /(\d+):(\d+):(\d+)/;
	my $hour = $1;
	my $minute = $2;
	my $second = $3;
	# Keep curr/prev time in milliseconds
	my $this_time = (($hour * 60 * 60) + ($minute * 60) + $second) * 1000;

	if ( $prev_time eq "" ) {
		# This is the first time stamp we have seen, so remember
		# it as the previous one to start.
		$prev_time = $this_time;
	} else {
		if ( $prev_time > $this_time ) {
			# We have detected a date roll-over, so add 24 hours to
			# the current date.
			$date_ms += (24 * 60 * 60 * 1000);
		}
		$prev_time = $this_time;
	}
	return $date_ms + $this_time;
}

sub cpu_is_online {
	my $cpu = shift;
	if (exists $online_cpus{$cpu}) {
		return $online_cpus{$cpu};
	} elsif (-e $dir . "/online-cpus.txt" and open(FH, $dir . "/online-cpus.txt")) {
		# Try a saved file of online status for this mpstat (this
		# relies on this file being generated at some point before
		# postprocessing, likely at stop-tools phase). This may not
		# exist depending on what version of pbench was used to get
		# the data.
		while (my $line = <FH>) {
			# Take the opportunity to read all the CPUs and populate
			# our hash
			if ($line =~ /^(\d+):\s*([0-1])$/) {
				$online_cpus{$1} = $2;
			}
		}
		close(FH);
		return $online_cpus{$cpu};
	} else {
		# If nothing can be found, just assume the cpu is online
		$online_cpus{$cpu} = 1;
		return $online_cpus{$cpu};
	}
}

my %mpstat; # For gen_data()
my $timestamp_ms;
my @cpumode_attributes = get_mpstat_cpumode_attributes;
my @nonbusy = (qw(idle iowait steal));
my @busy = grep(!/^idle?|^iowait?|^steal?/, @cpumode_attributes);
my %cpu_modes = ( 'idle' => \@nonbusy, 'busy' => \@busy );

open(MPSTAT_TXT, "$dir/mpstat-stdout.txt") || die "could not find $dir/mpstat-stdout.txt\n";
# All mpstat output processing assumes LANG=C environment when mpstat was run.
#
# The first line contains the date:
#    Linux 3.16.6-200.fc20.x86_64 (a.foo.example.com) 02/25/15 _x86_64_ (8 CPU)
# Subsequent lines are of the form:
#    23:59:56     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
#    00:00:06     all    0.01    0.00    0.03    0.00    0.00    0.00    0.00    0.00    0.00   99.96
#    00:00:06       0    0.10    0.00    0.10    0.00    0.00    0.00    0.00    0.00    0.00   99.80
#    00:00:06       1    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00
# containing only the time, so we need to combine the two to get a meaningful timestamp.
chomp(my $line = <MPSTAT_TXT>);
if ( $line =~ /\S+\s\S+\s\S+\s+(\d\d\/\d\d\/\d\d).*/ ) {
	# Linux 3.16.6-200.fc20.x86_64 (a.foo.example.com) 02/25/15 _x86_64_ (8 CPU)
	# Convert to milliseconds since the epoch
	$date_ms = 1000 * `date --date="$1" +%s.%N`;
} else {
	die("Bad mpstat-stdout.txt file format: first line should have the form: Linux 3.16.6-200.fc20.x86_64 (a.foo.example.com) 02/25/2015 _x86_64_ (8 CPU)");
}

while (my $line = <MPSTAT_TXT>) {
	chomp $line;
	if ( $line =~ /(\d\d:\d\d:\d\d)\s+(\S+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+)\s+(\d+\.\d+).*/ ) {
		(my $time, my $cpuid, my @values) = split(/\s+/, $line);
		next if ($cpuid ne "all" and not cpu_is_online($cpuid));
		$timestamp_ms = calc_timestamp($time);
		my %cpumode_util;
		@cpumode_util{@cpumode_attributes} = @values;
		$timestamp_ms = calc_timestamp($time);
		for my $mode (@cpumode_attributes) {
			next if ($mode eq "gnice");
			my $mode_label;
			if ($mode eq 'soft') { # Stay compatible with legacy label
				$mode_label = 'softirq';
			} else {
				$mode_label = $mode;
			}
			my $cpu_label = "cpu" . $cpuid;
			$mpstat{$cpu_label}{$cpu_label}{$mode_label}{$timestamp_ms} = $cpumode_util{$mode};
		}
	} elsif ( $line =~ /(\d\d:\d\d:\d\d)\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+\S+.*/ ) {
		# Run the timestamp calculation since the first head might
		# contain the last timestamp of the day.
		$timestamp_ms = calc_timestamp($1);
	}
}
close(MPSTAT_TXT);
my $file;
my $graph;
my %graph_threshold;
my %graph_type;
foreach $file (keys %mpstat) {
	foreach $graph (keys %{ $mpstat{$file} }) {
		$graph_type{$file}{$graph} = "stackedAreaChart";
	}
}
gen_data(\%mpstat, \%graph_type, \%graph_threshold, $dir);
