#!/usr/bin/perl

# Author: Andrew Theurer
#
# The purpose of this script is to convert pidstat data
# generated by pidstat-convert and located in ./pids
# into either (depending on how this script is invoked):
#
#   CSV files and statically generated graphs
#   with embedded javascript, using d3.js libraries
#
# Usage:
#
# pidstat-postprocess <dir>
#
#   dir = directory where pidstat-stdout.txt can be found
#
# Note:
#
#   pidstat output must be generated with "pidstat -l -H -w -u -h -d -r [-t] -p ALL"
#   and piped to pidstat-convert

use strict;
use warnings;

use lib $ENV{'pbench_lib_dir'};
no lib ".";
use GenData qw(gen_data);
use SysStat qw(get_pidstat_attributes);

my $dir = shift;

die "Non-default options, skipping post-processing for pidstat" if (-f $dir . "/pidstat.options");

my %pidstat;
my %graph_type;
my %graph_threshold;
my $vm_name;
my $line;
my $line2;
my $tid;
my @prev_stats;
my @curr_stats;
my $threads;

my %timestamps;
my @attributes = get_pidstat_attributes();
my $pids_dir = $dir . "/pids";
if (opendir(my $pids_dh, $pids_dir) || die "Could not open $pids_dir") {
        for my $pid (grep(/\d+/, readdir($pids_dh))) {
		my $pid_dir = $pids_dir . "/" . $pid;
		opendir(my $pid_dh, $pid_dir) || die "Could not open $pid_dir";
		# A PID may be recycled over time as 1 process exits and another later is created,
		# and therefore more than 1 process could have the same PID
		my $timestamp_ms;
        	for my $filename (grep(/[^.]/, readdir($pid_dh))) {
			my $cmd;
			my $pid_cmd;
			open(my $fh, $pid_dir . "/" . $filename) || die "Could not open $filename";
			my %old_stats;
			my $line_num = 0;
			while ($line = <$fh>) {
				$line_num++;
				if ($line_num == 1) {
					# get the real, unmodified cmdline first
					$cmd = $line;
					chomp $cmd;
					# convert qemu to a more readable label with VM name
					if ( $cmd =~ /qemu/ and $cmd =~ /name\s(\S+).*/ ) {
						$vm_name = $1;
						$cmd = "KVMguest--" . $vm_name;
					}
					# modify to be appropriate as a label for a series in a graph
					$cmd =~ s/%/%%/g; # avoid 'Invalid conversion in printf: "%=" ' errors if cmd contains a %.;
					$cmd =~ s/[\'|\"]//g; # quotes and vertical bars cause problems everywhere
					$cmd =~ s/\s/_/g; # convert spaces to _
					$cmd =~ s/,//g; # commas cause problems in the CSV files
					$cmd = substr $cmd, 0, 40; # limit the length
					$pid_cmd = $pid . "-" . $cmd;
					next;
				}
				my %stats;
				chomp $line;
				if ( $line =~ /^(\d+)(.*)/ ) {
					$timestamp_ms = $1 * 1000;
					my $stats_str = $2;
					if ($stats_str eq '') {
						# every single stat is a repeat from the previous sample
						%stats = %old_stats;
					} else {
						# some or all new stats are different from previous sample
						$stats_str =~ s/^,//;
						@stats{@attributes} = split(/,/,$stats_str);
						# copy any old stats to missing new stats
						for my $attribute (@attributes) {
							if ((not defined $stats{$attribute}) or ($stats{$attribute} eq '')) {
								if (defined $old_stats{$attribute}) {
									$stats{$attribute} = $old_stats{$attribute};
								} else {
									print "ERROR: old_stat for $attribute not found in file $filename line number $line_num\n";
									exit 1;
								}
							}
						}
					}
					%old_stats = %stats;
					my %file_io_kb_sec = ( 'Read' => $stats{'kB_rd_s'}, 'Write' => $stats{'kB_wr_s'} );
					my %memory_faults_sec = ( 'Minor' => $stats{'minflt_s'}, 'Major' => $stats{'majflt_s'} );
					my %ctxsw_sec = ( 'Voluntary' => $stats{'cswch_s'}, 'NonVoluntary' => $stats{'nvcswch_s'} );
					my %noncpu_tput = ( 'FileIoKbSec' => \%file_io_kb_sec, 'MemoryFaultsSec' => \%memory_faults_sec,
								'ContextSwitchesSec' => \%ctxsw_sec );
					my %memory_usage_kb = ( 'Resident' => $stats{'RSS'}, 'NonResident' => $stats{'VSZ'} - $stats{'RSS'} );
					$pidstat{'cpu_usage'}{'percent_cpu'}{$pid_cmd}{$timestamp_ms} = $stats{'CPU_PCT'};
					$pidstat{'file_io'}{'io_reads_KB_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'kB_rd_s'};
					$pidstat{'file_io'}{'io_writes_KB_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'kB_wr_s'};
					$pidstat{'memory_faults'}{'minor_faults_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'minflt_s'};
					$pidstat{'memory_faults'}{'major_faults_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'majflt_s'};
					$pidstat{'context_switches'}{'voluntary_switches_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'cswch_s'};
					$pidstat{'context_switches'}{'nonvoluntary_switches_sec'}{$pid_cmd}{$timestamp_ms} = $stats{'nvcswch_s'};
					$pidstat{'memory_usage'}{'virtual_size'}{$pid_cmd}{$timestamp_ms} = $stats{'VSZ'};
					$pidstat{'memory_usage'}{'resident_set_size'}{$pid_cmd}{$timestamp_ms} = $stats{'RSS'};
					$timestamps{$timestamp_ms}++;
				}
                	}
        	}
	}
}

# fill in any missing data with zeros
my $htmlpage;
my $graph;
my $pid;
my $timestamp_ms;
foreach  $htmlpage ( keys %pidstat ) {
	foreach  $graph ( keys %{ $pidstat{$htmlpage} } ) {
		foreach $pid ( keys %{ $pidstat{$htmlpage}{$graph} } ) {
			foreach $timestamp_ms ( sort {$a <=> $b} (keys %timestamps ) ) {
				if (! defined($pidstat{$htmlpage}{$graph}{$pid}{$timestamp_ms})) {
					$pidstat{$htmlpage}{$graph}{$pid}{$timestamp_ms} = 0;
				}
			}
		}
	}
}
$graph_type{cpu_usage}{percent_cpu} = "stackedAreaChart";
$graph_threshold{cpu_usage}{percent_cpu} = 1;
$graph_threshold{file_io}{io_reads_KB_sec} = 50;
$graph_threshold{file_io}{io_writes_KB_sec} = 50;
$graph_threshold{memory_faults}{minor_faults_sec} = 150;
$graph_threshold{memory_faults}{major_faults_sec} = 150;
$graph_threshold{context_switches}{voluntary_switches_sec} = 100;
$graph_threshold{context_switches}{nonvoluntary_switches_sec} = 100;
$graph_threshold{memory_usage}{virtual_size} = 100;
$graph_threshold{memory_usage}{resident_set_size} = 100;
gen_data(\%pidstat, \%graph_type, \%graph_threshold, $dir);
