#!/usr/bin/perl

# Author: Andrew Theurer
#
# This script is used to compare one or more benchmark results to each other.
# Usage: compare-bench-results <output-file-name> <dir1> <dir2> [<dirN>]
#
# The script will generate a txt and html file, comparing each matching
# iteration from all results directories to each other.  The html output
# uses nvd3 to graph clustered bars for each iteration.  Currently the
# html uses a javascript file for graph data, but we will move to
# csv data soon.
#
# One goal is for this script to compare results from any of the
# pbench benchmark scripts (comparing results of same benchmark).
# However, currently only pbench-fio and pbench-uperf work.  Other
# benchmark scripts will be updated to support the standard result
# format that this script understands.


use strict;
use warnings;
use File::Basename;
my $script = basename($0);
my $comparison_name = shift(@ARGV);
my @dirs = @ARGV;
my %results;
my @labels;
my @values;
my $iteration;
my @iterations;
my $label;
my $line;
my $dir;
my %results_by_same_header;
my $primary_metric;
my $dir_num = 0;
my %result_data_json; # used to generate json file for multiBarHorizontal graph


sub mysortlabels {
	my @la = split('-', $a);
	my @lb = split('-', $b);
	my $maxcnt = $#la >= $#lb ? $#lb : $#la;
	my $i = 0;
	for (my $i = 0; $i < $maxcnt; $i++) {
		my $va;
		my $vb;
		if ($la[$i] =~ /(^\d+)/) {
			$va = int($1);
			if ($lb[$i] =~ /(^\d+)/) {
				$vb = int($1);
				#printf "%s, %s, %d, %d\n", $a, $b, $va, $vb;
				if ($va < $vb) {
					return -1;
				} else {
					if ($va > $vb) {
						return 1;
					}
				}
			}
		}
		if ($la[$i] lt $lb[$i]) {
			return -1;
		} else {
			if ($la[$i] gt $lb[$i]) {
				return 1;
			}
		}
	}
	return 0;
}

print "\n";
foreach $dir (@dirs) {
	open(CSV_FILE, "$dir/summary-result.csv") || die "$script: could not open file $dir/summary-result.csv: $!\n";
	while (<CSV_FILE>) {
		my $line = $_;
		chomp $line;

		## A csv file typically looks like this:
		#
		#            iteration,             Gbps,stddev%,   Gbps/serverCPU,stddev%,   Gbps/clientCPU,stddev%,        serverCPU,stddev%,        clientCPU,stddev%
		#   1-tcp_stream-1B-1i,           0.0175, 1.1228,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000
		#   2-tcp_stream-1B-8i,           0.1216, 0.2694,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000
		#  3-tcp_stream-1B-64i,           0.1216, 0.3697,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000
		#  4-tcp_stream-64B-1i,           1.0400, 5.0127,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000
		#  5-tcp_stream-64B-8i,           5.1456, 0.0943,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000
		#            iteration,        trans/sec,stddev%,     latency_usec,stddev%,trans/sec/serverCPU,stddev%,trans/sec/clientCPU,stddev%,        serverCPU,stddev%,        clientCPU,stddev%
		#      37-tcp_rr-1B-1i,       36661.8927, 0.3049,          27.2930, 0.3784,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000
		#      38-tcp_rr-1B-8i,      267535.0295, 0.3590,          29.9142, 0.3607,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000
		#     39-tcp_rr-1B-64i,      701442.4952, 0.5962,          91.3187, 0.6645,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000
		#     40-tcp_rr-64B-1i,       36158.1772, 0.8615,          27.6609, 0.7810,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000
		#     41-tcp_rr-64B-8i,      261930.0239, 0.0966,          30.5464, 0.0663,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000,           0.0000, 0.0000
		#
		## Note that the header may be periodically re-written as above.  If this happens, you must ensure the following lines use the new header column labels when importing the data
		## It is possible that one benchmark has iterations with different result values than another iteration.  For example, uperf can have Gbps or trans/sec.  When comparing 1 or
		## more iterations from different benchmark runs, the labels of the results (the column header label) must match across all of the iteration result data that is being compared.
		##
		## The first column is always the iteration, followed by 1 or more columns, which are benchmark metrics, with the first one being the most important

		if ($line =~ /^\s*iteration,.*/ ) {
			#print "header: [$line] number of values: $#values\n";
			# we have a new header, and we must use these labels for importing data
			@labels = split(/,\s*/,$line);
			# don't include "iteration" in the labels, that's the first key in our hash
			shift @labels;
			$primary_metric = $labels[0];
			$primary_metric =~ s/\s/_/g; #we don't want spaces in our metrics
			#print "primary_metric: [$primary_metric]\n";
		} else {
			my $value;
			@values = split(/,/,$line);
			if ($#values > $#labels) {
				$iteration = shift(@values);

				# Strip the leading spaces, and also strip the iteration number
				# It's possible that comparing different results could not have matching iteration numbers,
				# but the iteration description should match
				$iteration =~ s/^\s*\d+\-//;
				my $index = 0;
				foreach $value (@values) {
					$value =~ s/\s+//;
					my $this_label = $labels[$index];
					$this_label =~ s/\s/_/g; #we don't want spaces in our metrics
					# when we only have "stddevpct", prepend the previous label so we know what context the stddev is for
					if ( $this_label =~ /stddev/ ) {
						$this_label = $labels[$index-1] . "-" . $this_label;
					}
					$results{$primary_metric}{$iteration}{$this_label}{$dir} = "$value";
					$index++;
				}
			}
		}
	}
	close CSV_FILE;
	#print "result_$dir_num: $dir\n";
	$dir_num++;
}
print "\n";

my $iteration_spacing = 28;
my $value_spacing = 12;
my $values_spacing = $value_spacing * $dir_num;
my %ratios;
my %dir_ratios;
my %dir_ratios_delta;
my $height_mod;

# output the txt comparison
open(TXT_FILE, ">$comparison_name.txt") || die "$script: could not open file $comparison_name.txt: $!\n";
foreach $primary_metric ( sort keys %results ) {
	#print "primary_metric: [$primary_metric]\n";
	my $i = 1;
	foreach $iteration ( sort mysortlabels keys %{ $results{$primary_metric} } ) {
		@labels = (sort keys %{ $results{$primary_metric}{$iteration} });
		#print "labels: [@labels]\n";
		# shift around the array until the primary metric is first
		while ($labels[0] ne $primary_metric) {
			unshift(@labels, pop(@labels));
		}
		# right before printing the first iteration's values, print a header
		if ( $i == 1 ) {
			printf TXT_FILE "%${iteration_spacing}s", "";
			foreach $label (@labels) {
				if ( $label =~ /stddev/ ) {
					next;
				}
				printf TXT_FILE "[%${values_spacing}s]", $label;
			}
			printf TXT_FILE "\n";

			# also print a header under each label, identifying the results directory number for each column
			printf TXT_FILE "%${iteration_spacing}s", "";
			foreach $label (@labels) {
				if ( $label =~ /stddev/ ) {
					next;
				}
				my $dir_num_list = "";
				my $dir_num = 0;
				while ( defined $dirs[$dir_num] ) {
					$dir_num_list = $dir_num_list . sprintf "%${value_spacing}s", "result_" . $dir_num;
					$dir_num++;
				}
				$dir_num_list =~ s/^,//;
				printf TXT_FILE "[%${values_spacing}s]", $dir_num_list;
			}
			printf TXT_FILE "\n";
		}

		# now we can print the values from the results from a common iteration
		my $row1 = sprintf "%${iteration_spacing}s", $iteration;
		my $row2 = sprintf "%${iteration_spacing}s", "";
		#printf "Doing ratios: \$primary_metric: \"%s\", \$iteration: \"%s\", ...\n", $primary_metric, $iteration;
		foreach $label (@labels) {
			if ( $label =~ /stddev/ ) {
				next;
			}
			my $values_list = "";
			my $ratios_list = "";
			my $value_base;
			my $dir_num = 1;
			foreach $dir (@dirs) {
				#printf "\$label = \"%s\", \$dir = \"%s\" \$dir_num = %d :: ", "$label", "$dir", $dir_num;
				my $value;
				my $ratio;
				if ($results{$primary_metric}{$iteration}{$label}{$dir}) {
					$value = sprintf "%.1f", $results{$primary_metric}{$iteration}{$label}{$dir};
					#print "$value\n";
					if ($dir_num == 1) {
						#print "base\n";
						$value_base = $value;
						$ratio = "ratio->";
					} else {
						if ($value_base > 0)  {
							$ratio = $value / $value_base;
							#printf "pushing ratio[$ratio] in to \$ratios\{\$primary_metric\}\{\$label\}\{\$dir\}\n";
							push( @{ $ratios{$primary_metric}{$label}{$dir} }, $ratio);
							$ratio = sprintf "%.2fx", $ratio;
						} else {
							#print "no ratio\n";
							$ratio = "";
						}
					}
				} else {
					$value = "n/a";
					if ($dir_num eq 1) {
						$value_base = -1;
					}
					$ratio = "";
				}
				#printf "\$value = \"%s\", \$ratio = \"%s\"\n", "$value", "$ratio";
				$values_list = $values_list . sprintf "%${value_spacing}s", $value;
				$ratios_list = $ratios_list . sprintf "%${value_spacing}s", $ratio;
				$dir_num++;
			}
			$values_list =~ s/^,//;
			$row1 = $row1 . sprintf "[%${values_spacing}s]", $values_list;
			$row2 = $row2 . sprintf "[%${values_spacing}s]", $ratios_list;
		}
		printf TXT_FILE "$row1\n";
		printf TXT_FILE "$row2\n";
		$i++;
	}

	# for each label (each benhcmark metric) print the combined ratio from all iterations
	my $last_row = sprintf "%${iteration_spacing}s", "RATIO GEO_MEAN-->";
	# the last iteration is as good as any to get the list of labels
	#printf "Doing geo-means ... \$primary_metric = %s\n", "$primary_metric";
	@labels = (sort keys %{ $ratios{$primary_metric} });
	# shift around the array until the primary metric is first
	my $labels_len = @labels;
	if ($labels_len > 1) {
		while ($labels[0] ne $primary_metric) {
			unshift(@labels, pop(@labels));
		}
	}
	foreach $label (@labels) {
		if ( $label =~ /stddev/ ) {
			next;
		}
		my $dir_num = 1;
		my $ratios_list = "";
		foreach $dir (@dirs) {
			#printf "\$primary_metric = %s, \$label = %s, \$dir = %s\n", "$primary_metric", "$label", "$dir";
			if ($dir_num == 1) {
				$ratios_list = $ratios_list . sprintf "%${value_spacing}s", "1.00x";
				$dir_ratios{$label}{$dir} = 1;
			} else {
				if (not defined $ratios{$primary_metric}{$label}{$dir}) {
					#printf "\$ratios{%s}{%s}{%s} is undefined\n", "$primary_metric", "$label", "$dir";
				} else {
					#print "ratios: [@{ $ratios{$primary_metric}{$label}{$dir} }]\n";
					my $dir_ratio = 1;
					my $ratio;
					my $nr_ratios = 0;
					foreach $ratio ( @{ $ratios{$primary_metric}{$label}{$dir} } ) {
						$dir_ratio *= $ratio;
						$nr_ratios++;
					}
					$dir_ratio = $dir_ratio **(1/$nr_ratios);
					$dir_ratios{$label}{$dir} = $dir_ratio;
					$dir_ratios_delta{$label}{$dir} = $dir_ratio - 1;
					$dir_ratio = sprintf "%.2fx", $dir_ratio;
					$ratios_list = $ratios_list . sprintf "%${value_spacing}s", "$dir_ratio";
				}
			}
			$dir_num++;
		}
		$height_mod = ($dir_num -1)*8 + 110;
		$last_row = $last_row . sprintf "[%${values_spacing}s]", $ratios_list;
	}
	printf TXT_FILE "$last_row\n";
}

close(TXT_FILE);

# output the json data for primary metric, to be used by multiBarChart in nvd3
# Note: this only includes the primary metric right now, but should be enhanced to do all metrics
# FIXME: convert to csv when GenData pm is redy to support multiBar graph
foreach $primary_metric ( sort keys %results ) { # one json file for each primary_metric

	open(HTML_FILE, ">$comparison_name-$primary_metric.html");
	printf HTML_FILE "<!DOCTYPE HTML>\n";
	printf HTML_FILE "<html>\n";
	printf HTML_FILE "  <head>\n";
	printf HTML_FILE "    <meta charset=\"utf-8\">\n";
	printf HTML_FILE "    <link href=\"/static/css/v0.1/nv.d3.css\" rel=\"stylesheet\" type=\"text/css\" media=\"all\">\n";
	printf HTML_FILE "    <script src=\"/static/js/v0.1/function-bind.js\"></script>\n";
	printf HTML_FILE "    <script src=\"/static/js/v0.1/d3.v3.js\"></script>\n";
	printf HTML_FILE "    <script src=\"/static/js/v0.1/nv.d3.js\"></script>\n";
	printf HTML_FILE "    <script src=\"$comparison_name-$primary_metric.js\"></script>\n";
	printf HTML_FILE "\n";
	printf HTML_FILE "    <style>\n";
	printf HTML_FILE "      text {\n";
	printf HTML_FILE "        font: 12px sans-serif;\n";
	printf HTML_FILE "      }\n";
	printf HTML_FILE "      .resultname {\n";
	printf HTML_FILE "        font-weight: bold;\n";
	printf HTML_FILE "      }\n";
	printf HTML_FILE "      .resultvalue {\n";
	printf HTML_FILE "        text-align: right;\n";
	printf HTML_FILE "      }\n";
	printf HTML_FILE "      svg.chart {\n";
	printf HTML_FILE "        margin: 0px;\n";
	printf HTML_FILE "        padding: 0px;\n";
	printf HTML_FILE "        height: 100%%;\n";
	printf HTML_FILE "        width: 100%%;\n";
	printf HTML_FILE "      }\n";
	printf HTML_FILE "      div.chart {\n";
	printf HTML_FILE "        display: block;\n";
	printf HTML_FILE "        overflow: auto;\n";
	printf HTML_FILE "        margin: 0px;\n";
	printf HTML_FILE "        padding: 0px;\n";
	printf HTML_FILE "        height: 90%%;\n";
	printf HTML_FILE "        width: 100%%;\n";
	printf HTML_FILE "      }\n";
	printf HTML_FILE "      table.chart, table.chart thead th, table.chart tbody td {\n";
	printf HTML_FILE "        border: 1px solid black;\n";
	printf HTML_FILE "      }\n";
	printf HTML_FILE "      table.chart thead th, table.chart tbody td {\n";
	printf HTML_FILE "        padding: 4px;\n";
	printf HTML_FILE "      }\n";
	printf HTML_FILE "      table.chart {\n";
	printf HTML_FILE "        height: 10%%;\n";
	printf HTML_FILE "        margin: 0px auto;\n";
	printf HTML_FILE "      }\n";
	printf HTML_FILE "      html, body {\n";
	printf HTML_FILE "        margin: 0px;\n";
	printf HTML_FILE "        padding: 0px;\n";
	printf HTML_FILE "        height: 100%%;\n";
	printf HTML_FILE "        width: 100%%;\n";
	printf HTML_FILE "      }\n";
	printf HTML_FILE "    </style>\n";
	printf HTML_FILE "\n";
	printf HTML_FILE "  </head>\n";
	printf HTML_FILE "  <body>\n";
	printf HTML_FILE "    <div class=\"chart\">\n";
	printf HTML_FILE "      <svg class=\"chart\"></svg>\n";
	printf HTML_FILE "    </div>\n";

	open(JS_FILE, ">$comparison_name-$primary_metric.js");
	print JS_FILE "$primary_metric = [";
	my $i=1;
	printf HTML_FILE "    <table class=\"chart\">\n";
	printf HTML_FILE "      <thead>\n";
	printf HTML_FILE "        <tr>\n";
	printf HTML_FILE "          <th>comparison</th><th class=\"resultvalue\">geo-mean-ratio</th><th class=\"resultvalue\">buffalo-wings</th>\n";
	printf HTML_FILE "        </tr>\n";
	printf HTML_FILE "      </thead>\n";
	printf HTML_FILE "      <tbody>\n";
	my $result_one_dir = "";
	foreach $dir (@dirs) {
		if ($i > 1) {
			printf HTML_FILE "        <tr>\n";
			my $dr;
			my $drd;
			if (not defined $dir_ratios{$primary_metric}{$dir}) {
				$dr = "n/a";
				$drd = "n/a";
			} else {
				$dr = sprintf "%.2f", $dir_ratios{$primary_metric}{$dir};
				if (not defined $dir_ratios{$primary_metric}{$dir}) {
					$drd = "n/a";
				} else {
					$drd = sprintf "%.2f", $dir_ratios_delta{$primary_metric}{$dir};
				}
			}
			printf HTML_FILE "          <td class=\"resultname\"><a href=\"%s\">result_%d</a>/<a href=\"%s\">result_1</a></td><td class=\"resultvalue\">%s</td><td class=\"resultvalue\">%s</td>\n", $dir, $i, $result_one_dir, $dr, $drd;
			printf HTML_FILE "        </tr>\n";
		} else {
			$result_one_dir = $dir;
		}
		my $key_separator;
		if ( $i != 1 ) {
			$key_separator = ",";
		} else {
			$key_separator = "";
		}
		print JS_FILE "$key_separator\n";
		print JS_FILE "  {\n";
		print JS_FILE "    key: \'$dir\',\n";
		print JS_FILE "    values: [";
		my $j = 1;
		foreach $iteration ( sort mysortlabels keys %{ $results{$primary_metric} } ) { # one label per $iteration
			my $value;
			my $iteration_separator;
			if ($results{$primary_metric}{$iteration}{$primary_metric}{$dir}) {
				$value = sprintf "%.1f", $results{$primary_metric}{$iteration}{$primary_metric}{$dir};
			} else {
				next;
			}

			if ( $j != 1 ) {
				$iteration_separator = ",";
			} else {
				$iteration_separator = "";
			}
			print JS_FILE "$iteration_separator\n";
			print JS_FILE "       {\n";
			print JS_FILE "         \"label\" : \"$iteration\",\n";
			print JS_FILE "         \"value\" : $value\n";
			print JS_FILE "       }";
			$j++;
		}
		print JS_FILE "\n     ]";
		print JS_FILE "\n  }";
		$i++;
	}
	printf HTML_FILE "      </tbody>\n";
	printf HTML_FILE "    </table>\n";
	printf JS_FILE "\n]\n";
	close(JS_FILE);

	# genrate the html file
	# FIXME: this script should use GenData from ./tool-scripts/postprocess
	# GenData needs to first support multiBar graphs

	printf HTML_FILE "\n";
	printf HTML_FILE "\n";
	printf HTML_FILE "    <script>\n";
	printf HTML_FILE "      var strP = /[^a-zA-Z\-]/g;\n";
	printf HTML_FILE "      var numsP = /[^0-9]/g;\n";
	printf HTML_FILE "      function sortParams(a,b) {\n";
	printf HTML_FILE "        var a_str = a.label.replace(strP, '');\n";
	printf HTML_FILE "        var b_str = b.label.replace(strP, '');\n";
	printf HTML_FILE "        if (a_str === b_str) {\n";
	printf HTML_FILE "          var a_num = parseInt(a.label.replace(numsP, \"\"), 10);\n";
	printf HTML_FILE "          var b_num = parseInt(b.label.replace(numsP, \"\"), 10);\n";
	printf HTML_FILE "          return a_num === b_num ? 0 : a_num > b_num ? 1 : -1;\n";
	printf HTML_FILE "        } else {\n";
	printf HTML_FILE "          return a_str > b_str ? 1 : -1;\n";
	printf HTML_FILE "        }\n";
	printf HTML_FILE "      }\n";
	printf HTML_FILE "\n";
	printf HTML_FILE "      for (var i = 0; i < " . $primary_metric . ".length; i++) {\n";
	printf HTML_FILE "        %s[i].values = %s[i].values.sort(sortParams);\n", $primary_metric, $primary_metric;
	printf HTML_FILE "      }\n";
	printf HTML_FILE "\n";

	printf HTML_FILE "      var chart;\n";
	printf HTML_FILE "      nv.addGraph(function() {\n";
	printf HTML_FILE "        chart = nv.models.multiBarHorizontalChart()\n";
	printf HTML_FILE "            .x(function(d) { return d.label })\n";
	printf HTML_FILE "            .y(function(d) { return d.value })\n";
	printf HTML_FILE "            .margin({top: 30, right: 20, bottom: 50, left: 175})\n";
	printf HTML_FILE "            .showValues(true)    //Show bar value next to each bar.\n";
	printf HTML_FILE "            .tooltips(true);     //Show tooltips on hover.\n";
	printf HTML_FILE "        chart.yAxis.tickFormat(d3.format(',.2f'));\n";
	printf HTML_FILE "        d3.select('svg.chart')\n";
	printf HTML_FILE "            .datum($primary_metric)\n";
	printf HTML_FILE "            .call(chart);\n";
	printf HTML_FILE "        nv.utils.windowResize(chart.update);\n";
	printf HTML_FILE "        return chart;\n";
	printf HTML_FILE "      });\n";
	printf HTML_FILE "    </script>\n";
	printf HTML_FILE "  </html>\n";
	printf HTML_FILE "</html>\n";
	close(HTML_FILE);
}
