#!/usr/bin/perl

use strict;
use NOCpulse::CommandQueue;
use NOCpulse::Gritch;
use NOCpulse::Debug;
use NOCpulse::SatCluster;
use NOCpulse::NotificationQueue;
use Getopt::Long;

# Save invocation options so I can restart myself
my $cmd = $0;
if ($cmd !~ m,^/,) {
  chomp(my $pwd = `pwd`);
  $cmd = "$pwd/$cmd";
}
my @restart = ($0, @ARGV);

my $debuglevel;
my $loglevel;
&GetOptions('debug=i'    => \$debuglevel,
            'loglevel=i' => \$loglevel);

# Global variables
my $cfg      = new NOCpulse::Config;
my $queue    = new NOCpulse::CommandQueue();
my $cluster  = NOCpulse::SatCluster->newInitialized($cfg);
my $clusterid = $cluster->get_id();
my $nodeid   = $cluster->get_nodeId();
my $INTERVAL = $cfg->get('CommandQueue', 'pollingInterval');
my $RETRIES  = 3;

# Set up for memory bloat checking
my $MAXMEM           = 50000;
my $MEMCHECKINTERVAL = 300;
my $LASTMEMCHECK     = time;

# Set up for debugging and log output and error notification
my $mama     = new NOCpulse::Gritch($cfg->get('CommandQueue', 'gritchdb'));
my $debug    = $queue->debug();

# Set up a heartbeat file
my $hbfile  = $cfg->get('CommandQueue', 'heartbeatFile');
$queue->heartbeatfile($hbfile);
my $hbfreq  = $queue->heartbeatfreq();

if ($debuglevel) {
  my $debugstream = $debug->addstream(LEVEL => $debuglevel);

  # -- Print gritches to STDOUT if --debug
  $mama->recipient($debugstream);

}  else {

  # -- Send gritches via the notification system otherwise
   $mama->recipient(NOCpulse::NotificationQueue->new( Debug=> $debug, Config => $cfg, Gritcher => $mama));
}

# -- Always add the log file
my $logfile         = $cfg->get('CommandQueue', 'exelog');
my $logbasename     = $logfile; $logbasename =~ s,^.*/,,;
my $archive         = $cfg->get('netsaint', 'archiveDir');
my @t               = gmtime(time);


$loglevel = $cfg->get('CommandQueue', 'exelogLevel') unless ($loglevel);
my $log   = $debug->addstream(LEVEL  => $loglevel,
                              FILE   => $logfile,
		              APPEND => 1);

# -- Configure log file output
$log->linenumbers(1);
$log->timestamps(1);
$log->autoflush(1);


# Make /tmp the CWD for running commands
chdir("/tmp");



# Here we go ...
$debug->dprint(1, '*' x 75, "\n");
$debug->dprint(1, "Starting command execution daemon for cluster $clusterid node $nodeid\n");
$debug->dprint(1, '*' x 75, "\n");
$debug->dprint(1, "Logging to $logfile\n");
$debug->dprint(1, "Log level: $loglevel\n");
$debug->dprint(1, "Heartbeat file: $hbfile\n");
$debug->dprint(1, "Heartbeat frequency: $hbfreq seconds\n");
$debug->dprint(1, "Restart command: @restart\n");


# Daemon loop
while (1) {

  $debug->dprint(1, "Starting queue run\n");

  # Update heartbeat file
  $queue->heartbeat();

  # Try up to $RETRIES times to fetch commands
  my($rv, $i);
  for ($i = 0; $i < $RETRIES; $i++) {
    $debug->dprint(3, "\tAttempting to fetch commands (attempt $i)\n");
    last if ($rv = $queue->fetch_commands());
    $debug->dprint(2, "\tfetch_commands failed: $@\n");
  }

  my $num_run;
  if (! defined($rv)) {

    my $subject = 'Unable to fetch commands'; 
    my $message = "Unable to fetch commands in $RETRIES tries: $@\n";

    $debug->dprint(1, $message);
    $mama->gritch($subject, $message);

  } else {

    my $commandsref = $queue->commands();
    $debug->dprint(2, "\tFetched ", scalar(keys %$commandsref), " commands\n");

    my $cid;
    foreach $cid (sort {$a <=> $b} keys %$commandsref) {
      my $command = $commandsref->{$cid};

      # 'run' executes conditionally, i.e. only if the command is
      # in sequence, hasn't been run yet (or was interrupted and is
      # restartable), and hasn't yet expired.
      $debug->dprint(2, "\tRunning command $cid\n");
      $debug->dprint(3, "\t\tCommand line is:  ", $command->command_line, "\n");
      $debug->dprint(4, "\t\tCommand is:\n", $command->as_str());

      my $rv = $command->run();

      if (defined($rv)) {

        $num_run++;

      } elsif ($rv == -1) {

	my $subject = 'Command execution failed'; 
	my $message = "Failed to execute command $cid: $@\n";

	$debug->dprint(1, $message);
	$mama->gritch($subject, $message);

      } else {

        $debug->dprint(2, "\tNot executing command $cid: $@\n");

      }

      $debug->dprint(2, "\tFinished with command $cid\n");

    }
    $debug->dprint(1, "Finished processing commands\n");

  }


  # Check for bloat
  if (time - $LASTMEMCHECK > $MEMCHECKINTERVAL) {
    $debug->dprint(1, "Checking bloat\n");
    &memcheck($MAXMEM);
    $LASTMEMCHECK = time;
  }

  # Update heartbeat file
  $queue->heartbeat();

  # Sleep $INTERVAL seconds if there were no commands processed
  if ($num_run) {
    $debug->dprint(1, "Not sleeping after processing commands\n");
  } else {
    $debug->dprint(1, "No commands executed, sleeping $INTERVAL seconds\n");
    sleep $INTERVAL;
  }
}



sub memcheck {
  my $maxmem = shift;
  # chomp(my $mem = `ps -o vsz="" -p $$`);
  local *PROC;
  open PROC, "/proc/$$/stat" or do {
    $debug->dprint(1, "\tError reading [/proc/$$/stat]: $!\n");
    return;
  };
  my $line = <PROC>;
  close PROC;
  my $mem = (split /\s/, $line)[22] / 1024;
  if ($mem > $maxmem) {
    $debug->dprint(1, "\tI am bloated ($mem kb) and will now restart myself\n");
    exec @restart;
  } else {
    $debug->dprint(1, "\tI am not bloated ($mem kb)\n");
  }
}
