#!/usr/bin/bash
#
# Watchdog Functions
# Copyright (C) 2019-2024 by Thomas Dreibholz
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Contact: dreibh@simula.no
#


# ###### Main program #######################################################
if [ $# -ne 1 ] ; then
   echo >&2 "Usage: $0 configuration_file"
   exit 1
fi
WATCHDOG_CONFIG="$1"


WATCHDOG_FAILCOUNT_FILE="watchdog-failcount"
WATCHDOG_COMMAND=""   # false
WATCHDOG_MAXFAILURES="3"
WATCHDOG_MAXFINALTRIALS="2"
WATCHDOG_LOOPMODE=0
WATCHDOG_FAILURE_ACTION='echo "Something is going wrong!"'
WATCHDOG_FAILURE_FINAL='echo "Something is going *really* wrong!"'

if [ ! -e $WATCHDOG_CONFIG ] ; then
  exit 0
fi
. $WATCHDOG_CONFIG || exit 0

if [ "$WATCHDOG_COMMAND" = "" ] ; then
   # Nothing to do!
   exit 0
fi


if eval "$WATCHDOG_COMMAND" >/dev/null ; then
   failures=0
   if [ -e "$WATCHDOG_FAILCOUNT_FILE" ] ; then
      echo -en "\x1b[32m`date +%FT%H:%M:%S`: Check succeeded. Resetting failure counter.\x1b[0m\n"
   fi
   rm -f "$WATCHDOG_FAILCOUNT_FILE"
else
   failures=`cat "$WATCHDOG_FAILCOUNT_FILE" 2>/dev/null || echo "0"`
   let failures=1+$failures || failures=1

   if [ $failures -ge $WATCHDOG_MAXFAILURES ] ; then
      echo -en "\x1b[34m`date +%FT%H:%M:%S`: Check failed $failures/$WATCHDOG_MAXFAILURES --> Performing action!\x1b[0m\n"
      if [ $WATCHDOG_LOOPMODE -eq 0 ] ; then
         echo "-1000000" >"$WATCHDOG_FAILCOUNT_FILE"
      else
         rm -f "$WATCHDOG_FAILCOUNT_FILE"
      fi
      eval $WATCHDOG_FAILURE_ACTION
   elif [ $failures -lt 0 -a $failures -ge -1000000 ] ; then
      let finalTrials=$failures+1000000
      echo -en "\x1b[34m`date +%FT%H:%M:%S`: FINAL check failed $finalTrials/$WATCHDOG_MAXFINALTRIALS: $WATCHDOG_COMMAND\x1b[0m\n"
      if [ $finalTrials -eq $WATCHDOG_MAXFINALTRIALS ] ; then
         echo -en "\x1b[33m`date +%FT%H:%M:%S`: Check failed FINALLY --> Performing final action!\x1b[0m\n"
         echo "-1111111" >"$WATCHDOG_FAILCOUNT_FILE"
         eval $WATCHDOG_FAILURE_FINAL
      else
         echo "$failures" >"$WATCHDOG_FAILCOUNT_FILE"
      fi
   elif [ $failures -gt 0 ] ; then
      echo -en "\x1b[34m`date +%FT%H:%M:%S`: Check failed $failures/$WATCHDOG_MAXFAILURES: $WATCHDOG_COMMAND\x1b[0m\n"
      echo "$failures" >"$WATCHDOG_FAILCOUNT_FILE"
   fi
fi
