#!/usr/bin/python3
# LAVA Mobility Watchdog - Copyright 2021 LAVA Controls - Stephen Loeckle

import configparser
import inspect
from lavamobilitylib import version
import logging, logging.handlers
import os
import shlex
import subprocess
import sys
import threading
from time import time, sleep
    
def ping(host=None, count=1, socketid=0):
    gotthefunc = inspect.stack()[0][3]
    msg = ('Entered {} function'.format(gotthefunc))
    sendlog(gotthefunc, msg, 10)
    ts = time()
    s = {}
    min = 0
    avg = 0
    max = 0
    up = 0
    msg = 'Beginning ping {}'.format(host)
    sendlog(gotthefunc, msg, 10)
    try:
        p = subprocess.run(['ping', '-c', str(count), '-W', '2', host], stdout=subprocess.PIPE)
    except:
        pass
        msg = ("Unexpected Error in {}!:".format(gotthefunc))
        sendlog(gotthefunc, msg, 40)
        err = traceback.format_exc()
        msg = ('Unexpected Error in {0} Detail: {1}'.format(gotthefunc,err))
        sendlog(gotthefunc, msg, 10)
        up = 0
        return(up,avg)
    else:
        resultcode = p.returncode
        msg = 'Ping result code = {}'.format(resultcode)
        sendlog(gotthefunc, msg, 40)
        p = p.stdout.decode('utf-8')
        for line in p.splitlines():
            if 'rtt' in line:
                line = line.replace(' ','').replace('ms','')
                line = line.split('=')
                min, avg, max, mdev = line[1].split('/')
                min = round(float(min),2)
                avg = round(float(avg),2)
                max = round(float(max),2)
            else:
                min = 0
                avg = 0
                max = 0

    msg = 'Ping complete. Testing results: {} {} {}'.format(min,avg,max)
    sendlog(gotthefunc, msg, 10)
    if resultcode == 0:
        up = 1
    else:
        up = 0
    msg = ('Exited {} function'.format(gotthefunc))
    sendlog(gotthefunc, msg, 10)
    return(up,avg)
                
def sendlog(functionname, msg, level):
    id = functionname
    logger = logging.getLogger('main.{}'.format(id))
    if level == 10:
        logger.log(logging.DEBUG, msg)
    if level == 20:
        logger.log(logging.INFO, msg)
    if level == 30:
        logger.log(logging.WARNING, msg)
    if level == 40:
        logger.log(logging.ERROR, msg)
    if level == 50:
        logger.log(logging.CRITICAL, msg)
    
def main():
    gotthefunc = inspect.stack()[0][3]
    msg = ('Entered {} function'.format(gotthefunc))
    sysinfo = {}
    sysinfo['SYSTEM'] = {}
    sysinfo['SYSTEM']['LAVAMOBILITYLOGFILE'] = ('/var/log/lavamobility/lmwatchdog.log')
    sysinfo['SYSTEM']['LMWATCHDOGCONFIGFILE'] = ('/etc/lavamobility/lmwatchdog.conf')
    
    down = 0
    iface = 'ppp0'
    pinghost = 'google.com'
    pinginterval = 15
    rebootcmd = '/usr/sbin/reboot'
    restartifacewait = 30
    restartstart = 5
    restartend = 9
    restartenabled = 1
    rebootstart = 10
    rebootenabled = 1
    restartnetworking = 'service networking restart'
    stopiface = '/usr/sbin/ifdown {0}'.format(iface)
    startiface = '/usr/sbin/ifup {0}'.format(iface)
    wasdown = 0
    
    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(processName)-10s %(name)s %(levelname)-8s %(message)s',handlers=[logging.handlers.RotatingFileHandler(sysinfo['SYSTEM']['LAVAMOBILITYLOGFILE'], mode='a', maxBytes=10000*1024, backupCount=30, encoding=None, delay=False)])
    logger = logging.getLogger('main')
    
    config = configparser.ConfigParser()
    try:
        config.read([sysinfo['SYSTEM']['LMWATCHDOGCONFIGFILE']])
    except configparser.Error as e:
        msg = ('Unable to read configuration file {}! Loading defaults!'.format(sysinfo['SYSTEM']['LMWATCHDOGCONFIGFILE']))
        sendlog(gotthefunc, msg, 10)
        pass
    except:
        msg = ('Error in {0} Reading Config! Loading defaults!'.format(gotthefunc))
        sendlog(gotthefunc, msg, 10)
        err = traceback.format_exc()
        msg = ('Error Detail: {0}'.format(err))
        sendlog(gotthefunc, msg, 10)
        pass
    else:
        msg = ('CONFIG: Loaded config file')
        sendlog(gotthefunc, msg, 10)
        
    try:
        sysinfo['SYSTEM']['IFACE'] = config.get('system', 'iface')
    except (configparser.NoOptionError, configparser.InterpolationMissingOptionError):
        msg = ('No monitored iface in configuration file! Defaulting to {}!'.format(iface))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['IFACE'] = iface
    except configparser.Error as e:
        msg = ('Config read entry monitored iface error! Defaulting to {}!'.format(iface))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['IFACE'] = iface
    except:
        msg = ('Error in {0} Reading Config! Defaulting to {}!'.format(gotthefunc,iface))
        sendlog(gotthefunc, msg, 40)
        err = traceback.format_exc()
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['IFACE'] = iface
        
    try:
        sysinfo['SYSTEM']['PINGINTERVAL'] = config.get('system', 'pinginterval')
    except (configparser.NoOptionError, configparser.InterpolationMissingOptionError):
        msg = ('No pinginterval in configuration file! Defaulting to {}!'.format(pinginterval))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['PINGINTERVAL'] = pinginterval
    except configparser.Error as e:
        msg = ('Config read entry pinginterval error! Defaulting to {}!'.format(pinginterval))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['PINGINTERVAL'] = pinginterval
    except:
        msg = ('Error in {0} Reading Config! Defaulting to {}!'.format(gotthefunc,pinginterval))
        sendlog(gotthefunc, msg, 40)
        err = traceback.format_exc()
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['PINGINTERVAL'] = pinginterval
        
    try:
        sysinfo['SYSTEM']['PINGHOST'] = config.get('system', 'pinghost')
    except (configparser.NoOptionError, configparser.InterpolationMissingOptionError):
        msg = ('No pinghost in configuration file! Defaulting to {}!'.format(pinghost))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['PINGHOST'] = pinghost
    except configparser.Error as e:
        msg = ('Config read entry pinghost error! Defaulting to {}!'.format(pinghost))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['PINGHOST'] = pinghost
    except:
        msg = ('Error in {0} Reading Config! Defaulting to {}!'.format(gotthefunc,pinghost))
        sendlog(gotthefunc, msg, 40)
        err = traceback.format_exc()
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['PINGHOST'] = pinghost
        
    try:
        sysinfo['SYSTEM']['RESTARTIFACEWAIT'] = config.get('system', 'restartifacewait')
    except (configparser.NoOptionError, configparser.InterpolationMissingOptionError):
        msg = ('No restartifacewait in configuration file! Defaulting to {}!'.format(restartifacewait))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTIFACEWAIT'] = restartifacewait
    except configparser.Error as e:
        msg = ('Config read entry restartifacewait error! Defaulting to {}!'.format(restartifacewait))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTIFACEWAIT'] = restartifacewait
    except:
        msg = ('Error in {0} Reading Config! Defaulting to {}!'.format(gotthefunc,restartifacewait))
        sendlog(gotthefunc, msg, 40)
        err = traceback.format_exc()
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTIFACEWAIT'] = restartifacewait
        
    try:
        sysinfo['SYSTEM']['RESTARTSTART'] = config.get('system', 'restartstart')
    except (configparser.NoOptionError, configparser.InterpolationMissingOptionError):
        msg = ('No restartstart in configuration file! Defaulting to {}!'.format(restartstart))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTSTART'] = restartstart
    except configparser.Error as e:
        msg = ('Config read entry restartstart error! Defaulting to {}!'.format(restartstart))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTSTART'] = restartstart
    except:
        msg = ('Error in {0} Reading Config! Defaulting to {}!'.format(gotthefunc,restartstart))
        sendlog(gotthefunc, msg, 40)
        err = traceback.format_exc()
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTSTART'] = restartstart
        
    try:
        sysinfo['SYSTEM']['RESTARTEND'] = config.get('system', 'restartend')
    except (configparser.NoOptionError, configparser.InterpolationMissingOptionError):
        msg = ('No restartend in configuration file! Defaulting to {}!'.format(restartend))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTEND'] = restartend
    except configparser.Error as e:
        msg = ('Config read entry restartend error! Defaulting to {}!'.format(restartend))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTEND'] = restartend
    except:
        msg = ('Error in {0} Reading Config! Defaulting to {}!'.format(gotthefunc,restartend))
        sendlog(gotthefunc, msg, 40)
        err = traceback.format_exc()
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTEND'] = restartend
        
    try:
        sysinfo['SYSTEM']['REBOOTSTART'] = config.get('system', 'rebootstart')
    except (configparser.NoOptionError, configparser.InterpolationMissingOptionError):
        msg = ('No rebootstart in configuration file! Defaulting to {}!'.format(rebootstart))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['REBOOTSTART'] = rebootstart
    except configparser.Error as e:
        msg = ('Config read entry rebootstart error! Defaulting to {}!'.format(rebootstart))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['REBOOTSTART'] = rebootstart
    except:
        msg = ('Error in {0} Reading Config! Defaulting to {}!'.format(gotthefunc,rebootstart))
        sendlog(gotthefunc, msg, 40)
        err = traceback.format_exc()
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['REBOOTSTART'] = rebootstart
        
    try:
        sysinfo['SYSTEM']['REBOOTENABLED'] = config.get('system', 'rebootenabled')
    except (configparser.NoOptionError, configparser.InterpolationMissingOptionError):
        msg = ('No rebootenabled in configuration file! Defaulting to {}!'.format(rebootenabled))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['REBOOTENABLED'] = rebootenabled
    except configparser.Error as e:
        msg = ('Config read entry rebootenabled error! Defaulting to {}!'.format(rebootenabled))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['REBOOTENABLED'] = rebootenabled
    except:
        msg = ('Error in {0} Reading Config! Defaulting to {}!'.format(gotthefunc,rebootenabled))
        sendlog(gotthefunc, msg, 40)
        err = traceback.format_exc()
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['REBOOTENABLED'] = rebootenabled
        
    try:
        sysinfo['SYSTEM']['RESTARTENABLED'] = config.get('system', 'restartenabled')
    except (configparser.NoOptionError, configparser.InterpolationMissingOptionError):
        msg = ('No restartenabled in configuration file! Defaulting to {}!'.format(restartenabled))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTENABLED'] = restartenabled
    except configparser.Error as e:
        msg = ('Config read entry restartenabled error! Defaulting to {}!'.format(restartenabled))
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTENABLED'] = restartenabled
    except:
        msg = ('Error in {0} Reading Config! Defaulting to {}!'.format(gotthefunc,restartenabled))
        sendlog(gotthefunc, msg, 40)
        err = traceback.format_exc()
        sendlog(gotthefunc, msg, 10)
        sysinfo['SYSTEM']['RESTARTENABLED'] = restartenabled
            
    msg = 'LAVA Mobility Watchdog {} Startup'.format(version())
    sendlog(gotthefunc, msg, 10)
    
    while True:
        isup = None
        isup,avg = ping(sysinfo['SYSTEM']['PINGHOST'],sysinfo['SYSTEM']['PINGINTERVAL'])
        if isup == 1:
            if wasdown == 1:
                msg = ('{} was down and is now back up without rebooting. Average ping time {}'.format(sysinfo['SYSTEM']['PINGHOST'],avg))
                wasdown = 0
            else:
                msg = ('{} is up. Average ping time {}'.format(sysinfo['SYSTEM']['PINGHOST'],avg))
            sendlog(gotthefunc, msg, 10)
            down = 0
        else:
            msg = ('{} is down. Average ping time {}'.format(sysinfo['SYSTEM']['PINGHOST'],avg))
            down += 1
        if down >= sysinfo['SYSTEM']['RESTARTSTART'] and down <= sysinfo['SYSTEM']['RESTARTEND'] and sysinfo['SYSTEM']['RESTARTENABLED'] == 1:
#            msg = ('{} is down. Stopping {} interface.'.format(pinghost,iface))
#            sendlog(gotthefunc, msg, 10)
#            p = subprocess.Popen(shlex.split(stopiface), shell=True, start_new_session=True)
#            sleep(5)
#            msg = ('Starting {} interface.'.format(iface))
#            sendlog(gotthefunc, msg, 10)
#            p = subprocess.Popen(shlex.split(startiface), shell=True, start_new_session=True)
#            msg = ('Started {} interface. Sleeping for 30 seconds.'.format(iface))
#            sendlog(gotthefunc, msg, 10)
            p = subprocess.Popen(shlex.split(restartnetworking), start_new_session=True)
            msg = ('Network daemon restarted. Sleeping for 30 seconds.'.format(sysinfo['SYSTEM']['IFACE']))
            sendlog(gotthefunc, msg, 10)            
            sleep(sysinfo['SYSTEM']['RESTARTIFACEWAIT'])
            wasdown = 1
        elif down >= sysinfo['SYSTEM']['REBOOTSTART'] and sysinfo['SYSTEM']['REBOOTENABLED'] == 1:
            msg = ('{} is down. Restarting {} interface did not work. Rebooting.'.format(sysinfo['SYSTEM']['PINGHOST'],sysinfo['SYSTEM']['IFACE']))
            sendlog(gotthefunc, msg, 10)
            p = subprocess.Popen(rebootcmd, start_new_session=True)
        sleep(pinginterval)
        
if __name__ == '__main__':
   main()

