#!/bin/bash
# Copyright 2017 Lucid Networks sloeckle
# Package Version 1.11-2 - Errorcheck script Version 24

. /opt/nexp-tools/nexp-tools.lib

procname='errorcheck'
export DISPLAY=:0
export EMAIL=$HOSTNAME'@lucidnetworks.net'
repeatinterval=30

time=$(date +"%T")
rand=$(od -N 4 -t uL -An /dev/urandom | tr -d " ")
dir=/tmp/$rand
systeminfo
mkdir $dir &>/dev/null

source /etc/nexp-tools/nexp-tools.conf

iscontrolbox

isuptime

ismaint

isrunning

runningnow

exitprocess()
{
echo "bwgreyscreen=\"$bwgreyscreen\"" > /var/lib/nexp-tools/errorcheck.state
echo "bwgreyscreencheckcount=\"$bwgreyscreencheckcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "bwgreyscreenstarttime=\"$bwgreyscreenstarttime\"" >> /var/lib/nexp-tools/errorcheck.state
echo "bwgreyscreenrestartcount=\"$bwgreyscreenrestartcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "bwgreyscreenrefreshcount=\"$bwgreyscreenrefreshcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprwhitescreen=\"$sprwhitescreen\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprwhitescreencheckcount=\"$sprwhitescreencheckcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprwhitescreenstarttime=\"$sprwhitescreenstarttime\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprwhitescreenrestartcount=\"$sprwhitescreenrestartcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprwhitescreenrefreshcount=\"$sprwhitescreenrefreshcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprplaylist=\"$sprplaylist\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprplaylistcheckcount=\"$sprplaylistcheckcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprplayliststarttime=\"$sprplayliststarttime\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprplaylistrestartcount=\"$sprplaylistrestartcount\"" >> /var/lib/nexp-tools/errorcheck.state
rm -f /var/lib/nexp-tools/errorcheck.run
rm -rf $dir &>/dev/null
}

BWGREYSCREENTHRESHOLD=5
SPRWHITESCREENTHRESHOLD=5
timeregex='([0-9]{1,2}:[0-9]{2}(AM|PM))'

#Set default variables

bwgreyscreen=0
bwgreyscreencheckcount=0
bwgreyscreenrefreshcount=0
bwgreyscreenrestartcount=0
bwgreyscreenstarttime=""
sprplaylist=0
sprplaylistcheckcount=0
sprplaylistrestartcount=0
sprplayliststarttime=""
sprwhitescreen=0
sprwhitescreencheckcount=0
sprwhitescreenrefreshcount=0
sprwhitescreenrestartcount=0
sprwhitescreenstarttime=""

#See if state file exists then load it, overwriting default variables

if [ -a /var/lib/nexp-tools/errorcheck.state ]; then
  source /var/lib/nexp-tools/errorcheck.state
fi

#Write state file fixing missing file or data

echo "bwgreyscreen=\"$bwgreyscreen\"" > /var/lib/nexp-tools/errorcheck.state
echo "bwgreyscreencheckcount=\"$bwgreyscreencheckcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "bwgreyscreenstarttime=\"$bwgreyscreenstarttime\"" >> /var/lib/nexp-tools/errorcheck.state
echo "bwgreyscreenrestartcount=\"$bwgreyscreenrestartcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "bwgreyscreenrefreshcount=\"$bwgreyscreenrefreshcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprwhitescreen=\"$sprwhitescreen\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprwhitescreencheckcount=\"$sprwhitescreencheckcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprwhitescreenstarttime=\"$sprwhitescreenstarttime\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprwhitescreenrestartcount=\"$sprwhitescreenrestartcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprwhitescreenrefreshcount=\"$sprwhitescreenrefreshcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprplaylist=\"$sprplaylist\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprplaylistcheckcount=\"$sprplaylistcheckcount\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprplayliststarttime=\"$sprplayliststarttime\"" >> /var/lib/nexp-tools/errorcheck.state
echo "sprplaylistrestartcount=\"$sprplaylistrestartcount\"" >> /var/lib/nexp-tools/errorcheck.state

## Get master screenshot for analysis

scrot -q 100 $dir/$time-$HOSTNAME-analysis.png &>/dev/null

## Determine system resolution

currentresolution=$(xdpyinfo | grep dimensions: | awk '{print $2}')
if [[ $currentresolution == *"3840x2160"* ]]; then
  background="nexp3840x2160.jpg"
else
  background="nexp1920x1080.jpg"
fi

echo "Current resolution is $currentresolution"

## Check to see if browser is running

pslist=$(ps -ax)
if [[ $pslist != *"chrome"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="\n$systeminfo\nProcess list:\n\n$pslist\n\n$link"
      echo -e "$text" | mutt -s "$HOSTNAME Google Chrome appears to not be running. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  killproc chrome
  rotatechromelogs
  sleep 5
  rm -rf /home/nexp/.cache/google-chrome/*
  rm -rf /home/nexp/.config/google-chrome/Default/Web\ Data
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  sleep 5
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    pslist=$(ps -ax)
    text="\n$systeminfo\nProcess list:\n\n$pslist\n\n$link"
    echo -e "$text" | mutt -s "$HOSTNAME Google Chrome Started. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
  exitprocess
  exit 1
fi

if [[ $pslist == *"gnome-terminal"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="\n$systeminfo\nProcess list:\n\n$pslist\n\n$link"
      echo -e "$text" | mutt -s "$HOSTNAME Gnome Terminal appears to be running. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  killproc gnome-terminal
  sleep 5
  if [[ $APPNOTIFICATION == true ]]; then
    pslist=$(ps -ax)
    text="\n$systeminfo\nProcess list:\n\n$pslist\n\n$link"
    echo -e "$text" | mutt -s "$HOSTNAME Gnome Terminal killed. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
  exitprocess
  exit 1
fi

activewindow=$(xdotool getwindowfocus getwindowname)
if [[ $activewindow != *"Chrome"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="\n$systeminfo\nActive window: $activewindow\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Google Chrome appears to not be the window of focus. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "Shutting down Chrome"
  killproc chrome
  rotatechromelogs
  sleep 5
  echo "Clearing cache"
  rm -rf /home/nexp/.cache/google-chrome/*
  rm -rf /home/nexp/.config/google-chrome/Default/Web\ Data
  echo "Starting up Chrome"
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  if [[ $APPNOTIFICATION == true ]]; then
    activewindow=$(xdotool getwindowfocus getwindowname)
    text="\n$systeminfo\nActive windows: $activewindow\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Google Chrome Restarted. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
  exitprocess
  exit 1
fi

## Image analysis
identify -verbose $dir/$time-$HOSTNAME-analysis.png > $dir/$time-$HOSTNAME-analysis.txt
loadarray $dir/$time-$HOSTNAME-analysis.txt

## Brandwatch specific error checks
echo "Looking for Brandwatch Grey Screen"
echo "Looking for Brandwatch Grey Screen"
searcharray "2073600: ( 64, 64, 69) #404045 srgb(64,64,69)"
foundgrey=$foundinarray
searcharray "Colors: 1"
foundonecolor=$foundinarray
if [[ $foundgrey == 1 && $foundonecolor == 1 ]]; then
    echo "   Looks like we found one. Processing"
    if (( bwgreyscreen == 0 )); then
        echo "    Looks like a new issue"
        bwgreyscreen=1
        bwgreyscreenrestartcount=0
        bwgreyscreenrefreshcount=0
        bwgreyscreenstarttime=$(date)
        bwgreyscreencheckcount=1
        exitprocess
        exit 1
    else
        if (( (bwgreyscreencheckcount < BWGREYSCREENTHRESHOLD) )); then
            echo "    We haven't hit our threshold of $BWGREYSCREENTHRESHOLD yet. We are at $bwgreyscreencheckcount"
            echo "    Other stats:"
            echo "        bwgreyscreencheckcount: $bwgreyscreencheckcount"
            echo "        bwgreyscreenrefreshcount: $bwgreyscreenrefreshcount"
            echo "        bwgreyscreenrestartcount: $bwgreyscreenrestartcount"
            echo "        bwgreyscreenstarttime: $bwgreyscreenstarttime"
            ((bwgreyscreencheckcount++))
            exitprocess
            exit 1
        else
            echo "    Stats:"
            echo "        bwgreyscreencheckcount: $bwgreyscreencheckcount"
            echo "        bwgreyscreenrefreshcount: $bwgreyscreenrefreshcount"
            echo "        bwgreyscreenrestartcount: $bwgreyscreenrestartcount"
            echo "        bwgreyscreenstarttime: $bwgreyscreenstarttime"
            if (( (bwgreyscreenrefreshcount == 0) && (bwgreyscreenrestartcount == 0) )); then
                link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
                if [[ $APPNOTIFICATION == true ]]; then
                    subj="$HOSTNAME Brandwatch Grey Screen Detected. Beginning Corrective Action!"
                    body="This grey screen issue began at $bwgreyscreenstarttime. The system is beginning remediation efforts. Periodic notices will be automatically sent when the system deems that refreshing the browser is not working and Chrome needs to be restarted in an effort to resolve the issue. A final notice will be sent when the grey screen is gone.\n\n The alarm threshold is $BWGREYSCREENTHRESHOLD. The system checked $bwgreyscreencheckcount time(s) before alerting you.\n\nRefreshing Chrome now.\n\n$systeminfo\n\n$link"
                    echo -e $body | mutt -s "$subj" -- $APPMAILTO
                    echo "       Sending email"
                fi
                xdotool key F5
                sleep 1
                xdotool key space
                ((bwgreyscreenrefreshcount++))
                ((bwgreyscreencheckcount++))
            else
                echo "    Looks like a old issue"
                if (( (bwgreyscreenrefreshcount % 30 == 0) && (bwgreyscreenrefreshcount != 0) )); then
                    echo "      Restart Chrome"
                    killproc chrome
                    rotatechromelogs
                    sleep 5
                    /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
                    sleep 5
                    remediationtime=$(date +"%T")
                    scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
                    convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
                    ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
                    link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
                    if [[ $APPNOTIFICATION == true ]]; then
                        subj="$HOSTNAME Brandwatch Grey Screen Continues To Be Detected. Continuing Corrective Action!"
                        body="The system has attempted grey screen remediation without success.\n\nPlease notify Brandwatch as this issue does not appear to be resolving itself. The system will continue remediation efforts.\n\nRestarting Chrome again.\n\nSystem Stats:\n\n    grey Screen Alarm Threshold: $BWGREYSCREENTHRESHOLD\n\nError Stats:\n\n    Initial Reporting Time: $bwgreyscreenstarttime\n    Browser Restarts: $bwgreyscreenrestartcount\n    Browser Refreshes: $bwgreyscreenrefreshcount\n\n$systeminfo\n\n$link"
                        echo -e $body | mutt -s "$subj" -- $APPMAILTO
                        echo "       Sending email"
                   fi
                   ((bwgreyscreenrestartcount++))
                   ((bwgreyscreenrefreshcount++))
                   ((bwgreyscreencheckcount++))
               else
                   echo "      Refresh Chrome"
                   xdotool key F5
                   sleep 1
                   xdotool key space
                   ((bwgreyscreenrefreshcount++))
               fi
           fi
           bwgreyscreencheckcount=0
       fi
    fi
    exitprocess
    exit 1
else
    echo "  Don't see a grey screen"
    if (( (bwgreyscreenrefreshcount != 0) || (bwgreyscreenrestartcount != 0) )); then
        resolutiontime=$(date)
        echo "    Used to have a grey screen. It's been resolved"
        remediationtime=$(date +"%T")
        scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
        convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
        ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
        link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
        if [[ $APPNOTIFICATION == true ]]; then
            subj="$HOSTNAME Brandwatch Grey Screen Issue Resolved!"
            body="The grey screen condition appears to be resolved. See attached screenshot.\n\nError Stats:\n\n    Initial Reporting Time: $bwgreyscreenstarttime\n    Resolution Time: $resolutiontime\n    Browser Restarts: $bwgreyscreenrestartcount\n    Browser Refreshes: $bwgreyscreenrefreshcount\n\n$systeminfo\n\n$link"
            echo -e $body | mutt -s "$subj" -- $APPMAILTO
            echo "       Sending email"
        fi
    fi
    bwgreyscreen=0
    bwgreyscreenrefreshcount=0
    bwgreyscreenrestartcount=0
    bwgreyscreenstarttime=""
fi

## OCR
convert $dir/$time-$HOSTNAME-analysis.png -colorspace Gray $dir/$time-$HOSTNAME-analysis.tif &>/dev/null
ocr=$(tesseract $dir/$time-$HOSTNAME-analysis.tif stdout)

## Checking to see if desktop is showing

echo "Checking to see if desktop is showing"

if [[ $ocr == *"N-EXP"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Desktop appears to be showing. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found Desktop showing"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Google Chrome Restarted. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

## Chrome browser specific error checks

echo "Checking for Chrome browser specific errors"

if [[ ($ocr == *"Gnugle Chrnme"* || $ocr == *"Google Chrome"*) && $ocr =~ $timeregex ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Google Chrome appears to be hung on startup. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found Google Chrome Hung Problem"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Google Chrome Restarted. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

if [[ $ocr == *"Restore pages"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Google Chrome Restore Pages dialog visible. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found Restore Pages Dialog"
  xdotool key space
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Google Chrome Restore Pages dialog. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

if [[ $ocr == *"This webpage is not available"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME This webpage is not available Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found 'This webpage is not available' Error"
  xdotool key F5
  sleep 1
  xdotool key space
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME This webpage is not available Error. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

if [[ $ocr == *"Aw, Snap"* || $ocr == *"Aw,Snap"* || $ocr == *"Aw‘ Snap‘"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Google Aw, Snap! Crash. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found Aw, Snap! Error"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Google Aw, Snap! Crash. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

if [[ $ocr == *"Shockwave Flash has crash"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Google Shockware Flash Crash. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found Google Shockwave Flash Crash"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Google Shockware Flash Crash. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

if [[ $ocr == *"WebGL hit a snag"* || $ocr == *"wethhitasnag"* || $ocr == *"RELOAD Learn more"* || $ocr == *"webGL hita snag"* || $ocr == *"webGLhitasnag"* || $ocr == *"IGNORE RELOAD"* || $ocr == *"w:bc.L hit a snag"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Google WebGL Crash. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found Google Shockwave Flash Crash"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Google WebGL Crash. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

## Web server error checks

echo "Checking for web server errors"

if [[ $ocr == *"500 Internal Server Error"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME 500 Internal Server Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found 500 Internal Server Error"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
     text="$ocr\n\n\n$systeminfo\n\n$link"
     echo -e $text | mutt -s "$HOSTNAME 500 Internal Server Error. Action Completed!" -- $APPMAILTO
     echo "       Sending email"
  fi
fi

if [[ $ocr == *"Bad Reques"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Bad Request Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found Bad Request"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
     text="$ocr\n\n\n$systeminfo\n\n$link"
     echo -e $text | mutt -s "$HOSTNAME Bad Request Error. Action Completed!" -- $APPMAILTO
     echo "       Sending email"
  fi
fi

## Load balancer error checks

echo "Checking for load balancer errors"

if [[ $ocr == *"503 Service Unavailable"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME 503 Service Unavailable Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found 500 Service Unavailable"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME 503 Service Unavailable Error. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

if [[ $ocr == *"An error occurred."* && $ocr == *"nginx"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Unknown NGINX Server Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found NGINX error"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Unknown NGINX Server Error. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

## YouTube error checks

echo "Checking for YouTube errors"

if [[ $ocr == *"An error occurred"* && $ocr == *"Learn More"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Unknown YouTube Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found uselessly generic error"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Unknown YouTube Error. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

if [[ $ocr == *"An enm occurred"* && $ocr == *"Pieasevy agam lam"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Unknown YouTube Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found uselessly generic error"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Unknown YouTube Error. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

if [[ $ocr == *"An enov occurved"* && $ocr == *"Uy agam"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Unknown YouTube Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found uselessly generic error"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Unknown YouTube Error. Action Completed!" -- $APPMAILTO
    echo "       Sending email"
  fi
fi

## Sprinklr specific error checks

echo "Checking for Sprinklr specific errors"

echo "Looking for Sprinklr Playlist"
if [[ $ocr == *"Command Center > Settings"* || $ocr == *"Command Cemer > Settings"* || $ocr == *"Command Center > Settings Seavch"* || $ocr == *"My Dashbaams"* || $ocr == *"Command Cemer > Semngs >"* || $ocr == *"My Dashbuams"* || $ocr == *"Command Center > Senings"* || $ocr == *"Command Center > Semngs"* ]]; then
  echo "   Looks like we found one. Processing"
  if ((sprplaylist == 0 )); then
    echo "    Looks like a new issue"
    sprplaylist=1
    sprplaylistrestartcount=0
    sprplayliststarttime=$(date)
    sprplaylistcheckcount=1
    exitprocess
    exit 1
  else
      if (( sprplaylistrestartcount == 0 )); then
         link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
         if [[ $APPNOTIFICATION == true ]]; then
           subj="$HOSTNAME Sprinklr Playlist Detected. Beginning Corrective Action!"
           body="This playlist issue began at $sprplayliststarttime. The system is beginning remediation efforts. Periodic notices will be automatically sent when the system deems that refreshing the browser is not working and Chrome needs to be restarted in an effort to resolve the issue. A final notice will be sent when the playlist is gone.\n\n The alarm threshold is $BWGREYSCREENTHRESHOLD. The system checked $sprplaylistcheckcount time(s) before alerting you.\n\nRestarting Chrome now.\n\n$systeminfo\n\n$link"
           echo -e $body | mutt -s "$subj" -- $APPMAILTO
           echo "	Sending email"
         fi
         echo "      Restart Chrome"
         killproc chrome
         rotatechromelogs
         sleep 5
         /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
         ((sprplaylistrestartcount++))
         ((sprplaylistcheckcount++))
         exitprocess
         exit 1
      fi
      if (( (sprplaylistrestartcount % 5 == 0) )); then
          killproc chrome
          rotatechromelogs
          sleep 5
          /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
          if (( (sprplaylistrestartcount % 30 == 0) && (sprplaylistrestartcount != 0) )); then
              remediationtime=$(date +"%T")
              scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
              convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
              ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)              
              link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
              if [[ $APPNOTIFICATION == true ]]; then
                  subj="$HOSTNAME Sprinklr Playlist Continues To Be Detected. Continuing Corrective Action!"
                  body="The system has attempted playlist remediation without success.\n\nPlease notify Sprinklr as this issue does not appear to be resolving itself. The system will continue remediation efforts.\n\nRestarting Chrome again.\n\nSystem Stats:\n\n    Error Stats:\n\n    Initial Reporting Time: $sprplayliststarttime\n    Browser Restarts: $sprplaylistrestartcount\n    \n\n$systeminfo\n\n$link"
                  echo -e $body | mutt -s "$subj" -- $APPMAILTO
                  echo "       Sending email"
              fi
          fi
          ((sprplaylistrestartcount++))
          ((sprplaylistcheckcount++))
          exitprocess
          exit 1
      fi
  fi
else
  echo "  Don't see a playlist"
  if (( sprplaylistrestartcount != 0 )); then
    resolutiontime=$(date)
    echo "    Used to have a playlist. It's been resolved"
    remediationtime=$(date +"%T")
    scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
    convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
    ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
    link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
    if [[ $APPNOTIFICATION == true ]]; then
      subj="$HOSTNAME Sprinklr Playlist Issue Resolved!"
      body="The playlist condition appears to be resolved. See attached screenshot.\n\nError Stats:\n\n    Initial Reporting Time: $sprplayliststarttime\n    Resolution Time: $resolutiontime\n    Browser Restarts: $sprplaylistrestartcount\n    \n\n$systeminfo\n\n$link"
      echo -e $body | mutt -s "$subj" -- $APPMAILTO
      echo "       Sending email"
    fi
  fi
  sprplaylist=0
  sprplaylistrestartcount=0
  sprplayliststarttime=""
fi

if (( sprplaylist == 1 )); then
  exitprocess
  exit 1
fi

if [[ $ocr == *"create your ﬁrsl Dashboard"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
    text="$ocr\n\n\n$systeminfo\n\n$link"
    echo -e $text | mutt -s "$HOSTNAME Sprinklr Create Your First Dashboard Problem!" -- $APPMAILTO
    echo "       Sending email"
  fi
else 
  echo "  Found no dashboards error"
fi

if [[ $ocr == *"Your accoum has been locked"* ]]; then
  echo "  Found locked account error"
  if [[ $REPEATNOTIFICATION == true ]]; then
	  if [ -f /var/run/nexp-tools/sprinklraccountlock ]; then
		source /var/run/nexp-tools/sprinklraccountlock
		((errorcount++))
		echo "errorcount=$errorcount" > /var/run/nexp-tools/sprinklraccountlock
			if (( $errorcount >= $repeatinterval )); then
                                link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
                                if [[ $APPNOTIFICATION == true ]]; then
                                    text="$ocr\n\n\n$systeminfo\n\n$link"
                                    echo -e $text | mutt -s "$HOSTNAME Sprinklr Account Locked Error!"  -- $APPMAILTO
				    echo "errorcount=1" > /var/run/nexp-tools/sprinklraccountlock
                                fi
			else
				echo "errorcount=$errorcount" > /var/run/nexp-tools/sprinklraccountlock
			fi
	  else
		echo "errorcount=1" > /var/run/nexp-tools/sprinklraccountlock
                link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
		if [[ $APPNOTIFICATION == true ]]; then
                  text="$ocr\n\n\n$systeminfo\n\n$link"
		  echo -e $text | mutt -s "$HOSTNAME Sprinklr Account Locked Error!"  -- $APPMAILTOa
                  echo "       Sending email"
		fi
	  fi
  else
	if [ ! -f /var/run/nexp-tools/sprinklraccountlock ]; then
		echo "errorcount=1" > /var/run/nexp-tools/sprinklraccountlock
                link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
		if [[ $APPNOTIFICATION == true ]]; then
                  text="$ocr\n\n\n$systeminfo\n\n$link"
		  echo -e $text | mutt -s "$HOSTNAME Sprinklr Account Locked Error!"  -- $APPMAILTO
                  echo "       Sending email"
		fi
	else
		source /var/run/nexp-tools/sprinklraccountlock
		errorcount=$((errorcount+1))
		echo "errorcount=$errorcount" > /var/run/nexp-tools/sprinklraccountlock
	fi
  fi
else
  if [ -f /var/run/nexp-tools/sprinklraccountlock ]; then
	source /var/run/nexp-tools/sprinklraccountlock
	text+="\tError iteration = $errorcount\n"
        remediationtime=$(date +"%T")
        scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
        convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
        ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
        link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
	if [[ $APPNOTIFICATION == true ]]; then
            text="$ocr\n\n\n$systeminfo\n\n$link"
	    echo -e $text | mutt -s "$HOSTNAME Sprinklr Account Locked Resolved!"  -- $APPMAILTO
            echo "       Sending email"
	fi
	rm -f /var/run/nexp-tools/sprinklraccountlock &>/dev/null
  fi
fi

if [[ $ocr == *"Internal Sewer Error"* || $ocr == *"Internal Server Error"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Sprinklr Internal Server Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found Internal Sewer Error"
  killproc chrome
  rotatechromelogs
  sleep 5
  /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
  for i in {15..0}; do echo -ne "Waiting: $i "'\r'; sleep 1; done; echo
  remediationtime=$(date +"%T")
  scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
  convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
  ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
  link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
     text="$ocr\n\n\n$systeminfo\n\n$link"
     echo -e $text | mutt -s "$HOSTNAME Sprinklr Internal Server Error. Action Completed!" -- $APPMAILTO
     echo "       Sending email"
  fi
fi

echo "Looking for Sprinklr White Screen"
searcharray "2073600: (255,255,255) #FFFFFF gray(255)"
foundwhite=$foundinarray
searcharray "Colors: 1"
foundonecolor=$foundinarray
if [[ $foundwhite == 1 && $foundonecolor == 1 ]]; then
    echo "   Looks like we found one. Processing"
    if (( sprwhitescreen == 0 )); then
        echo "    Looks like a new issue"
        sprwhitescreen=1
        sprwhitescreenrestartcount=0
        sprwhitescreenrefreshcount=0
        sprwhitescreenstarttime=$(date)
        sprwhitescreencheckcount=1
        exitprocess
        exit 1
    else
#        if (( (sprwhitescreencheckcount < SPRWHITESCREENTHRESHOLD) && (sprwhitescreenrefreshcount != 0 || sprwhitescreenrestartcount != 0) )); then
        if (( (sprwhitescreencheckcount < SPRWHITESCREENTHRESHOLD) )); then
            echo "    We haven't hit our threshold of $SPRWHITESCREENTHRESHOLD yet. We are at $sprwhitescreencheckcount"
            echo "    Other stats:"
            echo "        sprwhitescreencheckcount: $sprwhitescreencheckcount"
            echo "        sprwhitescreenrefreshcount: $sprwhitescreenrefreshcount"
            echo "        sprwhitescreenrestartcount: $sprwhitescreenrestartcount"
            echo "        sprwhitescreenstarttime: $sprwhitescreenstarttime"
            ((sprwhitescreencheckcount++))
            exitprocess
            exit 1
        else
            echo "    Stats:"
            echo "        sprwhitescreencheckcount: $sprwhitescreencheckcount"
            echo "        sprwhitescreenrefreshcount: $sprwhitescreenrefreshcount"
            echo "        sprwhitescreenrestartcount: $sprwhitescreenrestartcount"
            echo "        sprwhitescreenstarttime: $sprwhitescreenstarttime"
            if (( (sprwhitescreenrefreshcount == 0) && (sprwhitescreenrestartcount == 0) )); then
                link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
                if [[ $APPNOTIFICATION == true ]]; then
                    subj="$HOSTNAME Sprinklr White Screen Detected. Beginning Corrective Action!"
                    body="This white screen issue began at $sprwhitescreenstarttime. The system is beginning remediation efforts. Periodic notices will be automatically sent when the system deems that refreshing the browser is not working and Chrome needs to be restarted in an effort to resolve the issue. A final notice will be sent when the white screen is gone.\n\n The alarm threshold is $SPRWHITESCREENTHRESHOLD. The system checked $sprwhitescreencheckcount time(s) before alerting you.\n\nRefreshing Chrome now.\n\n$systeminfo\n\n$link"
                    echo -e $body | mutt -s "$subj" -- $APPMAILTO
                    echo "       Sending email"
                fi
                xdotool key F5
                sleep 1
                xdotool key space
                ((sprwhitescreenrefreshcount++))
                ((sprwhitescreencheckcount++))
            else
                echo "    Looks like a old issue"
                if (( (sprwhitescreenrefreshcount % 30 == 0) && (sprwhitescreenrefreshcount != 0) )); then
                    echo "      Restart Chrome"
                    killproc chrome
                    rotatechromelogs
                    sleep 5
                    /usr/bin/google-chrome $CHROMEOPTS file:///usr/share/nexp-tools/$background &
                    sleep 5
                    remediationtime=$(date +"%T")
                    scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
                    convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
                    ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
                    link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
                    if [[ $APPNOTIFICATION == true ]]; then
                        subj="$HOSTNAME Sprinklr White Screen Continues To Be Detected. Continuing Corrective Action!"
                        body="The system has attempted white screen remediation without success.\n\nPlease notify Brandwatch as this issue does not appear to be resolving itself. The system will continue remediation efforts.\n\nRestarting Chrome again.\n\nSystem Stats:\n\n    white Screen Alarm Threshold: $SPRWHITESCREENTHRESHOLD\n\nError Stats:\n\n    Initial Reporting Time: $sprwhitescreenstarttime\n    Browser Restarts: $sprwhitescreenrestartcount\n    Browser Refreshes: $sprwhitescreenrefreshcount\n\n$systeminfo\n\n$link"
                        echo -e $body | mutt -s "$subj" -- $APPMAILTO
                        echo "       Sending email"
                   fi
                   ((sprwhitescreenrestartcount++))
                   ((sprwhitescreenrefreshcount++))
                   ((sprwhitescreencheckcount++))
               else
                   echo "      Refresh Chrome"
                   xdotool key F5
                   sleep 1
                   xdotool key space
                   ((sprwhitescreenrefreshcount++))
               fi
           fi
           sprwhitescreencheckcount=0
       fi
    fi
    exitprocess
    exit 1
else
    echo "  Don't see a white screen"
    if (( (sprwhitescreenrefreshcount != 0) || (sprwhitescreenrestartcount != 0) )); then
        resolutiontime=$(date)
        echo "    Used to have a white screen. It's been resolved"
        remediationtime=$(date +"%T")
        scrot -q 100 $dir/$remediationtime-$HOSTNAME-remediation.png &>/dev/null
        convert $dir/$remediationtime-$HOSTNAME-remediation.png -colorspace Gray $dir/$remediationtime-$HOSTNAME-remediation.tif
        ocr=$(tesseract $dir/$remediationtime-$HOSTNAME-remediation.tif stdout)
        link="$(curl -F "pic=@$dir/$remediationtime-$HOSTNAME-remediation.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
        if [[ $APPNOTIFICATION == true ]]; then
            subj="$HOSTNAME Sprinklr White Screen Issue Resolved!"
            body="The white screen condition appears to be resolved. See attached screenshot.\n\nError Stats:\n\n    Initial Reporting Time: $sprwhitescreenstarttime\n    Resolution Time: $resolutiontime\n    Browser Restarts: $sprwhitescreenrestartcount\n    Browser Refreshes: $sprwhitescreenrefreshcount\n\n$systeminfo\n\n$link"
            echo -e $body | mutt -s "$subj" -- $APPMAILTO
            echo "       Sending email"
        fi
    fi
    sprwhitescreen=0
    sprwhitescreenrefreshcount=0
    sprwhitescreenrestartcount=0
    sprwhitescreenstarttime=""
fi

# System error detection

if [[ $ocr == *"Could not switch the monitor configuration"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Monitor Configuration Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found Could not switch the monitor configuration Error"
  echo "Found Could not switch the monitor configuration Error" > /var/lib/nexp-tools/rebootreason
  exitprocess
  sudo reboot
fi

if [[ $ocr == *"Could not apply the stored conﬁguration for monitors"* ]]; then
  link="$(curl -F "pic=@$dir/$time-$HOSTNAME-analysis.png" http://172.25.0.30/remediation/upload.php?subdir=master)"
  if [[ $APPNOTIFICATION == true ]]; then
      text="$ocr\n\n\n$systeminfo\n\n$link"
      echo -e $text | mutt -s "$HOSTNAME Monitor Configuration Error. Taking Corrective Action!" -- $APPMAILTO
      echo "       Sending email"
  fi
  echo "  Found Could not apply the stored conﬁguration for monitors Error"
  echo "Found Could not apply the stored conﬁguration for monitors Error" > /var/lib/nexp-tools/rebootreason
  exitprocess
  sudo reboot
fi

exitprocess

