Monitoring plugins & scripts

You will need custom scripts to monitor the LogFiles and the JournalLog.

Below are some of these scripts which I created.

================================================

JournalLog - including masklist&greplist (to have additional keywords via "grep -e" and/or "grep -v")

------------------------------------------------------------------------------------------------------------------------------------------------------

#
# variables
PATH=/sbin:/usr/sbin:/usr/ccs/bin:/bin:/usr/bin:/etc:/usr/contrib/bin:/users/root:/usr/lib:/usr/lib/acct:/usr/ucb:/usr/dt/bin:/usr/bin/X11

#if [ -z "$1" ]
#then
# echo "Usage: $0 [LOGFILE]"
# exit 111
#
#fi
#export LOGFILE="$1"

STATE_OK=0
STATE_WARNING=1
STATE_CRITICAL=2
STATE_UNKNOWN=3
STATE_DEPENDENT=4

#
# Define the functions
#

#
# maintenance mode
#
function maintenance {
if [ -f /NOMONITOR ]
then
echo "NOMONITOR"
exit ${STATE_OK}
fi
}

#
# journalchk -- Check the logfiles
#
#
function journalchk {
HOST=`hostname`
export CMD="sudo journalctl"
export LOGBASENAME="_journal_mon"
export MAX_CHR=900
ETCDIR=/usr/local/nagios/etc
TMPDIR=/usr/local/nagios/var/tmp
if [ ! -d ${TMPDIR} ]
then
mkdir -p $TMPDIR
fi

#journalctl |grep -i -e "-- Logs begin at"
#
#-- Logs begin at Fri 2020-11-20 18:10:00 CET, end at Sat 2020-11-28 08:59:54 CET. --
#
#journalctl --since "2015-06-26 23:15:00" --until "2015-06-26 23:20:00"

export OUTPUT=${TMPDIR}/OUTPUT${LOGBASENAME}.out
export RESULT=${TMPDIR}/RESULT${LOGBASENAME}.out

CHKSUDO=`sudo -n journalctl -n --lines=1 >/dev/null 2>&1 ; echo $?`
if [ "${CHKSUDO}" != "0" ]
then
echo "Sudo rights are NOT set for this user...exit"
exit ${STATE_UNKNOWN}
fi

POINT=".pointer.journal_mon_${HOST}.txt"
export LASTJOURNALCHK="2000-01-01 00:00:00"

if [ ! -f ${TMPDIR}/${POINT} ]
then
touch ${TMPDIR}/${POINT}
fi
#Get the values of the logfile from the last run
. ${TMPDIR}/${POINT}

if [ -z "${LASTJOURNALCHK}" ]
then
LASTJOURNALCHK="2000-01-01 00:00:00"
fi

ENDDATE=`${CMD} -n --lines=1 2>/dev/null |grep -i -e "-- Logs begin at" | cut -d " " -f 12,13`
if [ -z "${ENDDATE}" ]
then
echo "Error: journalctl command gave no ENDDATE result - ${ENDDATE}...exit"
exit ${STATE_UNKNOWN}
fi
${CMD} --since "${LASTJOURNALCHK}" --until "${ENDDATE}" > ${OUTPUT} 2>/dev/null
BEGINDATE=`tail -n +1 ${OUTPUT} 2>/dev/null |grep -i -e "-- Logs begin at" | cut -d " " -f 6,7`
NUMLINES=`tail -n +1 ${OUTPUT} 2>/dev/null |grep -v -e "-- Logs begin at" |wc -l | awk '{ print $1 }'`
echo "LASTJOURNALCHK=\"${ENDDATE}\"" > ${TMPDIR}/${POINT}

if [ "${NUMLINES}" -gt "0" ]
then
> ${RESULT}
if [ -s ${ETCDIR}/inputs/masklist.input${LOGBASENAME} ]
then
WNUMOUT=`tail -n +1 ${OUTPUT} |grep -v -e "-- Logs begin at" | grep -v -f ${ETCDIR}/inputs/masklist.input${LOGBASENAME} |grep -i -e "warn" |wc -l`
CNUMOUT=`tail -n +1 ${OUTPUT} |grep -v -e "-- Logs begin at" | grep -v -f ${ETCDIR}/inputs/masklist.input${LOGBASENAME} |grep -i -e "error" -e "fail" -e "emerg" -e "alert" -e "crit" |wc -l`
tail -n +1 ${OUTPUT} |grep -v -e "-- Logs begin at" | grep -v -f ${ETCDIR}/inputs/masklist.input${LOGBASENAME} |grep -i -e "error" -e "fail" -e "warn" -e "emerg" -e "alert" -e "crit" >> ${RESULT}
if [ -s ${ETCDIR}/inputs/greplist.input${LOGBASENAME} ]
then
CNUMGREPLIST=`tail -n +1 ${OUTPUT} |grep -v -e "-- Logs begin at" | grep -f ${ETCDIR}/inputs/greplist.input${LOGBASENAME} |wc -l`
CNEWNUMOUT=`expr ${CNUMOUT} + ${CNUMGREPLIST}`
CNUMOUT=${CNEWNUMOUT}
tail -n +1 ${OUTPUT} |grep -v -e "-- Logs begin at" | grep -f ${ETCDIR}/inputs/greplist.input${LOGBASENAME} >> ${RESULT}
fi
else
WNUMOUT=`tail -n +1 ${OUTPUT} |grep -v -e "-- Logs begin at" | grep -i -e "warn" |wc -l`
CNUMOUT=`tail -n +1 ${OUTPUT} |grep -v -e "-- Logs begin at" | grep -i -e "error" -e "fail" -e "emerg" -e "alert" -e "crit" |wc -l`
tail -n +1 ${OUTPUT} |grep -v -e "-- Logs begin at" | grep -i -e "error" -e "fail" -e "warn" -e "emerg" -e "alert" -e "crit" >> ${RESULT}
if [ -s ${ETCDIR}/inputs/greplist.input${LOGBASENAME} ]
then
CNUMGREPLIST=`tail -n +1 ${OUTPUT} |grep -v -e "-- Logs begin at" | grep -f ${ETCDIR}/inputs/greplist.input${LOGBASENAME} |wc -l`
CNEWNUMOUT=`expr ${CNUMOUT} + ${CNUMGREPLIST}`
CNUMOUT=${CNEWNUMOUT}
tail -n +1 ${OUTPUT} |grep -v -e "-- Logs begin at" | grep -f ${ETCDIR}/inputs/greplist.input${LOGBASENAME} >> ${RESULT}
fi
fi

NUMRES=`wc ${RESULT} |awk '{ print $1 }'`
if [ "${NUMRES}" -gt "1" ]
then
tail -n +1 ${RESULT} |tr '\n' ' ' >> ${RESULT}_1line
mv ${RESULT}_1line ${RESULT}
fi
tail -n +1 ${RESULT} |cut -c 1-${MAX_CHR} > ${RESULT}_cut 2>&1 ; mv ${RESULT}_cut ${RESULT}

if [ "${CNUMOUT}" -gt "0" ]
then
echo "${CNUMOUT} HITS - "
cat ${RESULT}
exit ${STATE_CRITICAL}
else
if [ "${WNUMOUT}" -gt "0" ]
then
echo "${WNUMOUT} HITS - "
cat ${RESULT}
exit ${STATE_WARNING}
else
if [ -s ${ETCDIR}/inputs/masklist.input${LOGBASENAME} ]
then
MASKIFUSED=1
else
MASKIFUSED=0
fi
if [ -s ${ETCDIR}/inputs/greplist.input${LOGBASENAME} ]
then
GREPIFUSED=1
else
GREPIFUSED=0
fi
echo "OK - journal_mon (M${MASKIFUSED}G${GREPIFUSED}): added new ${NUMLINES} lines with no hits"
exit ${STATE_OK}
fi
fi
else
if [ -s ${ETCDIR}/inputs/masklist.input${LOGBASENAME} ]
then
MASKIFUSED=1
else
MASKIFUSED=0
fi
if [ -s ${ETCDIR}/inputs/greplist.input${LOGBASENAME} ]
then
GREPIFUSED=1
else
GREPIFUSED=0
fi
echo "OK - journal_mon (M${MASKIFUSED}G${GREPIFUSED}): no new lines"
exit ${STATE_OK}
fi
}

maintenance
journalchk

exit 0

------------------------------------------------------------------------------------------------------------------------------------------------------

LogFiles - including masklist&greplist (to have additional keywords via "grep -e" and/or "grep -v")

And logfile rotation detection...!

------------------------------------------------------------------------------------------------------------------------------------------------------

#!/bin/bash
#
# Author: Ad Kuijpers
# Copyright: Copyright (c) Ad Kuijpers https://www.unix4life.com https://www.u4l.nl
#
# This file is under Revision Control System.
# So do NOT edit this file !
#
# RCS info:
# $Header: /usr/local/nagios/libexec/ak72/RCS/log_mon,v 1.3 2020/11/28 10:52:58 root Exp root $
# $Source: /usr/local/nagios/libexec/ak72/RCS/log_mon,v $
#
# File: log_mon
#
# Description: The log monitor script for using with Nagios monitoring
#
# Use: Usage by Nagios
#
# REQUIREMENTS!!
#
# zgrep/gzip package must be installed on the target system!
#
# /etc/sudoers must contain these lines:
# nagios ALL=(ALL) NOPASSWD: /usr/bin/tail -n +1 *
# nagios ALL=(ALL) NOPASSWD: /bin/zgrep *
# nagios ALL=(ALL) NOPASSWD: /bin/ls *
#
#
#
#
# Name Date Description
# Ad Kuijpers 020104 Initial version
# Ad Kuijpers 220916 Made script usable for generic usage with check on warning, error and failed
# Ad Kuijpers 121016 Added: unable to read logfile because of permissions
# Ad Kuijpers 171016 Added: logfile rotation detection
# Ad Kuijpers 181016 Added: inputs/greplist.input for additional keywords to grep
# Ad Kuijpers 240117 Removed the first test if file exists : -f $LOGFILE
# Ad Kuijpers 240117 Added: specific inputs/greplist.input${LOGBASENAME} for one logfilename
# Ad Kuijpers 170917 Make ${LOGBASENAME} the same as ${POINT} variable to make it's name unique for input files
# Ad Kuijpers 200917 Added: show filters in output of check if OK status
# Ad Kuijpers 220917 Tested and validated all possible situations..
# Ad Kuijpers 040119 Added sudo option for ls when file/dir is not readable
# Ad Kuijpers 050119 Changed find command to an ls type (sudo needed for find)
# Ad Kuijpers 070119 Changed the ls logfile-name commands filtering
# Ad Kuijpers 300919 Added to CRITICAL grep: -e "emerg" -e "alert" -e "crit"
# Ad Kuijpers 011019 Changed the log OK message with additional Masking used via M0/1 and Grep used via G0/1
# Ad Kuijpers 301020 Added detection of logfile rotation during run of this script AND logfile missing!! (via ${CAT} output to stderr/2)
# Ad Kuijpers 161120 ADD also zcat to /etc/sudoers!!!!!!
# Ad Kuijpers 171120 Limit the output of the check on the host where it is running (also done in ak72_nrpe)
# Ad Kuijpers 211120 In the past changed the find command in ls....it initiated an error => make a change AND test IT!!!
# Ad Kuijpers 211120 Added CHKPOINT in all the steps for better checking of error/problem situations
# Ad Kuijpers 221120 Tested and validated all possible situations..
# Ad Kuijpers 241120 Sometimes the cat command will give "write error broken pipe"...to solve this issue added "| ${TAIL} -n +1" to all cat's
# Ad Kuijpers 251120 Replaced most of the "cat" commands with "tail -n +1"
# Ad Kuijpers 261120 Detect logrotate better and limit the sudo command usage to "ls, tail and zgrep"
# Ad Kuijpers 281120 RCS version active
# Ad Kuijpers 281120 At last step: check the total lines to be less and detect logrotation
#

------------------------------------------------------------------------------------------------------------------------------------------------------