I wrote this little script to process hourly Shadow logs with ngrep through a date range specified as input from the analyst. ngrep is a must have utility if you have to do any content matching on raw packet logs or right off the wire. As the script stands right now it is great for applying your own regular expressions against the file while doing in-depth packet analysis.

See http://www.whitehats.ca/main/members/Malik/malik_shadow/malik_ngrep.html for a more verbose description and some samples of its usage.

See also http://www.whitehats.ca/main/members/Malik/malik_shadow/malik_ngrep_hour.html

This script is provided without any warranty or guarantee of fitness for use (especially on a production system). Use it at your own risk.

#!/bin/bash
#
# ngrep Shadow Log extraction script
# Written by Jamie French (j.french@whitehats.ca)
# Original file design by Guy Bruneau (seeker@whitehats.ca)
# Whitehats.ca 11 Jun 2002
#
# The purpose of this script is to extract and save logs of interest to a text file
# based upon regular expression content searches through packet payload from Shadow
# IDS.
#
# This script was mainly written to look at hourly Shadow logs on the analysis station
# (through ssh). The timezone should be changed to reflect the local timezone.
# Some variables should be assigned specific to your sites configurations.

clear

YEAR=`(set \`date\`; echo $6)`
HOUR=0
HOUR2=0
DAY=1
DAY2=1

# Change these variables to reflect where things are located
MYOUTPUT=/home/shadow/
MYNGREPPATH=/usr/local/bin/ngrep

echo ""
echo " ###########################################"
echo " ###########################################"
echo " ###                                     ###"
echo " ###        ngrep_hour script by         ###"
echo " ###             Whitehats.ca            ###"
echo " ###       version 1.2 17 Jun 2002       ###"
echo " ###              J.French               ###"
echo " ###                                     ###"
echo " ###########################################"
echo " ###########################################"
echo ""
sleep 1

clear

# Change these sensor names to reflect your sensor names
# designed specifically to handle multiple sensors on 1 analysis station
echo "Enter SHADOW sensor number process: "
echo ""
echo "    1 = sensor1"
echo "    2 = sensor2"
echo "    3 = sensor3"
echo "    4 = sensor4"
# echo "    5 = sensor5"
# echo "    6 = sensor6"
echo ""
read SENSOR
echo ""

if [ $SENSOR == 1 ]; then
 SENSOR=sensor1
elif [ $SENSOR == 2 ]; then
 SENSOR=sensor2
elif [ $SENSOR == 3 ]; then
 SENSOR=sensor3
elif [ $SENSOR == 4 ]; then
 SENSOR=sensor4
# elif [ $SENSOR == 3 ]; then
#  SENSOR=sensor5
# elif [ $SENSOR == 4 ]; then
#  SENSOR=sensor6
else 
 echo "Invalid Selection! Please re-run the script and make the correct selection."
 exit
fi

echo "Enter Month to process (Mmm): "
read MONTH 
echo ""

if [ $MONTH = Jan ]; then
 MONTH2=01
elif [ $MONTH = Feb ]; then
 MONTH2=02
elif [ $MONTH = Mar ]; then
 MONTH2=03
elif [ $MONTH = Apr ]; then
 MONTH2=04
elif [ $MONTH = May ]; then
 MONTH2=05
elif [ $MONTH = Jun ]; then
 MONTH2=06
elif [ $MONTH = Jul ]; then
 MONTH2=07
elif [ $MONTH = Aug ]; then
 MONTH2=08
elif [ $MONTH = Sep ]; then
 MONTH2=09
elif [ $MONTH = Oct ]; then
 MONTH2=10
elif [ $MONTH = Nov ]; then
 MONTH2=11
elif [ $MONTH = Dec ]; then
 MONTH2=12
else
 echo "You entered your month in the wrong format - please re-run the ngrep_daily script"
 echo ""
 exit
fi

echo "Enter beginning day to process (#):"
echo "      (first day in the range of days without leading zero's)"
read DAY
echo ""
echo "Enter ending day to process (#):"
echo "      (last day in the range of days without leading zero's)"
read DAY2
echo ""

DAY=$(echo $DAY | sed -e '/0[0-9]/s/\(0\)\([0-9]\)/\2/g')
DAY2=$(echo $DAY2 | sed -e '/0[0-9]/s/\(0\)\([0-9]\)/\2/g')

if [ $DAY -gt 31 ]; then
echo "The beginning day entered is greater than 31 - please re-run the ngrep_daily script choosing a valid day (range 1 - 31)"
exit
elif [ $DAY -lt 1 ]; then
echo "The beginning day entered is less than 1 - please re-rerun the ngrep_daily script choosing a valid day (range 1 -31)"
exit
elif [ $DAY2 -gt 31 ]; then
echo "The ending day entered is greater than 31 - please re-run the ngrep_daily script choosing a valid day (range 1 - 31)"
exit
elif [ $DAY2 -lt 1 ]; then
echo "The ending day entered is less than 1 - please re-rerun the ngrep_daily script choosing a valid day (range 1 -31)"
exit
elif [ $DAY2 -lt $DAY ]; then
echo "The ending day is before the beginning day - please re-run the ngrep_daily script and enter a beginning day that is before the ending day"
exit
else
echo "Enter filename to save data: "
echo "      (this will be written to $MYOUTPUT directory with 'results_' prepended to the file name)"
read FILE2
FILE=results_$FILE2.txt
echo "testing file is equal to $FILE"
echo ""

clear
echo "Detects usernames and passwords from FTP traffic = 'USER|PASS' tcp port 21"
echo "Detects all packets without the ASCII string 'hydr' in them = '^[hydr]'"
echo "Detects all packets with upper or lowercase letters followed by uppercase AR = '[A-Za-z]ER'"
echo "        this would find 'TOM:1px solid #104A7B; BORDER-RIGHT:1px solid'"
echo "                        ':#DBEAF5; BORDER-BOTTOM:1px solid #104A7B; BOR'"
echo "                        'r=\"#ffffff\">TERMS OF USE.. &nb'"
echo "Detects some variations of the string /bin/sh = '/*bin*/*sh'"
echo "        this would find '/bin/sh'"
echo "                        '/usr/bin/sh'"
echo "                        '/usr/bin/bash'"
echo "                        '/myhiddendirectory/binladen/shalom' - so be aware of false positives..."
echo "Detects some stuff than a space, newline, and a tab in a URL = '[^ \n\r]+'"
echo "Detects anything followed by whitespace, period, or comma = '(?=[\s\.,])'"
echo ""
echo "Enter the regular expression to search for..."
echo "     http://gnosis.cx/publish/programming/regular_expressions.html for some samples of regular expressions"
read MYSEARCH
echo ""
echo "Enter the desired tcpdump style filter (ex: tcp and dst port 80 ?): "
echo "     ***DO NOT*** use tcpdump options (like -Xvn) - just provide the filter"
read FILTER
echo ""
fi

echo "The start date is $MONTH $DAY, timezone GMT and the regex is \"$MYSEARCH\" and the filter is \"$FILTER\" on $SENSOR" > $MYOUTPUT$FILE

while [ $DAY -le $DAY2 ]; do  # Process days
              if [ $DAY -le 9 ]; then
                DAY3=0${DAY}
              else
                DAY3=$DAY
              fi
              HOUR=0
              echo ""
              echo "#+#+#+#+#" >> $MYOUTPUT$FILE
              echo "Processing logs from $MONTH $DAY" >> $MYOUTPUT$FILE
              echo "Processing $MONTH $DAY"
                while [ $HOUR -le 23 ]; do  # No process each hour in the day
                     while [ $HOUR -lt 10 ]; do  # We need to put the date into a format that matches Shadow's naming scheme with leading zero's
                       HOUR2=0${HOUR}
                       SHADOWPATH=/tcpdump_results/$SENSOR/$MONTH$DAY3/
                       FILENAME=tcp.$YEAR$MONTH2$DAY3$HOUR2.gz
                       echo "$FILENAME"
                       echo ""
                       echo "" >> $MYOUTPUT$FILE
                       echo "$MONTH $DAY3 $HOUR2" >> $MYOUTPUT$FILE
                       /bin/gunzip -c $SHADOWPATH$FILENAME | $MYNGREPPATH -qt -I - "$MYSEARCH" $FILTER >> $MYOUTPUT$FILE
                       HOUR=$(( $HOUR + 1 ))
                     done
                   HOUR2=$HOUR
                   SHADOWPATH=/tcpdump_results/$SENSOR/$MONTH$DAY3/
                   FILENAME=tcp.$YEAR$MONTH2$DAY3$HOUR2.gz
                   echo "$FILENAME"
                   echo ""
                   echo "" >> $MYOUTPUT$FILE
                   echo "$MONTH $DAY3 $HOUR2" >> $MYOUTPUT$FILE
                   echo "#+#+#+#+#" >> $MYOUTPUT$FILE
                   /bin/gunzip -c $SHADOWPATH$FILENAME | $MYNGREPPATH -qt -I - "$MYSEARCH" $FILTER >> $MYOUTPUT$FILE
                   HOUR=$(( $HOUR + 1 ))
                done
              HOUR2=$HOUR
              DAY=$(( $DAY + 1 ))
done


more $MYOUTPUT$FILE
exit