I co-wrote this little script with Seeker to process hourly Shadow logs with ngrep. ngrep is a must have utility if you have to do any content matching on raw packet logs or right off the wire. If you had the time or inclination, it probably wouldn't take much to change the year from a static value to the current year - actually I've done this with the ngrep_daily script.. If your really dedicated, you might take the Snort ruleset and strip out the rule name and content strings, then cycle the Shadow raw logs through in a for loop, writing output to files with the rule name. Sounds a lot like what Snort already does (so why replicate it manually - just install Snort if your going to go this far. As the script stands right now it is great for applying your own regular expressions against the file while doing in-depth packet analysis.

See http://www.whitehats.ca/main/members/Malik/malik_shadow/malik_ngrep.html for a more verbose description and some samples of its usage.

See also http://www.whitehats.ca/main/members/Malik/malik_shadow/malik_ngrep_daily.html

This script is provided without any warranty or guarantee of fitness for use (especially on a production system). Use it at your own risk.

#!/bin/sh
#
# Shadow Log extraction script
# Written by Jamie French (j.french@whitehats.ca)
# Original file design by Guy Bruneau (seeker@whitehats.ca)
# Whitehats.ca 12 Jun 2002
#
# The purpose of this script is to extract and save to a text file log of interest
# from Shadow sensors. The year in this filter must be changed on the 1st of Jan of
# every year.
#
# This script was mainly written to look at hourly Shadow logs on the analysis station
# through ssh and if necessary pull them back through scp to another computer. The
# timezone should also be changed to reflect the local timezone.
#
# The ngrep binary should be placed in the analyst users path and chgrp'ed to their
# group, somewhere like /usr/local/bin and chmod'ed to 0750 . Seeing as ngrep can put
# the NIC into promiscuous mode, it is important to secure this utility properly.
#

clear

echo ""
echo " ###########################################"
echo " ###########################################"
echo " ###                                     ###"
echo " ###        ngrep_hour script by         ###"
echo " ###             Whitehats.ca            ###"
echo " ###       version 1.0 12 Jun 2002       ###"
echo " ###        J.French & Guy Bruneau       ###"
echo " ###                                     ###"
echo " ###########################################"
echo " ###########################################"
echo ""
sleep 1

clear

echo "Enter SHADOW sensor number process: "
echo ""
#
# You will need to modify the name of each sensor to match your sensor names
#
echo "    1 = sensor1"
echo "    2 = sensor2"
echo "    3 = sensor3"
echo "    4 = sensor4"
echo ""
read SENSOR
echo ""

if [ $SENSOR == 1 ]; then
 SENSOR=sensor1
elif [ $SENSOR == 2 ]; then
 SENSOR=sensor2
elif [ $SENSOR == 3 ]; then
 SENSOR=sensor3
elif [ $SENSOR == 4 ]; then
 SENSOR=sensor4
else
 echo "Invalid Selection! Please re-run the script and make the correct selection."
 exit
fi

echo "Enter date to process (MmmDD): "
read DATE
echo ""
echo "Enter filename to save data: "
echo "      (this will be in the /home/shadow directory with 'result_' prepended to the file name)"
read FILE
echo ""
echo "Enter hour to process: "
read HOUR
echo ""

clear
echo "Detects usernames and passwords from FTP traffic = 'USER|PASS' tcp port 21"
echo "Detects all packets without the ASCII string 'hydr' in them = '^[hydr]'"
echo "Detects all packets with upper or lowercase letters followed by uppercase AR = '[A-Za-z]ER'"
echo "        this would find 'TOM:1px solid #104A7B; BORDER-RIGHT:1px solid'"
echo "                        ':#DBEAF5; BORDER-BOTTOM:1px solid #104A7B; BOR'"
echo "                        'r=\"#ffffff\">TERMS OF USE.. &nb'"
echo "Detects some variations of the string /bin/sh = '/*bin*/*sh'"
echo "        this would find '/bin/sh'"
echo "                        '/usr/bin/sh'"
echo "                        '/usr/bin/bash'"
echo "                        '/myhiddendirectory/binladen/shalom' - so be aware of false positives..."
echo "Detects some stuff than a space, newline, and a tab in a URL = '[^ \n\r]+'"
echo "Detects anything followed by whitespace, period, or comma = '(?=[\s\.,])'"
echo ""
echo "Enter the regular expression to search for..."
echo "     http://gnosis.cx/publish/programming/regular_expressions.html for some samples of regular expressions"
read MYSEARCH
echo ""
echo "Enter the desired tcpdump style filter (ex: tcp and dst port 80 ?): "
echo "     ***DO NOT*** use tcpdump options (like -Xvn) - just provide the filter"
read FILTER
echo ""


echo "The date is $DATE (MmmDD) timezone GMT and the regex is $MYSEARCH and the filter is $FILTER on $SENSOR" > /home/shadow/result_$FILE.txt
# /usr/local/bin/ngrep -q -t -A 5 -I /tmp/tcp.2002061112 "$MYSEARCH" $FILTER > /home/shadow/result_$FILE.txt
# /bin/gunzip -c /tcpdump_results/$SENSOR/$DATE/tcp.2002*$HOUR.gz | /usr/local/bin/ngrep -q -t -A 5 -I - "$MYSEARCH" $FILTER >> /home/shadow/result_$FILE.txt

/bin/gunzip -c /tcpdump_results/$SENSOR/$DATE/tcp.2002*$HOUR.gz | /usr/local/bin/ngrep -qt -I - "$MYSEARCH" $FILTER >> /home/shadow/result_$FILE.txt


echo "done"
sleep 1
more /home/shadow/result_$FILE.txt
exit