Re: Removing HTTP headers from tcpdump logs

[Bill McCarty <bmccarty () apu edu>]

Hey Chris,

I run tcpflow to obtain files containing application-layer data. Then, I 
run a homebrew Python script that can strip HTTP headers, gunzip, and untar 
captured files. The script is a work in progress rather than a product. 
But, I've included it below, so fellow Python programmers can tame it and 
use it for their own purposes. The script leaves the original files intact 
and so is fairly safe to run. But, please use at your own risk.

Cheers,

#!/usr/bin/python

import os
import string
import sys

MAXSIZE = 10000000

#print sys.argv[1:]

for file in sys.argv[1:]:

   if not os.path.isfile(file): continue
   #print file

   size = os.path.getsize(file)
   if size > MAXSIZE:
       print "File %s is too large to process (%d bytes)" % (file, size)
       continue

   handle = open(file, "rb")
   data = handle.read()
   handle.close()

   if data[0:4] != "HTTP": continue
   print "Processing file: ", file,

   offset = string.find(data, "\r\n\r\n")
   #print "offset=", offset

   data = data [offset+4:]

   newfile = file + ".data"
   handle = open(newfile, "wb")
   handle.write(data)
   handle.close()

   cmd = "file %s" % (newfile)

   pipe = os.popen(cmd)
   lines = pipe.read()
   pipe.close()

   #print lines

   if string.find(lines, "gzip") != -1:
       #print "Uncompressing", newfile
       print "uncompressing ",

       os.rename(newfile, newfile + ".gz")

       gzipfile = newfile + ".gz"

       cmd = "gunzip %s" % (gzipfile)
       error = os.system(cmd)
       if error != 0:
           print "Error %d executing command %s" % (error, cmd)
           continue

   cmd = "file %s" % (newfile)

   pipe = os.popen(cmd)
   lines = pipe.read()
   pipe.close()

   #print lines

   if string.find(lines, "GNU tar") != -1:
       os.rename(newfile, newfile + ".tar")
       print "renamng as .tar ",
   elif string.find(lines, "ELF") != -1:
       os.rename(newfile, newfile + ".elf")
       print "renaming as .elf ",

   print "done."

sys.exit(0)


---------------------------------------------------
Bill McCarty