Question

I am trying to capture the output of a tcpdump/grep pipeline from Python. I am using Python 2.6 on Mac OS 10.6.7.

When I try it with dmesg/grep, the caller receives output from the subprocesses, as expected.

When I try it with tcpdump/grep, select never returns anything.

What am I doing wrong?

#! /usr/bin/python

def tcpdump():
    import subprocess, fcntl, os


    # This works

#   cmd1 = ['sudo', 'dmesg']
#   cmd2 = ['grep', '-E', '.*']


    # This doesn't work

    # sudo tcpdump -i en0 -n -s 0 -w - | grep -a -o -E "Host\: .*|GET \/.*"
    cmd1 = ['sudo', 'tcpdump', '-i', 'en0', '-n', '-s', '0', '-w', '-']
    cmd2 = ['grep', '-a', '-o', '-E', 'Host\: .*|GET \/.*']


    p1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE)
    p2 = subprocess.Popen(cmd2, stdout=subprocess.PIPE, stdin=p1.stdout)

    # set stdout file descriptor to nonblocking
    flags = \
    fcntl.fcntl(p2.stdout.fileno(), fcntl.F_GETFL)

    fcntl.fcntl(p2.stdout.fileno(), fcntl.F_SETFL, (flags | os.O_NDELAY | os.O_NONBLOCK))

    return p2


def poll_tcpdump(proc):
    import select

    txt = None

    while True:

        # wait 1/10 of a second and check whether proc has written anything to stdout
        readReady, _, _ = select.select([proc.stdout.fileno()], [], [], 0.1)

        if not len(readReady):
            break

        for line in iter(proc.stdout.readline, ""):

            if txt is None:
                txt = ''

            txt += line

        break

    return txt


proc = tcpdump()

while True:
    text = poll_tcpdump(proc)

    if text:
        print '>>>> ' + text

Answer 1

Try

cmd2 = ['grep', '--line-buffered', '-a', '-o', '-E', 'Host\: .*|GET \/.*']