Question

Yesterday, I asked if my asynchronous use of libpcap was making me lose packets. Today, I looked further and it seems that the problem is not on the asynchronous use of libpcap, but on the use of pcap_next_ex. Occasionally (10 runs out of a 1000), pcap_next_ex will return before the pcap handle timeout expired, telling the program that there were no packets to be read (even though they are there).

The following proof-of-concept reproduces the problem. It depends on libpcap, pthread, boost and libcrafter (a nice C++ packet crafting library). Basically, it sends a set of TCP-SYN packets to a destination and tries to obtain the responses using libpcap. A thread calling pcap_loop is run in parallel – while the main program misses some responses (as described above), the thread captures all the packets.

#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/format.hpp>
#include <boost/make_shared.hpp>
#include <boost/shared_ptr.hpp>

#include <crafter.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include <pcap.h>
#include <sys/types.h>
#include <sys/socket.h>

using namespace boost;
using namespace Crafter;
using namespace std;

int captureThreadCount = 0;

typedef vector<pair<shared_ptr<Packet>, posix_time::time_duration> > PacketTimestamp;
PacketTimestamp capturedThreadPackets;

void captureThreadCallback(u_char* user, const struct pcap_pkthdr* h, const u_char* bytes)
{
    shared_ptr<Packet> packet = make_shared<Packet>();
    packet->PacketFromIP(bytes + 16, h->caplen - 16);
    posix_time::time_duration timestamp = posix_time::seconds(h->ts.tv_sec) +
        posix_time::microseconds(h->ts.tv_usec);
    capturedThreadPackets.push_back(make_pair(packet, timestamp));
    ++captureThreadCount;
}

void* captureThread(void* arg)
{
    pcap_t* pcap = (pcap_t*) arg;
    pcap_loop(pcap, -1, captureThreadCallback, NULL);
}

int main(int argc, char* argv[])
{
    if (argc != 5) {
        cout << "Usage: " << argv[0] << " <source ip> <destination ip> <port> <# tries>" << endl;
        exit(1);
    }

    InitCrafter();

    // Parameters.
    string sourceIp = argv[1];
    string destinationIp = argv[2];
    int port = atoi(argv[3]);
    int nTries = atoi(argv[4]);

    char errorBuffer[1024];
    // Socket for sending,
    int sd = socket(PF_INET, SOCK_RAW, IPPROTO_RAW);
    // And sockaddr_in to send to.
    struct sockaddr_in sin;
    sin.sin_family = AF_INET;
    sin.sin_port = htons(port);
    sin.sin_addr.s_addr = inet_addr(destinationIp.c_str());    
    // One pcap for the main thread (calling pcap_next),
    pcap_t* pcapForNext = pcap_open_live(NULL, BUFSIZ, false, 1000, errorBuffer);
    // Another pcap for a capture thread (calling pcap_loop),
    pcap_t* pcapCapture = pcap_open_live(NULL, BUFSIZ, false, 1000, errorBuffer);
    // Both filtered for SYN+ACK or RST+ACK from destination:port to source.
    string filterExpression = (boost::format("ip src %s and dst %s and src port %d and ((tcp[tcpflags] & (tcp-syn|tcp-ack) != 0) or (tcp[tcpflags] & (tcp-rst|tcp-ack) != 0))") % destinationIp % sourceIp % port).str();
    struct bpf_program filter;
    pcap_compile(pcapForNext, &filter, filterExpression.c_str(), false, 0);
    pcap_setfilter(pcapForNext, &filter);
    pcap_setfilter(pcapCapture, &filter);
    pcap_freecode(&filter);
    // Don't forget the capture thread!
    pthread_t thread;
    pthread_create(&thread, NULL, captureThread, pcapCapture);

    // Some statistics.
    int packetsSent = 0;
    int packetsReceived = 0;
    int packetsTimeout = 0;
    int packetsFailed = 0;

    // Let's probe.
    for (int i = 0; i < nTries; ++i) {
        // Create packet,
        IP ipHeader;
        ipHeader.SetSourceIP(sourceIp);
        ipHeader.SetDestinationIP(destinationIp);
        TCP tcpHeader;
        tcpHeader.SetSrcPort(12345 + i);
        tcpHeader.SetDstPort(port);
        tcpHeader.SetFlags(TCP::SYN);
        shared_ptr<Packet> packet = make_shared<Packet>(ipHeader / tcpHeader);
        // Check the time,
        struct timeval tv;
        gettimeofday(&tv, NULL);
        posix_time::time_duration sentTime =
            posix_time::seconds(tv.tv_sec) + posix_time::microseconds(tv.tv_usec);
        // And send it.
        if (packet->SocketSend(sd) > 0) {
            cerr << "Sent packet " << i << " at " << sentTime << "." << endl;
            ++packetsSent;
        }
        else {
            cerr << "Sending packet " << i << " failed." << endl;
            continue;
        }

        struct pcap_pkthdr* pktHeader;
        const u_char* pktData;
        int r;
        // Wait for the response.
        if ((r = pcap_next_ex(pcapForNext, &pktHeader, &pktData)) > 0) {
            posix_time::time_duration timestamp =
                posix_time::seconds(pktHeader->ts.tv_sec) +
                posix_time::microseconds(pktHeader->ts.tv_usec);
            cerr << "Response " << i << " received at " << timestamp << "." << endl;
            ++packetsReceived;
        }
        else if (r == 0) {
            cerr << "Timeout receiving response for " << i << "." << endl;
            ++packetsTimeout;
        }
        else {
            cerr << "Failed receiving response for " << i << "." << endl;
            ++packetsFailed;
        }
    }

    // Wait (to ensure "fast packets" are captured by the capture thread),
    usleep(500000); // 500 ms.

    for (PacketTimestamp::iterator i = capturedThreadPackets.begin();
         i != capturedThreadPackets.end(); ++i) {
        TCP* tcpLayer = GetTCP(*i->first);
        cout << "Captured packet " << (tcpLayer->GetDstPort() - 12345) << " at " << i->second << "." << endl;
    }

    cout << "SNT=" << packetsSent <<
        ", RCV=" << packetsReceived <<
        ", TIM=" << packetsTimeout <<
        ", FLD=" << packetsFailed <<
        ", CAP=" << captureThreadCount << "." << endl;

    CleanCrafter();
}

Answer 1

This seems to be a problem with libpcap using memory mapping in Linux (it happens even on the latest libpcap, 1.3.0). If libpcap is compiled without memory mapping support (as described in a message from the tcpdump mailing list archive), the problem goes away.