Question

To learn more about the epoll notification facility in linux, I’ve been working on an http server. The structure of the server is to basically have an array of event structs which represent requests, which is then iterated over in some nested for and while loops.

When I first wrote this primitive event loop, it worked fairly well. However, when I refactored the code, the loop become much less reliable. In particular, I began to get a bunch (about 75% of requests) of epoll_ctl errors, with the errno being set to BADFD. epoll_ctl apparently believes that my socket file descriptor is not really a socket file descriptor.

I’m really confused why there should be such a drop in performance, however, since the refactor consisted only of a) cleaning up the parsing of requests and b) moving the main function up to the beginning of the program. I checked out a previous commit of the server, and compared the main function to that the new version–it is the same.

Does anyone have some insight into what may be happening here? It’d be appreciated.

Here is the code of the main function for reference:

int main(int argc, char *argv[]) {
    c("in main");
    char *progname=argv[0];
    int sockfd, newsockfd, portno, clilen, n, pid, epollfd;
    struct sockaddr_in serv_addr, cli_addr;

    initFt();
    if (argc < 2 ) {
        fprintf(stderr, "\nERROR: No Port Provided\n");
        exit(EXIT_FAILURE);
    }
    if (!createSocket(&sockfd)) {
        fprintf(stderr, "%s ERROR, COULD NOT CREATE SERVER SOCKET\n", progname);
        exit(EXIT_FAILURE);
    }

    serv_addr.sin_family = AF_INET;
    serv_addr.sin_port = htons(atoi(argv[1]));
    serv_addr.sin_addr.s_addr = INADDR_ANY;
    Bind(&sockfd, &serv_addr);
    listen(sockfd, 5);

    struct epoll_event *events = calloc(SOMAXCONN,  sizeof(struct epoll_event));
    struct epoll_event event;
    makeSocketNB(&sockfd);
    event.events = EPOLLIN | EPOLLET;
    event.data.fd = sockfd;

    if ((epollfd = createpoll())<0) {
        fprintf(stderr, "epoll create error\n");
        exit(EXIT_FAILURE);
    }
    if (epoll_ctl(epollfd, EPOLL_CTL_ADD, sockfd, &event) == -1) {
        fprintf(stderr, "error with epoll_ctl on sockfd");
        exit(EXIT_FAILURE);
    }

    while (1) {
        int n, e; //e for events
        e = epoll_wait(epollfd, events, SOMAXCONN, -1);

        for (n=0; n<e; n++) {
            if ((events[n].events & EPOLLERR) || (events[n].events & EPOLLHUP) || !(events[n].events & EPOLLIN)) {
                fprintf (stderr, "epoll error\n");
                close (events[n].data.fd);
                continue;
            }
            else if (events[n].data.fd == sockfd) {
                while (1) {
                    newsockfd=accept(sockfd, (struct sockaddr *) &cli_addr, &clilen);
                    if (newsockfd>0) {
                        makeSocketNB(&newsockfd);
                        fprintf(stderr, "Accepted a new connection on fd %d, made nonblocking\n", newsockfd);
                    }
                    else if (errno == EAGAIN || errno == EWOULDBLOCK) {
                        fprintf (stderr, "we have accepted all the clients on this event\n");
                        break;
                    }
                    event.data.fd = newsockfd;
                    event.events = EPOLLIN | EPOLLET;
                    if (epoll_ctl(epollfd, EPOLL_CTL_ADD, newsockfd, &event)<0) {
                        fprintf(stderr, "epoll_ctl error\n");
                        if (errno == EEXIST) {
                            fprintf(stderr, "fd already registered\n");
                        }
                        else if (errno == EBADF) {
                            fprintf(stderr, "fd bad\n%d\n", newsockfd);
                            break;
                        }
                        else if (errno == ENOMEM) {
                            fprintf(stderr, "no memory\n");
                        }
                        else if (errno == ENOSPC) {
                            fprintf(stderr, "enospc\n");
                        }
                        exit(EXIT_FAILURE);
                    }
                    continue;
                }
            }
            else { // there is stuff for us to read
                handleResponse(&events[n].data.fd);
                fprintf(stderr, "we are about to close file descriptor %d\n", events[n].data.fd);
                close (events[n].data.fd);
                fprintf(stderr, "connection closed\n");
            }
        }
    }

    exit(EXIT_SUCCESS);
}

Answer 1

Your accept code doesn’t really handle errors: it only handles the “success” case and the wouldblock case.

if (newsockfd>0) {
    /* success */
} else if (errno == EAGAIN || errno == EWOULDBLOCK) {
    /* wouldblock */
}

/* Could be error but you call `epoll_ctl` anyway. */