Question

I’m implementing a simple multi-threaded web server for a school assignment and have run into some synchronization issues with the connection file descriptors being used for each connection. My initial problem was that one thread would sometimes close the file descriptor (conn_fd) for a file descriptor also being used in another thread. This would cause bad file descriptor errors when the other thread would try to send() or recv().

My workaround for this was to store whether each file descriptor up to 1000(arbitrary number and error prone I know) was currently open. If the file descriptor returned by accept() is already open my progam calls fcntl(conn_fd, F_DUPFD, 0); to create a duplicate file descriptor so that one thread doesn’t inadvertently close the connection that another thread will need to use. My program seems to be working better than it was before I started keeping track of open file descriptors but I still have some synchronization problem that I can’t figure out how to resolve. The conn_fd in the start routing for each thread, process_connection_request() seems to be getting clobbered.

I’ve tried using Helgrind while running Siege against my server to isolate the problem. Unfortunately my code never crashes while being run under Helgrind. It does indicate a potential race condition with conn_fd but I thought wrapping the mutex that I have around it in both main() and process_connection_request() would resolve that problem. I have never developed any multi-thread or socket programs before and I suspect that there’s something simple I’m missing. Any insights and suggestions as to how I can resolve my problem with bad file descriptors when sending and receiving are greatly appreciated.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <fcntl.h>
#include <time.h>
#include <signal.h>
#include <pthread.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>

#define SERVER_PORT 50040
#define MAX_LISTEN_BACKLOG 1024
#define MAX_FILENAME_LENGTH 255
#define REQUEST_BUFF_SIZE 8192
#define THREAD_POOL_SIZE 16

// function prototypes
int int_len(int i);
void *process_connection_request(void *conn_fd);
void sig_handler(int sig);

pthread_t thread;
pthread_attr_t thread_attr;
pthread_mutex_t conn_fd_mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t stats_mutex = PTHREAD_MUTEX_INITIALIZER;

int sock_fd;
int conn_fds_open[1000];

int main(void)
{
    struct sockaddr_in server_addr;
    struct sockaddr_in client_addr;
    int conn_fd;
    int client_len = sizeof(client_addr);

    if((sock_fd = socket(PF_INET, SOCK_STREAM, 0)) < 0)
    {
        perror("socket() Failed");
        exit(EXIT_FAILURE);     
    }

    // set socket options so that we can reuse the socket
    const int sock_opt_val = 1;
    const socklen_t sock_opt_len = sizeof(sock_opt_val);
    setsockopt(sock_fd, SOL_SOCKET, SO_REUSEADDR, (void *) &sock_opt_val, sock_opt_len);

    memset(&conn_fds_open, 0, 1000 * sizeof(int));
    memset(&server_addr.sin_zero, 0, sizeof(server_addr.sin_zero));
    server_addr.sin_family = AF_INET;
    server_addr.sin_port = htons(SERVER_PORT);
    server_addr.sin_addr.s_addr = INADDR_ANY;

    if(bind(sock_fd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0)
    {
        perror("bind() Failed");
        close(sock_fd);
        exit(EXIT_FAILURE);         
    }

    if(listen(sock_fd, MAX_LISTEN_BACKLOG) < 0)
    {
        perror("listen() Failed");
        close(sock_fd);
        exit(EXIT_FAILURE);     
    }

    if(pthread_attr_init(&thread_attr) != 0)
    {
        perror("pthread_attr_init Failed");
        close(sock_fd);
        exit(EXIT_FAILURE);         
    }

    if(pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_DETACHED) != 0)
    {
        perror("pthread_attr_setdetachstate Failed");
        close(sock_fd);
        exit(EXIT_FAILURE);     
    }

    signal(SIGINT, sig_handler);
    printf("sock_fd %d\n", sock_fd);

    while(1)
    {
        if((conn_fd = accept(sock_fd, (struct sockaddr *)&client_addr, &client_len)) < 0)
        {
            perror("accept() Failed");
            close(sock_fd);
            exit(EXIT_FAILURE);     
        }

        pthread_mutex_lock(&conn_fd_mutex);
        if(conn_fds_open[conn_fd] == 1)
        {
            conn_fd = fcntl(conn_fd, F_DUPFD, 0);
        }

        conn_fds_open[conn_fd] = 1;
        printf("main fd: %d\n", conn_fd);
        pthread_mutex_unlock(&conn_fd_mutex);
        pthread_create(&thread, &thread_attr, process_connection_request, (void *)&conn_fd);
    }
}

int int_len(int i)
{
    return (i == 0) ? 1 : floor(log10(abs(i))) + 1;
}

void *process_connection_request(void *conn_fd_ptr)
{
    FILE *requested_file = NULL;
    FILE *stats_file = NULL;
    char *request_buff;
    char *response_buff;
    char *file_buff;
    char *stats_buff;
    char *filename_start;
    char *filename_stop;
    char requested_filename[MAX_FILENAME_LENGTH];
    int conn_fd = *(int *)conn_fd_ptr;
    int requested_file_size;
    int response_buff_size;
    int stats_buff_size;
    int amt_sent = 0;
    int response_code;
    time_t now;
    char time_buff[30];
    time(&now);
    strftime(time_buff, 30, "%a, %d %b %Y %X GMT", gmtime(&now));

    printf("thread fd: %d\n", conn_fd);

    if((request_buff = calloc(REQUEST_BUFF_SIZE, sizeof(char))) == NULL)
    {
        perror("Calloc Failed");
        close(conn_fd);
        close(sock_fd);
        exit(EXIT_FAILURE);     
    }

    if(recv(conn_fd, (void *)request_buff, REQUEST_BUFF_SIZE, 0) < 0)
    {
        perror("recv() Failed");
        close(conn_fd);
        close(sock_fd);
        exit(EXIT_FAILURE); 
    }

    // extract the filename from the request header
    filename_start = &request_buff[5];
    filename_stop = strstr(request_buff, " HTTP");

    if((strncmp(request_buff, "GET /", 5) != 0) || (filename_stop == NULL))
    {
        perror("Invalid Request");
        close(conn_fd);
        close(sock_fd);
        exit(EXIT_FAILURE);         
    }

    strncpy(requested_filename, &request_buff[5], filename_stop - filename_start);
    free(request_buff);
    requested_filename[filename_stop - filename_start] = '\0';

    if((requested_file = fopen(requested_filename, "r")) != NULL)
    {
        response_code = 200;
        fseek(requested_file, 0, SEEK_END);
        requested_file_size = ftell(requested_file);
        fseek(requested_file, 0, SEEK_SET);
        file_buff = calloc(requested_file_size + 1, sizeof(char));
        response_buff = calloc((83 + strlen(time_buff) + int_len(requested_file_size) + requested_file_size), sizeof(char));

        if(file_buff == NULL || response_buff == NULL)
        {
            perror("Calloc Failed");
            close(conn_fd);
            close(sock_fd);
            exit(EXIT_FAILURE);             
        }

        fread(file_buff, 1, requested_file_size, requested_file);
        response_buff_size = sprintf(response_buff, "HTTP/1.1 200 OK\nDATE: %s\nContent-Length: %d\nConnection: close\nContent-Type: text/html\n\n%s", time_buff, requested_file_size, file_buff);
        free(file_buff);
        fclose(requested_file);     
    }
    else
    {
        response_code = 404;
        response_buff = malloc(25 * sizeof(char));
        strcpy(response_buff, "HTTP/1.1 404 Not Found\n\n");
        response_buff_size = 25;
    }

    while(amt_sent < response_buff_size)
    {
        int ret = send(conn_fd, response_buff + amt_sent, response_buff_size - amt_sent, 0);
        if (ret < 0)
        {
            perror("send() Failed.");
            close(conn_fd);
            close(sock_fd);
            exit(EXIT_FAILURE); 
        }
        amt_sent += ret;        
    }

    free(response_buff);
    pthread_mutex_lock(&conn_fd_mutex);
    conn_fds_open[conn_fd] = 0;
    close(conn_fd);
    pthread_mutex_unlock(&conn_fd_mutex);

    // yield to any other connection threads before writing to the stats file
    pthread_yield();    

    pthread_mutex_lock(&stats_mutex);
    if((stats_file = fopen("stats.txt", "a")) != NULL)
    {
        if((stats_buff = malloc((strlen(time_buff) + 51 + strlen(requested_filename)) * sizeof(char))) != NULL)
        {
            stats_buff_size = sprintf(stats_buff, "Date - %s | Response Code - %d | Requested File - %s\n", time_buff, response_code, requested_filename);
            fwrite(stats_buff, stats_buff_size, 1, stats_file);
            free(stats_buff);
        }

        fclose(stats_file);
    }
    pthread_mutex_unlock(&stats_mutex);
}

void sig_handler(int sig)
{
    close(sock_fd);
    exit(0);
}

Answer 1

A possible race condition occurs because the socket (file) descriptor returned by accept() is passed to the thread function by reference.

It then is assigend to the thread function specific copy of socket (file) descriptor asynchronously. The latter could happen after the value of the referenced socket (file) descriptor changed due to the next call to accept().

To change this

• either misuse the void pointer passed to the thread function as integer (not recommended)
• or dynamically create instances of an int to assign the result of the accept() calls to.

---

Also (as commented by Nemo) accept() always returns a fresh socket.