Question

I’m designing a server program in C++ to receive multiple client connections and pass them into threads, however I’ve reached an impasse.

The socket connections all work fine, as does the multi-threading – almost. Please see my code below (it compiles and runs fine).

I’ve tried to pare it down to the essentials for you to make it easy to follow and take up the least of your time. I’ve commented the code to help you see where the problem is, then I describe the problem in detail at the bottom. If you can help me then I would be very grateful!

#include <vector>
#include <boost/thread.hpp>
#include "unix_serverSocket.h"
#include "server.h"

extern const string socketAddress;


void do_stuff(ServerSocket *client)
{
    string in;
    string out;

    try
    {
        /* Gets input until the client closes the connection, then throws an exception, breaking out of the loop */
        while (true)
        {
            *client >> in;   /* Receives data from client socket connection */

            /* Assume the input is processed fine and returns the result into 'out' */

            sleep(3);   /* I've put sleep() here to test it's multithreading properly - it isn't */

            *client << out;   /* Returns result to client - send() is called here */

            /* If I put sleep() here instead it multithreads fine, so the server is waiting for send() before it accepts a new client */
        }
    }
    catch (SocketException &)
    {
        delete client;
        return;
    }
}


int main()
{
    try
    {
        ServerSocket server(socketAddress);

        while (true)
        {
            ServerSocket *client = new ServerSocket();

            /* See below */
            server.accept(*client);

            boost::thread newThread(do_stuff, client);
        }
    }
    catch (SocketException &e)
    {
        cout << "Error: " << e.description() << endl;
    }    

    return 0;
}

After a client socket connection has been passed to a thread, main() gets back to
the line:

server.accept(*client);

but then waits for the previous connection to send its result back to the
client via send() before it will accept a new connection – i.e. the server is waiting
for something to happen in the thread before it will accept a new client! I don’t
want it to do this – I want it to send the client connection to a thread then accept
more client connections straight away and pass them into more threads!

In case you’re wondering why I created a pointer to the socket here…

ServerSocket *client = new ServerSocket();

… if I don’t create a pointer then the recv() function called by the thread fails to receive data from the client, which seems to be due to the thread shallow copying the client socket connection and the garbage collector not understanding threads and thinking the client connection is no longer going to be used after it has been passed to the thread and so destroying it before recv() is called in the thread. Hence using a pointer created on the heap, which worked. Anyway, when I reworked the code using fork() instead of threads (which meant I didn’t need to create the socket on the heap), I still had the same problem with the server not being able to accept new clients.

I guess I need to change the server settings somehow so that it doesn’t wait for a client to send() before accepting a new one, however despite much Googling I’m still at a loss!

Here’s the relevant socket connection code in case it helps (the server and clients are all on the same box and thus connecting via local UNIX sockets):

class Socket
{
private:
    int sockfd;
    struct sockaddr_un local;

public:
    Socket();
    virtual ~Socket();

    bool create();
    bool bind(const string &);
    bool listen() const;
    bool accept(Socket &) const;

    bool send(const string &) const;
    int recv(string &) const;

    void close();

    bool is_valid() const 
    { 
        return sockfd != -1;
    }
};


bool Socket::create()
{
    sockfd = socket(AF_UNIX, SOCK_STREAM, 0);

    if (!is_valid())
    {
        return false;
    }

    int reuseAddress = 1;

    if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, (const char*) &reuseAddress, sizeof(reuseAddress)) == -1)
    {
        return false;
    }

    return true;
}


bool Socket::bind(const string &socketAddress)
{
    if (!is_valid())
    {
        return false;
    }

    local.sun_family = AF_UNIX;
    strcpy(local.sun_path, socketAddress.c_str());
    unlink(local.sun_path);
    int len = strlen(local.sun_path) + sizeof(local.sun_family);

    int bind_return = ::bind(sockfd, (struct sockaddr *) &local, len);

    if (bind_return == -1)
    {
        return false;
    }

    return true;
}


bool Socket::listen() const
{
    if (!is_valid())
    {
        return false;
    }

    int listen_return = ::listen(sockfd, MAXCLIENTCONNECTIONS);

    if (listen_return == -1)
    {
        return false;
    }

    return true;
}


bool Socket::accept(Socket &socket) const
{
    int addr_length = sizeof(local);

    socket.sockfd = ::accept(sockfd, (sockaddr *) &local, (socklen_t *) &addr_length);

    if (socket.sockfd <= 0)
    {
        return false;
    }
    else
    {
        return true;
    }
}


int Socket::recv(string &str) const
{
    char buf[MAXRECV + 1];

    str = "";

    memset(buf, 0, MAXRECV + 1);

    int status = ::recv(sockfd, buf, MAXRECV, 0);

    if (status == -1)
    {
        cout << "status == -1   errno == " << errno << "  in Socket::recv" << endl;
        return 0;
    }
    else if (status == 0)
    {
        return 0;
    }
    else
    {
        str = buf;
        return status;
    }
}


bool Socket::send(const string &str) const
{
    int status = ::send(sockfd, str.c_str(), str.size(), MSG_NOSIGNAL);

    if (status == -1)
    {
        return false;
    }
    else
    {
        return true;
    }
}


class ServerSocket : private Socket
{
public:
    ServerSocket(const string &);
    ServerSocket() {};
    virtual ~ServerSocket();

    void accept(ServerSocket &);

    const ServerSocket & operator << (const string &) const;
    const ServerSocket & operator >> (string &) const;
};


ServerSocket::ServerSocket(const string &socketAddress)
{   
    if (!Socket::create())
    {
        throw SocketException("Could not create server socket");
    }

    if (!Socket::bind(socketAddress))
    {
        throw SocketException("Could not bind to port");
    }

    if (!Socket::listen())
    {
        throw SocketException("Could not listen to socket");
    }
}


void ServerSocket::accept(ServerSocket &socket)
{   
    if (!Socket::accept(socket))
    {
        throw SocketException("Could not accept socket");
    }
}


const ServerSocket & ServerSocket::operator << (const string &str) const
{   
    if (!Socket::send(str))
    {
        throw SocketException("Could not write to socket");
    }

    return *this;
}


const ServerSocket & ServerSocket::operator >> (string &str) const
{
    if (!Socket::recv(str))
    {
        throw SocketException("Could not read from socket");
    }

    return *this;
}

Answer 1

I’ve figured it out! The reason the clients weren’t multithreading was that the program creating the client connections was doing so within a mutex – hence it wouldn’t create a new connection until the old one had received a reply from the server, and thus the server appeared to be only single-threading! So in short my server program above was fine and it was a problem at the client end – sorry for wasting your time – I didn’t even consider the possibility until I completely reworked the program structure by putting the threading at the client end instead, which then revealed the issue.

Thanks for all your help!