Question

I’ve been looking at this for a few hours. I’ve tried everything I can think of, and frankly It doesn’t make sense. I actively send and receive with the socket with no problems, but as soon as I change the data to a different message, same style, it stops recieving. I’m using TCP. I have a manager process send up to N router messages with table data. I later send a packet, same style, it receive it, and then stops receiving…. The code gets back to the top of the loop, but just doesn’t get any more data.

Oh the networking code I’m using is a copy and paste of beejs TCP server client code. http://beej.us/guide/bgnet/output/html/multipage/clientserver.html

Manager thread, this part works

for(vector< vector<int> >::iterator it = table.begin(); it!=table.end(); ++it ){
            vector< int > d = *it;

            for(vector<int>::iterator itA = d.begin(); itA!=d.end(); ++itA ){
                cout << "Sending... "<< *itA << endl;
                s <<*itA<<" ";
            }
            if (send(new_fd, s.str().c_str(), 13, 0) == -1)
                perror("Serv:send");
            sleep(2);
            logs << "Sent to router " << i <<":\n" << s.str();
            writeLog(logs.str().c_str());
            s.str("");
            logs.str("");


        }
        s<<"done";
        if (send(new_fd, s.str().c_str(), 13, 0) == -1)
            perror("Serv:send");
        writeLog(s.str().c_str());

manage 2, where only the first message gets through

 for(vector <vector <int > >::iterator it = toSendPackets.begin(); it != toSendPackets.end(); ++it){
    sleep(3);
    vector<int> tsp = *it;
    int a,b,c = 0;
    for(vector<int>::iterator itr = tsp.begin(); itr != tsp.end(); ++itr){
        if(c==0){
            a = *itr;
        }
        if(c==1){
            b = *itr;
        }

        c++;
    }
    ss.str("");
    ss << a << " " << b;
    for(int i = 0; i < numN; i++){
        int curSoc = socketList[i];
        stringstream sl;

        sl<<"sent:"<< ss.str().c_str();
        cout << "sending..  " << ss.str() << " to " << i << endl;
        if (send(curSoc, "HOP", strlen("HOP")+1, 0) == -1)
            perror("Serv:send");
        sleep(2);
        if (send(curSoc, ss.str().c_str(), strlen(ss.str().c_str())+1, 0) == -1)
            perror("Serv:send");
        writeLog(sl.str().c_str());
        sleep(1);

    }
}

Router code.

The manager code above and manager code 2 both send to this part of the code.
It gets the first send, in this case “HOP” and then nothing? I removed the HOP packet parsing, so it litterally should only state that something was read.

if(tid == 0){// TCP
    stringstream s;
    bool proc = true;
    while(!doneFlag){
        proc = true;
        cout << "TCP RECEIVING... " << endl;
        int numbytes = 0;
        while(numbytes==0){
            if ((numbytes = recv(sockfd, buf, MAXDATASIZE, 0)) == -1) {
                perror("recvROUTERThread0");
                exit(1);
            }
        }
        buf[numbytes] = '\0';
        numbytes = 0;
        if(strcmp("Quit",buf)==0){
            writeLog("Quit read",outName);
            doneFlag = true;
            close(net.sockfd);
            floodUDP("Quit");
            pthread_exit(NULL);

        }
        else if(strcmp("HOP",buf)==0){
            cout << "HOP READ" << endl;
            numbytes = 0;
            while(numbytes==0){
                if ((numbytes = recv(sockfd, buf, MAXDATASIZE, 0)) == -1) {
                    perror("recvROUTERThread0");
                    exit(1);
                }
            }
            s << id << "R: Receiving a routing command! " << buf;
            cout << s.str().c_str() << endl;
            writeLog(s.str().c_str(),outName);
            HOPpacket hpo = genHopOrig(s.str().c_str());
            if(hpo.s == atoi(id)){
                printHOP(hpo);
                //              cout << "PACKET " << pr << endl;
                stringstream sl;
                char* hop = generateHopPacket(hpo);
                sl << "Generating HOP packet and sending.. " << hop;
                writeLog(sl.str().c_str(),outName);
                sendHOP(hop);
            }
        }
        else{
            cout << "Table row data from manager" << endl;
            s.str("");
            s << id << "R: MANAGER MESSAGE: " << buf << endl;
            cout << s.str() << endl;
            writeLog(s.str().c_str(),outName);
            int intID = atoi(id);
            vector <int> tr = processTR(buf,intID,basePN);
            table.push_back(tr);
        }

    }
   }

My output. In this case there are 10 routers running. Note I didn’t change my prints to state that it was sending HOP then 0 5 ..

sending..  0 5 to 0

HOP READ
WRITTING Manager log:12-11-23::4:6:26:
sent:0 5

sending..  0 5 to 1
HOP READ
WRITTING Manager log:12-11-23::4:6:29:
sent:0 5

sending..  0 5 to 2
HOP READ
WRITTING Manager log:12-11-23::4:6:32:
sent:0 5

sending..  0 5 to 3
HOP READ
WRITTING Manager log:12-11-23::4:6:35:
sent:0 5

sending..  0 5 to 4
HOP READ
WRITTING Manager log:12-11-23::4:6:38:
sent:0 5

sending..  0 5 to 5
HOP READ
WRITTING Manager log:12-11-23::4:6:41:
sent:0 5

sending..  0 5 to 6
HOP READ
WRITTING Manager log:12-11-23::4:6:44:
sent:0 5

sending..  0 5 to 7
HOP READ
WRITTING Manager log:12-11-23::4:6:47:
sent:0 5

sending..  0 5 to 8
HOP READ
WRITTING Manager log:12-11-23::4:6:50:
sent:0 5

sending..  0 5 to 9
HOP READ
WRITTING Manager log:12-11-23::4:6:53:
sent:0 5

sending..  3 9 to 0
WRITTING Manager log:12-11-23::4:6:59:
sent:3 9

sending..  3 9 to 1
WRITTING Manager log:12-11-23::4:7:2:
sent:3 9

sending..  3 9 to 2
WRITTING Manager log:12-11-23::4:7:5:
sent:3 9

sending..  3 9 to 3
WRITTING Manager log:12-11-23::4:7:8:
sent:3 9

sending..  3 9 to 4
WRITTING Manager log:12-11-23::4:7:11:
sent:3 9

sending..  3 9 to 5
WRITTING Manager log:12-11-23::4:7:14:
sent:3 9

sending..  3 9 to 6
WRITTING Manager log:12-11-23::4:7:17:
sent:3 9

sending..  3 9 to 7
WRITTING Manager log:12-11-23::4:7:20:
sent:3 9

sending..  3 9 to 8
WRITTING Manager log:12-11-23::4:7:23:
sent:3 9

sending..  3 9 to 9
WRITTING Manager log:12-11-23::4:7:26:
sent:3 9

Answer 1

There is a problem when you recv data, TCP is a stream based socket not a message based one, so if you use:

send( sock, buf1, len1, 0 );  // Send HOP, since it is small, you OS merge this
send( sock, buf2, len2, 0 );  // with next send!

and then try to receive data using recv it is not guaranteed that you receive data in 2 separate calls to recv, so you may receive both sent buffers in one call to recv:

recv( sock, buf, len, 0 );  // This may receive both buffers in one call

so your next call to recv will be blocked for data that already received in first call! Also they may be another problem for when you send large buffer, then recv may receive less data than a single message passed using send.

You must define a protocol that define end of message in the received stream and then receive your data according to that protocol. for example, you may first send length of message or define something that indicate end of it(for example \0 or \r\n).

Sorry for my incomplete description of the error. In your comment you say that you have increased the HOP message size! But it certainly isn’t a good practice, also increased size is so small that never force OS to send it immediately( actually there is no certain size that force OS do that ). If you want OS to send your data immediately, you should disable Nagle algorithm using TCP_NO_DELAY option, but before doing that take a look at How do I use TCP_NODELAY?. Doing this is not a good practice either and beside that while doing this cause your packet sent immediately as you call send but it never force OS on receiver side to receive messages separately!! so what is the correct way of doing this?

I explain the problem in detail:

// I don't know exact value of MAXDATASIZE but I will assume it is 128
char buf[ MAXDATASIZE ];

int numbytes = recv( sock, buf, MAXDATASIZE, 0 );
if( numbyte == -1 ) {
    // Handle error
}

// I assume HOP_MSG is a defined constant that contain value of HOP message
if( strcmp(buf, HOP_MSG) == 0 ) { // <-- (1)
    while( (numbytes = recv(sock, buf, MAXDATASIZE, 0)) != -1 ) { // <-- (2)
        if( numbytes == 0 ) break;
    }
    if( numbytes == -1 ) {
        // Handle error
    }
}

But wait! in line that marked with (1) I assumed recv read HOP_MSG completely and only HOP_MSG, but why?? As I said before TCP is a stream protocol and there is no message boundary in it, so it may read only 2 bytes!! or it read 1KB( that is certainly more than HOP_MSG, so what should I do??

The working answer is something like follow:

int receive_till_zero( SOCKET sock, char* tmpbuf, int& numbytes ) {
    int i = 0;
    do {
        // Check if we have a complete message
        for( ; i < numbytes; i++ ) {
            if( buf[i] == '\0' ) {
                // \0 indicate end of message! so we are done
                return i + 1; // return length of message
            }
        }
        int n = recv( sock, buf + numbytes, MAXDATASIZE - numbytes, 0 );
        if( n == -1 ) {
            return -1; // operation failed!
        }
        numbytes += n;
    } while( true );
}
void remove_message_from_buffer( char* buf, int& numbytes, int msglen ) {
    // remove complete message from the buffer.
    memmove( buf, buf + msglen, numbytes - msglen );
    numbytes -= msglen;
}

void main() {
    SOCKET s;
    char buf[ MAXDATASIZE ];
    int numbytes = 0, msglen;
    // Initialize socket and connect to server, you already do that

    while( true ) {
        msglen = receive_till_zero( s, buf, numbytes );
        if( msglen == -1 ) {/* Handle error */}

        if( !strcmp(buf, HOP_MSG) ) {
            remove_message_from_buffer( buf, numbytes, msglen );
            msglen = receive_till_zero( s, buf, numbytes );
            if( msglen == -1 ) {/* Handle error */}

            std::cout << "Message received from server: " << buf << std::endl;
            remove_message_from_buffer( buf, numbytes, msglen );
        }
    }
}

By debugging this code you will certainly understand its purpose, receive_till_zero assume there is already some pending data in the buffer from previous call to recv, so it will first check if there is a complete message in the buffer or not and also it never assume receiving data completed just by one call to recv so it will call recv in a loop until it see a \0 in the buffer. After we finished with data in the buffer we call remove_message_from_buffer to eat that data and only that data, and not just start receiving from the start of buffer, since they may already some data in the buffer.

As you see code is a little complicated, for a better programming model and a better C++ code you may use boost::asio that have a very good design and work perfectly with C++ and iostream