Question

I am working on a client – server application in a multi-threaded environment.Both client and server have two threads.The main thread sends data using socket (IPv4-TCP) and the corresponding main thread on client side recvs the data.The send and recv functions are custom functions as part of my design.I have set three signal handlers in the other thread for SIGUSR1,SIGUSR2 and SIGINT on server. On receiving SIGINT, thread clean up is done to gracefully close all the sockets and terminate the threads, where as on receiving SIGUSR1,SIGUSR2 , I set two global flags which are used in the same thread and custom send function in main thread to do some operations on switching the socket id to IPv6.(There is logic to let the client know the socket is changed to IPv6). The custom send/recv functions have malloc and free functions.

The problem is when I use kill -SIGUSR1 pid on terminal to send signal to server process, after some send calls, the transmission hangs.For each send call I am sending packets, that has the data size to be sent,the actual data and an optional flag to indicate that the next data will be on another socket id. When I print the data size on client side, it is all zero after certain recv calls after signal is received.I am sure the SIGNAL has to be the cause as when I reverse the operation and have client send the data to server(uploading) using same send/recv function it works fine. I am able to switch socket ids. In both the cases I am sending signal to server process.The recv function recvs untill the the amount of data is same as indicated in the size part of that packet as TCP is stream based.
I am not sure why size becomes zero after some send calls on receiving the signal.I have used mutexes where ever global variables are used except in the signal handler part when they are set. The codes is as below.

Thread 2:

fn_sigUsrHandler(SIGUSR1);
fn_sigUsrHandler(SIGUSR2);
fn_sigUsrHandler(SIGINT);
void fn_sigUsrHandler(int p_signal)
{
/* Signal handler structure */
struct sigaction vl_sigStruct;
int vl_errno;
char vl_err_buff[256];

vl_sigStruct.sa_flags = SA_RESTART;
sigemptyset(&vl_sigStruct.sa_mask);


switch(p_signal)
{
    case SIGUSR1:
        vl_sigStruct.sa_handler = fn_switch;
        break;
    case SIGUSR2:
        vl_sigStruct.sa_handler = fn_switch;
        break;
    case SIGINT:
        vl_sigStruct.sa_handler = fn_cleanUP;
        break;
}

vl_errno = sigaction(p_signal, &vl_sigStruct, NULL);
if (vl_errno == -1) {
    fprintf(stdout,"Control Thread-Error in catching SIGUSR1:%s\n",fn_strerror_r(errno,vl_err_buff));
    exit(EXIT_FAILURE);
}


void fn_switch(int st_val){
/*
pthread_mutex_lock(&socket_mutex_3);
ip_proto_switch = 1;
vg_ctrlpacket_sent =1;
pthread_mutex_unlock(&socket_mutex_3);
*/

vg_proto_switch = 1;
vg_ctrlpacket_sent =1;

fprintf(stdout,"Signalled to switch\n");
}

Main thread:

int vl_err; /* Number of bytes sents */
char err_buff[256]; /* Buffer to hold error message*/
int vl_change_sock = 0;

if(p_flags != NO_DATA_TX)
{
    char *vl_bufData;
    st_packetData vst_packet; /* Structure holding the data to be sent */
    unsigned int vl_packetSize = sizeof(unsigned int) +     sizeof(vst_packet.vst_pad) + (int)p_len;//sss
    vl_bufData = (char *)malloc(vl_packetSize);
    memset(vl_bufData,'\0',vl_packetSize);

    pthread_mutex_lock(&socket_mutex_2);
        if(vg_ctrlpacket_recv == 1){
            ///strcpy(vst_packet.vst_pad,vg_change_socket);
            vl_change_sock = 1;
            vg_ctrlpacket_recv = 0;                 
            fprintf(stdout,"len:%d\n",strlen(vst_packet.vst_pad));
        }
    pthread_mutex_unlock(&socket_mutex_2);


    if(vl_change_sock == 1){


            char *vl_bufData2 = vl_bufData+sizeof(unsigned int);
            snprintf(vl_bufData2,(int)p_len,"%s",p_buffer);
            //memcpy(vl_bufData+sizeof(unsigned int)+(int)p_len,vg_change_socket,sizeof(vst_packet.vst_pad));//sss
        */

        snprintf(vl_bufData,sizeof(unsigned int)+1,"%u",vl_packetSize);
        memcpy(vl_bufData+sizeof(unsigned int),p_buffer,(int)p_len);//sss
        snprintf(vl_bufData+sizeof(unsigned int)+(int)p_len,sizeof(vst_packet.vst_pad)+1,"%s",vg_change_socket);

        vl_err = send(p_mysocket->socket_id,vl_bufData,vl_packetSize,p_flags);
        if(vl_err == -1)
        {
            fprintf(stderr,"mysocket-fn_send-TCP-vl_err err :%s\n",fn_strerror_r(errno,err_buff));//HS
            exit(EXIT_FAILURE);//HS 
        }

        if(debug > 0)
            fprintf(stdout,"The socket before change is :%d client side \n",p_mysocket->socket_id);

        if((p_mysocket->socket_id) == p_mysocket->sock_id[0])
            p_mysocket->socket_id = p_mysocket->sock_id[1];
        else
            p_mysocket->socket_id = p_mysocket->sock_id[0];

        if(debug > 0)
            fprintf(stdout,"The socket after change is :%d client side \n ",p_mysocket->socket_id);

    }
    else{
        snprintf(vl_bufData,sizeof(unsigned int)+1,"%u",vl_packetSize);
        memcpy(vl_bufData+sizeof(unsigned int),p_buffer,(int)p_len);//sss

        vl_err = send(p_mysocket->socket_id,vl_bufData,vl_packetSize,p_flags);
        if(vl_err == -1)
        {
            fprintf(stderr,"mysocket-fn_send-TCP-vl_err err :%s\n",fn_strerror_r(errno,err_buff));//HS
            exit(EXIT_FAILURE);//HS 
        }

    }   

    if(debug > 2){
        /*fprintf(stdout,"size of st_packetData:%d\n",sizeof(st_packetData));
        fprintf(stdout,"vl_packetSize:%d\n",vl_packetSize); 
        fprintf(stdout,"Memcopied-data:%s\n",vst_packet.vst_data);
        fprintf(stdout,"Memcopied-pad:%s\n",vst_packet.vst_pad);
        fprintf(stdout,"Memcopied-size:%d\n",vst_packet.vst_size);//sss
        fprintf(stdout,"Data from buffer:%s\n",p_buffer);
        fprintf(stdout,"data:%s\n",vl_bufData+sizeof(vst_packet.vst_size)+sizeof(vst_packet.vst_pad));*/
        fprintf(stdout,"Copied data:%-10.6s\n",vl_bufData);
        fprintf(stdout,"---------------------------\n");
    }


    //if(vl_err >=(sizeof(vst_packet.vst_size)+ sizeof(vst_packet.vst_pad))) //sss
    if(vl_err >=(sizeof(unsigned int)+ strlen(vg_change_socket)))
    {
        vl_err = vl_err-(sizeof(unsigned int) + strlen(vg_change_socket));//sss
    }
    if(debug > 2)
    {
        fprintf(stdout,"The socket id is :%d.. Thread:%s\n",p_mysocket->socket_id,p_mysocket->vst_nm_thread);
        fprintf(stdout,"The data tx %d.. Thread:%s\n",vl_err,p_mysocket->vst_nm_thread);
    }

    free((void *)vl_bufData);

}
else
{   
    vl_err = send(p_mysocket->socket_id,p_buffer,p_len,0);
    if(vl_err == -1)
    {
        fprintf(stderr,"mysocket-fn_send-TCP-vl_err err :%s\n",fn_strerror_r(errno,err_buff));//HS
        exit(EXIT_FAILURE);//HS 
    }
    if(debug>1)
    {
        fprintf(stdout,"The socket id is :%d.. Thread:%s\n",p_mysocket->socket_id,p_mysocket->vst_nm_thread);
        fprintf(stdout,"The data tx %d.. Thread:%s\n",vl_err,p_mysocket->vst_nm_thread);
    }
}
/* return number of bytes sent */
return vl_err;
}

Answer 1

Thanks all for your responses. After bit of further research, I found that the SIGUSR1 was handled by the main thread and not the control thread.As the fn_recv called had functions like malloc,free which I suppose are not re-entrant, it was causing the problem.

I solved it by masking the SIGUSR1 signal and creating a seperate thread to set the signal handler instead of control thread to avoid interference with system/function calls. I unmasked the SIGUSR1 signal in the new thread to make sure it is being handled only by this new thread.
I am now able to transfer data and switch the socket id multiple times without an stalls.
Please let me know if anybody need more details. Basically masking the signal in main thread and unmaksing it in child thread did the trick!