Question

I’m writing interprocess communication using localhost sockets in Python 3.2 and testing it on Windows. Here is some test code with a server and a client, sending messages to each other. Strangely, it fails randomly with the RuntimeError error, raised in the receive function, somewhere around the 5th or 10th connection.

#!/usr/bin/python
# -*- coding: utf-8 -*-

import socket
import pickle
import time
import logging
from multiprocessing import Process

def receive(conn):
    def ensure_receive(length):
        parts = []
        received = 0
        while received < length:
            chunk = conn.recv(length - received)
            if not chunk:
                raise RuntimeError("Connection broken")
            parts.append(chunk)
            received += len(chunk)
        return b''.join(parts)
    lengthString = ensure_receive(8)
    serialized = ensure_receive(int(lengthString))
    return pickle.loads(serialized)

def send(conn, message):
    def ensure_send(message):
        sent = 0
        while sent < len(message):
            sent += conn.send(message[sent:])
#            logging.warning("Now sending")
    serialized = pickle.dumps(message, 1)
    messageLength = len(serialized)
    ensure_send("{:8}".format(messageLength).encode('Latin-1'))
    ensure_send(serialized)

def client_function(clientLimit):
    for index in range(1, clientLimit + 1):
        print ("Client", index)
        try:
            conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            conn.connect(('localhost', 12333))
            send(conn, list(range(100000)))
            message = receive(conn)
            send(conn, list(range(100)))
#            time.sleep(0.01)
            conn.shutdown(socket.SHUT_WR)
            conn.close()
        except Exception:
            logging.exception("Socket error in client")

def server_function(clientLimit):
    newSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    newSocket.bind(('localhost', 12333))
    newSocket.listen(16)
    for _ in range(clientLimit):
        (conn, address) = newSocket.accept()
        time.sleep(0.01)
        message = receive(conn)
        send(conn, list(range(10)))
        message = receive(conn)
        conn.shutdown(socket.SHUT_WR)
        conn.close()

def test(clientLimit):
    server = Process(target = server_function, args = (clientLimit,))
    server.start()

    time.sleep(1)
    client = Process(target = client_function, args = (clientLimit,))
    client.start()

    client.join()
    server.join()

if __name__ == "__main__":
    test(100)

However, there are no errors if I uncomment time.sleep(0.01) in client_function, or if I change message order a bit.

Is there a way to make it work, without putting in random waits, and allowing for arbitrary protocols?

Answer 1

It is because of conn.shutdown(socket.SHUT_WR) in your server_function. What you need is socket.SHUT_RD, or better yet, get rid of the shutdown() call at all.