Question

import multiprocessing as mp
import time

def build(q):
    print 'I build things'
    time.sleep(10)
    #return 42
    q.put(42)

def run(q):
    num = q.get()
    print num

    if num == 42:
        print 'I run after build is done'
        return
    else:
        raise Exception("I don't know build..I guess")

def get_number(q):
    q.put(3)

if __name__ == '__main__':
    queue = mp.Queue()

    run_p = mp.Process(name='run process', target=run, args=(queue,))
    build_p = mp.Process(name='build process', target=build, args=(queue,))
    s3 = mp.Process(name='s3', target=get_number, args=(queue,))

    build_p.start()
    run_p.start()
    s3.start()

    print 'waiting on build'
    build_p.join(1) # timeout set to 1 second
    s3.join()

    print 'waiting on run'
    run_p.join()

    queue.close()
    print 'waiting on queue'
    queue.join_thread()
    print 'done'

My goal is to send build and run into different workers, and run has to get result back from build in order to proceed.

The above revised code based on your help will actually return exception, because s3 is returned before build has the chance.

The value in the front of the queue is now 3. How can we make sure we get the answer back from build process?

Thanks.

Answer 1

Your question is a little murky..the problem you are describing sounds synchronous so 3 processes are a little overkill.

Assuming you are just trying to pass values to run you could use the queue object.

import multiprocessing as mp
import time

def build(q):
    print 'I build things'
    time.sleep(5)
    q.put(42)
    return 

def run(q):
    while True:
        num = q.get()
        if num == 42:
            print 'I run after build is done'
            return
        else:
            print 'not the right number...'

    def get_number():
        return 41

if __name__ == '__main__':
    queue = mp.Queue()

    run_p = mp.Process(name='run process', target=run, args=(queue,))
    build_p = mp.Process(name='build process', target=build, args=(queue,))

    run_p.start()
    build_p.start()

    print 'waiting on build'
    build_p.join()
    print 'waiting on run'
    run_p.join()
    queue.close()
    print 'waiting on queue'
    queue.join_thread()
    print 'done'