Question

I have a simple example script constructed that defines three separate processes using multiprocessing in python. My objective is to have one parent thread that spawns two smaller threads that will collect and process data.

Currently, my implementation looks like this:

from Queue import Queue,Empty
from multiprocessing import Process
import time
import hashlib


class FillQueue(Process):
    def __init__(self,q):
        Process.__init__(self)
        self.q = q

    def run(self):
        i = 0
        while i is not 5:
            print 'putting'
            self.q.put('foo')
            i+=1
        self.q.put('|STOP|')

class ConsumeQueue(Process):
    def __init__(self,q):
        Process.__init__(self)
        self.q = q

    def run(self):
        print 'Consume'
        while True:
            try:
                value =  self.q.get(False)
                print value
                if value == '|STOP|':
                    print 'done'
                    break;
            except Empty:
                print 'Nothing to process atm'

class Ripper(Process):

    q = Queue()

    def __init__(self):
        self.fq = FillQueue(self.q)
        self.cq = ConsumeQueue(self.q)
        self.fq.daemon = True
        self.cq.daemon = True

    def run(self):
        try:
            self.fq.start()
            self.cq.start()
        except KeyboardInterrupt:
            print 'exit'

if __name__ == '__main__':
    r = Ripper()
    r.start()

As it runs presently, the output from the script on CLI looks like this:

putting
putting
putting
putting
putting
Consume
foo
foo
foo
foo
foo
|STOP|
done

Obviously, the way I am starting my two threads is blocking, since the consumer doesn’t even begin to process the items in the queue until the filler finishes adding items.

How should I rewrite this to make both threads begin immediately and not block, so the consumer will simply pass to the Empty except block while there is no work to process, but will exit completely when it receives the stop message?

EDIT: typo, had the start and run methods mixed up

Answer 1

try this:

from Queue import Empty
from multiprocessing import Process, Queue
import time
import hashlib


class FillQueue(object):
    def __init__(self, q): 
        self.q = q 

    def run(self):
        i = 0 
        while i < 5:
            print 'putting'
            self.q.put('foo %d' % i ) 
            i+=1
            time.sleep(.5)
        self.q.put('|STOP|')

class ConsumeQueue(object):
    def __init__(self, q): 
        self.q = q 

    def run(self):
        while True:
            try:
                value =  self.q.get(False)
                print value
                if value == '|STOP|':
                    print 'done'
                    break;
            except Empty:
                print 'Nothing to process atm'
                time.sleep(.2)


if __name__ == '__main__':
    q = Queue()
    f = FillQueue(q)
    c = ConsumeQueue(q)

    p1 = Process(target=f.run)
    p1.start()

    p2 = Process(target=c.run)
    p2.start()

    p1.join()
    p2.join()