Question

import os
import urllib
import workerpool

from datetime import datetime

class DownloadJob(workerpool.Job):    

    def __init__(self, fa):
       self.fa = fa 

    def run(self):        
       f = open(self.fa + '.txt','w')
       f.write('Example Note.......')
       f.close()  


pool = workerpool.WorkerPool(size=5)

def workfile():        
    range1 = 51
    range2 = 102
    fam1 = 555
    fam2 = 833

    ranges = range2 -range1
    fams = fam2 -fam1
    workname = "Python"

    path = os.getcwd()

    os.system('mkdir ' + str(workname))
    sTime = datetime.now()

    for a in range(ranges + 1):
        os.chdir(path + '\\' + str(workname))
        os.system('mkdir ' + str(range1 + a))
        os.chdir(path + '\\' + str(workname) + '\\' + str(range1 + a))

        for b in range(fams + 1):
            fa = str(fam1 + b)
            job = DownloadJob(fa)
            pool.put(job)
            pool.shutdown()
            pool.wait()                                              

        print 'Elapsed Time: %s' % (datetime.now() - sTime)

        z = open('info.txt','w')
        z.write('Elapsed Time: %s' % (datetime.now() - sTime))
        z.close()

    os.chdir(path + '\\' + str(workname))

    tumSure = open('info.txt','w')
    tumSure.write('Elapsed All Time: %s' % (datetime.now() - sTime))
    tumSure.close()

    print 'All Time: %s' % (datetime.now() - sTime)        
    print 'Workname : %s downloaded.' % (workname)

    quit()

workfile()

Hi all,

I have a code as above and I want to use thread logic for creating file. Folder numbers start wird range1, i.e. 51. Text files are created in this directory with the names 555.txt to 833.txt. But after creating the folder with name 52 it stops, failing to create 555.txt to 833.txt.

it think it stops because

pool.shutdown()
pool.wait()

How can I make the loop continue with no stop?

Answer 1

I think you should only shutdown the pool if you are finished with it, i.e. after the for a loop, maybe even in a try...finally clause.

It would look that way:

try:
    for a in range(ranges + 1):
        os.chdir(path + '\\' + str(workname))
        os.system('mkdir ' + str(range1 + a))
        os.chdir(path + '\\' + str(workname) + '\\' + str(range1 + a))

        for b in range(fams + 1):
            fa = str(fam1 + b)
            job = DownloadJob(fa)
            pool.put(job)
finally:
    pool.shutdown()
    pool.wait()

In this way the pool shutdown

a) happens only if all pool putting is done and
b) happens even if there is an exception in order to cleanly shutdown.

If the pool had a context manager, it would be even simpler. But AFAICS, it hasn’t.
Otherwise, you could do

with pool:
    for a in range(ranges + 1):
        os.chdir(path + '\\' + str(workname))
        os.system('mkdir ' + str(range1 + a))
        os.chdir(path + '\\' + str(workname) + '\\' + str(range1 + a))

        for b in range(fams + 1):
            fa = str(fam1 + b)
            job = DownloadJob(fa)
            pool.put(job)

But if you want, you can do

from contextlib import contextmanager

@contextmanager
def shutdown_wait(pool):
    try:
        yield pool
    finally:
        pool.shutdown()
        pool.wait()

...
with shutdown_wait(pool):
    for a ... [as above]