Question

I’m chaining several deferreds together, because I need the results from one function before I start the next. However, the code breaks after the first sucessful callback.

class SaveContents(Protocol):
    def __init__(self, finished, filesize, filename):
        self.finished = finished
        self.remaining = filesize
        self.outfile = open(filename, 'wb')

    def dataReceived(self, bytes):
        if self.remaining:
            display = bytes[:self.remaining]
            self.outfile.write(display)
            self.remaining -= len(display)
        else:
            self.outfile.close()

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        self.outfile.close()
        self.finished.callback(None)

def cbRequest(response):
    print 'Response version:', response.version
    print 'Response code:', response.code
    print 'Response phrase:', response.phrase
    print 'Response headers:'
    print 'Response length:', response.length
    print pformat(list(response.headers.getAllRawHeaders()))
    finished = Deferred()
    response.deliverBody(SaveContents(finished, response.length, 'test2.pdf'))
    return finished

def cbShutdown(ignored):
    reactor.stop()

def addBarcodeChain(result, infile, outfile, analyze, duplex):
    print "starting Chain with results {0}".format(result)
    d = addBarcode(infile, outfile, lastStatement=result.headers.getHeader('lastStatement'), analyze=analyze, duplex=duplex)
    return d

def addBarcode(infile, outfile, **kwargs):
    """Send the pdf file to the remote server for processing, then save the results."""
    agent = Agent(reactor)
    f = open('70935.pdf', 'rb')
    body = FileBodyProducer(f)
    fstr = 'filename={0}'.format(infile)
    stmnt = 'lastStatement={0}'.format(kwargs['lastStatement'])
    duplex = 'duplex={0}'.format(int(kwargs['duplex']))
    analyze = 'analyze={0}'.format(int(kwargs['analyze']))
    options = '&'.join([fstr, stmnt, duplex, analyze])
    d = agent.request(
        'POST',
        'http://127.0.0.1:7777?{0}'.format(options),
        Headers({'User-Agent': ['Twisted Web Client Example'],
                 'Content-Type': ['multipart/form-data; boundary=1024'.format()]}),
        body)
    return d

#===============================================
# Main methods
#===============================================
def main(infiles, output_path, output_filename, analyze, duplex, debug):
    logger.info("Start of processing {0}".format(infiles))
    if debug:
        logger.setLevel(logging.DEBUG)

    lastStatement = 0
    work = []
    for globFile in infiles:
        for f in glob(globFile):
            outname = '{0}/{1}{2}.pdf'.format(output_path, os.path.splitext(os.path.basename(f))[0], output_filename)
            work.append( (f, outname) )

    d = addBarcode(work[0][0], work[0][1], lastStatement=lastStatement, analyze=analyze, duplex=duplex)
    d.addCallback(cbRequest)
    d.addErrback(cbShutdown)
    for f, outname in work[1:]:
        d.addCallback(addBarcodeChain, f, outname, analyze=analyze, duplex=duplex)
        d.addCallback(cbRequest)
        d.addErrback(cbShutdown)

    d.addCallback(cbShutdown)
    d.addErrback(cbShutdown)

    reactor.run()

As near as I can figure, the deferred recursive loop in cbRequest is necessary for it to function correctly, but it does not pass any results on to the future callbacks, which is why addBarcodeChain fails when it attempts to use the result contents.

How would I adjust either cbRequest or SaveContents to pass forward the response object to future callbacks?

Answer 1

I figured it out. The relevant bit is the save contents class, as I suspected.

class SaveContents(Protocol):
    def __init__(self, finished, filesize, filename):
        self.finished = finished
        self.remaining = filesize
        self.outfile = open(filename, 'wb')

    def dataReceived(self, bytes):
        if self.remaining:
            display = bytes[:self.remaining]
            self.outfile.write(display)
            self.remaining -= len(display)
        else:
            self.outfile.close()

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        self.outfile.close()
        self.finished.callback(None)

Notably, when the connection is closed the connectionLost method gets called. When that happens, it is supposed to “clean up” the recursive callback loop, by setting self.finished.callback(None).

By changing this to self.finished.callback(self.response) and passing the response into the init method the response gets passed to future callbacks.

class SaveContents(Protocol):
    def __init__(self, finished, response, filesize, filename):
        self.finished = finished
        self.remaining = filesize
        self.response = response
        self.outfile = open(filename, 'wb')

    def dataReceived(self, bytes):
        if self.remaining:
            display = bytes[:self.remaining]
            self.outfile.write(display)
            self.remaining -= len(display)
        else:
            self.outfile.close()

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        self.outfile.close()
        self.finished.callback(self.response)

This solves the problem of the later callbacks getting None from their predecessors.