Question

is it possible to easily cap the kbps when using urllib2? If it is, any code examples or resources you could direct me to would be greatly appreciated.

Answer 1

There is the urlretrieve(url, filename=None, reporthook=None, data=None) function in the urllib module. If you implement the reporthook-function/object as either a token bucket, or a leaky bucket, you have your global rate-limit.

EDIT: Upon closer examination I see that it isn’t as easy to do global rate-limit with reporthook as I thought. reporthook is only given the downloaded amount and the total size, which on their own isn’t enough to information to use with the token-bucket. One way to get around it is by storing the last downloaded amount in each rate-limiter, but use a global token-bucket.

---

EDIT 2: Combined both codes into one example.

'''Rate limiters with shared token bucket.'''  import os import sys import threading import time import urllib import urlparse  class TokenBucket(object):     '''An implementation of the token bucket algorithm.     source: http://code.activestate.com/recipes/511490/      >>> bucket = TokenBucket(80, 0.5)     >>> print bucket.consume(10)     True     >>> print bucket.consume(90)     False     '''     def __init__(self, tokens, fill_rate):         '''tokens is the total tokens in the bucket. fill_rate is the         rate in tokens/second that the bucket will be refilled.'''         self.capacity = float(tokens)         self._tokens = float(tokens)         self.fill_rate = float(fill_rate)         self.timestamp = time.time()         self.lock = threading.RLock()      def consume(self, tokens):         '''Consume tokens from the bucket. Returns 0 if there were         sufficient tokens, otherwise the expected time until enough         tokens become available.'''         self.lock.acquire()         tokens = max(tokens,self.tokens)         expected_time = (tokens - self.tokens) / self.fill_rate         if expected_time <= 0:             self._tokens -= tokens         self.lock.release()         return max(0,expected_time)      @property     def tokens(self):         self.lock.acquire()         if self._tokens < self.capacity:             now = time.time()             delta = self.fill_rate * (now - self.timestamp)             self._tokens = min(self.capacity, self._tokens + delta)             self.timestamp = now         value = self._tokens         self.lock.release()         return value  class RateLimit(object):     '''Rate limit a url fetch.     source: http://mail.python.org/pipermail/python-list/2008-January/472859.html     (but mostly rewritten)     '''     def __init__(self, bucket, filename):         self.bucket = bucket         self.last_update = 0         self.last_downloaded_kb = 0          self.filename = filename         self.avg_rate = None      def __call__(self, block_count, block_size, total_size):         total_kb = total_size / 1024.          downloaded_kb = (block_count * block_size) / 1024.         just_downloaded = downloaded_kb - self.last_downloaded_kb         self.last_downloaded_kb = downloaded_kb          predicted_size = block_size/1024.          wait_time = self.bucket.consume(predicted_size)         while wait_time > 0:             time.sleep(wait_time)             wait_time = self.bucket.consume(predicted_size)          now = time.time()         delta = now - self.last_update         if self.last_update != 0:             if delta > 0:                 rate = just_downloaded / delta                 if self.avg_rate is not None:                     rate = 0.9 * self.avg_rate + 0.1 * rate                 self.avg_rate = rate             else:                 rate = self.avg_rate or 0.             print '%20s: %4.1f%%, %5.1f KiB/s, %.1f/%.1f KiB' % (                     self.filename, 100. * downloaded_kb / total_kb,                     rate, downloaded_kb, total_kb,                 )         self.last_update = now   def main():     '''Fetch the contents of urls'''     if len(sys.argv) < 4:         print 'Syntax: %s rate url1 url2 ...' % sys.argv[0]         raise SystemExit(1)     rate_limit  = float(sys.argv[1])     urls = sys.argv[2:]     bucket = TokenBucket(10*rate_limit, rate_limit)      print 'rate limit = %.1f' % (rate_limit,)      threads = []     for url in urls:         path = urlparse.urlparse(url,'http')[2]         filename = os.path.basename(path)         print 'Downloading '%s' to '%s'...' % (url,filename)         rate_limiter = RateLimit(bucket, filename)         t = threading.Thread(             target=urllib.urlretrieve,             args=(url, filename, rate_limiter))         t.start()         threads.append(t)      for t in threads:         t.join()      print 'All downloads finished'  if __name__ == '__main__':     main()