Question

The following code does not have any errors when run script.py -f filename but does not function? I imagine part of it is my function/variable naming scheme which is probably causing some confusion on my part. Any ideas?

#! /usr/bin/env python

import boto
import os, sys, glob
from optparse import OptionParser
from boto.s3.key import Key

BUCKET_NAME = ''
AWS_ACCESS_KEY_ID = ''
AWS_SECRET_ACCESS_KEY = ''

# function to determine file argument
def fname(arguments):
    files = []
    for arg in arguments:
        if '*' in arg or '?' in arg:
            # contains a wildcard character
            files.extend(glob.glob(arg))
        elif os.path.isdir(arg):
            # is a dictionary
            files.extend(glob.glob(os.path.join(arg, '*')))
        elif os.path.exists(arg):
            # is a file
            files.append(arg)
        else:
            # invalid?
            print '%s invalid' % arg
    return files

# function to display progress tick marks
def percent_cb(complete, total):
    sys.stdout.write('.')
    sys.stdout.flush()

# upload files to bucket
def upload(all_files, args, bucket):
    all_files = fname(args)
    complete = ''
    total = ''
    percent_cb(complete, total)
    for filename in all_files:
        #skip all directory entries which are not a file
        if not os.path.isfile(filename):
              continue
        k = Key(bucket)
        k.set_contents_from_filename(filename, cb=percent_cb, num_cb=20)

# check if file exists locally, if not: download it
def downnload(filename, keyString):
    if not os.path.exists(filename+keyString):
        l.get_contents_to_filename(filename+keyString)

# List bucket contents
def blist(bucket):
    for b in rs:
        print b.name

def main():
    usage = "usage: %prog [options] -f arg"
    parser = OptionParser(usage)
    parser.add_option('-d', '--download',
            action='store', dest='download',
            default=None, help='download files')
    parser.add_option('-f', '--file',
            action='store', dest='filename',
            default=None, help='upload file or wildcard')
    parser.add_option('-l', '--list',
            action='store', dest='blist',
            default=None, help='list buckets or contents of specified bucket')
    parser.add_option('-v', '--version',
            action="store_true", dest="show_version",
            default=False, help='displays the version number')

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit()
    (options, args) = parser.parse_args()

    if options.show_version:
        prog = os.path.basename(sys.argv[0])
        version_str = "1.0"
        print "version is: %s %s" % (prog, version_str)
        sys.exit(0)

# connect to the bucket
    conn = boto.connect_s3(AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY)
    bucket = conn.get_bucket(BUCKET_NAME)
    rs = conn.get_all_buckets()

# bucket file list
    bucket_list = bucket.list()
    for l in bucket_list:
        keyString = str(l.key)

    all_files = ''
    if options.filename is not None:
    upload(all_files, args, bucket)
    elif options.download is not None:
       downnload(options.filename, keyString)
    elif options.blist is not None:
       blist(options.bucket)
    else:
       parser.print_help()
        sys.exit(-1)

if __name__ == '__main__':
    main()

Answer 1

I’m not sure about the way you’re accessing boto here. You’re only using S3, right? The boto S3 docs use a different interface than the one you’re using. Here are some excerpts from code that works under Python 2.5:

from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError

class TimeoutException(Exception):
    pass

...

conn = S3Connection(access_key, secret_key)
try:
    bucket = get_bucket(conn, bucket_name)
except TimeoutException:
    sys.exit("Connection timed out; this usually means you're offline.")
except S3ResponseError, exception_data:
    sys.exit(exception_data.error_message)

...

key_name = os.path.basename(fname)
if bucket.get_key(key_name):
    print 'Already on S3, will not overwrite: ' + key_name
    return
key = bucket.new_key(key_name)
key.set_contents_from_filename(fname)

...

def get_bucket(conn, bucket_name):
    # If you try to get a bucket while offline, the function just
    # hangs. This times it out after two seconds.

    def timeout_handler(signum, frame):
        raise TimeoutException()

    old_handler = signal.signal(signal.SIGALRM, timeout_handler)

    # start timer
    signal.alarm(2)
    try:
        bucket = conn.get_bucket(bucket_name)
    except TimeoutException:
        bucket = None
    finally:
        signal.signal(signal.SIGALRM, old_handler)

    signal.alarm(0)
    if bucket:
        return bucket
    else:
        raise TimeoutException()