Question

I am trying to use the Bing api in python with the following code:

#!/usr/bin/python
from bingapi import bingapi  
import re
import json
import urllib
import cgi
import cgitb
from HTMLParser import HTMLParser

class MLStripper(HTMLParser):
    def __init__(self):
            self.reset()
            self.fed = []
    def handle_data(self, d):
            self.fed.append(d)
    def get_data(self):
            return ''.join(self.fed)

def strip_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data()

def strip_tags2(data):
    p = re.compile(r'<[^<]*?>')
    q = re.compile(r'[&;!@#$%^*()]*')
    data = p.sub('', data)
    return q.sub('', data)

def getUrl(item):
    return item['Url']

def getContent(item):
    return item['Description']

def getTitle(item):
    return item['Title']

def getInfo(qry, siteStr):
    qryStr = qry + "+" + siteStr
    #qryStr = u"%s" % qryStr.encode('UTF-8')
    query = urllib.urlencode({'q' : qryStr})
    url = 'http://api.bing.net/json.aspx?Appid=<myappid>&Version=2.2&Market=en-US&Query=%s&Sources=web&Web.Count=10&JsonType=raw' % (query)
    search_results = urllib.urlopen(url)
    j = json.loads(search_results.read())
    results = j['SearchResponse']['Web']['Results']
    return results

def updateRecent(qry):
    f = open("recent.txt", "r")
    lines = f.readlines()
    f.close()
    lines = lines[1:]

    if len(qry) > 50: #truncate if string too long
            qry = (qry[:50] + '...')
    qry = strip_tags2(qry) #strip out the html if injection try

    lines.append("\n%s" % qry)
    f = open("recent.txt", "w")
    f.writelines(lines)
    f.close()

if __name__ == '__main__':
    form = cgi.FieldStorage()
    qry = form["qry"].value
    qry = r'%s' % qry

    updateRecent(qry)

    siteStr = "(site:answers.yahoo.com OR site:chacha.com OR site:blurtit.com OR site:answers.com OR site:question.com OR site:answerbag.com OR site:stackexchange.com)"

    print "Content-type: text/html"
    print

    header = open("header.html", "r")
    contents = header.readlines()
    header.close()
    for item in contents:
            print item

    print """
    <div id="results">
    <center><h1>Results:</h1></center>
    """
    for item in getInfo(siteStr, qry):
            print "<h3>%s</h3>" % getTitle(item)
            print "<br />"
            print "%s" % getUrl(item)
            print "<br />"
            print "<p style=\"color:gray\">%s</p>" % getContent(item)
            print "<br />"
    print "</div>"

    footer = open("footer.html", "r")
    contents = footer.readlines()
    footer.close()
    for thing in contents:
            print thing

I prints a few results, and then gives me the following error:

UnicodeEncodeError: 'ascii' codec can't encode character u'\\u2026' in position 72:    ordinal not in range(128)

Can someone explain why this is happening? It clearly has something to do with how the url is getting encoded, but what is exactly is wrong? Thanks in advance!

Answer 1

That particular Unicode character is “HORIZONTAL ELLIPSIS”. One or more of your getXXXXX() functions are returning Unicode strings, one of which contains a non-ASCII character. I suggest declaring the encoding of your output, for example:

Content-Type: text/html; charset=utf-8

and explicitly encoding your output in that encoding.