Question

I want to use following codes to replace strings like “/xxxxx/” with “/xxxxx.html” in the page_data, but doesn’t work. page_data is bytes type which is downloaded by a crawler.

page_data.replace(each, neweach)

Only when I change them to:

page_data = page_data.replace(each, neweach)

the strings(each) in page_data are actually replaceed.

The whole code is below:

import os
import sys
import re
import urllib
import urllib2

class WebGet(object):
    base_url = ""
    urls_list = []
    history_list = []
    replace_ch={}

    def __init__(self, base_url):
        self.base_url = base_url[:-1]
        self.urls_list.append('/')
        self.replace_ch[">>"] = "%3E%3E"
        self.replace_ch["<<"] = "%3C%3C"
        self.replace_ch["::"] = "%3A%3A"

    def recurseGet(self):
        '''Get page data recursively'''
        while(len(self.urls_list) != 0):
            url_suffix = self.urls_list[0]
            self.urls_list.remove(url_suffix)
            self.history_list.append(url_suffix)
            url_to_get = self.base_url + url_suffix

            "Get page data with url"
            print "To get",url_to_get
            page_data = urllib2.urlopen(url_to_get).read()
            page_data_done = self.pageHandle(page_data)

            "Write the page data into file"
            if url_suffix[-1] == '/':
                url_suffix = url_suffix[:-1]
            if url_suffix == '':
                url_suffix = "index"
            elif url_suffix[0] == '/':
                url_suffix = url_suffix[1:]
            url_suffix.replace('/','\\')
            url_suffix.replace('>>','%3E%3E')
            url_suffix.replace('<<','%3C%3C')
            url_suffix.replace('::','%3A%3A')
            file_str = "e:\\reference\\"+url_suffix
            if file_str.rfind("\\") != 12:
                new_dir = file_str[:file_str.rfind("\\")]
                if os.path.isdir(file_str) == False:
                    os.mkdir(file_str)
            file_str = file_str.strip()+".html"
            print "write file",file_str
            f_page = open(file_str, "wb")
            f_page.write(page_data_done)
            f_page.close


    def pageHandle(self, page_data):
        page_data.replace("http://www.cplusplus.com/","/") #here the replace works

        re_rule = '<a href="/reference(/\S{2,40}/)\">'
        list_page_urls = re.findall(re_rule, page_data)
        for each in list_page_urls:
            neweach = each
            neweach = neweach[:-1]+".html"
            #page_data = page_data.replace(each, neweach)
            page_data.replace(each, neweach)
            if each in page_data:
                print "fail replace"
            if each in self.history_list:
                continue
            elif each in self.urls_list:
                continue
            elif each == '/':
                continue                
            self.urls_list.append(each)

        return page_data

def main():
    url = "http://www.cplusplus.com/reference/"
    fc = WebGet(url)
    fc.recurseGet()

if __name__ == "__main__":
    main()

Why could be this?

Answer 1

Because that’s what the replace method does: returns a copy of the string with the relevant characters replaced.

Apart from anything else, strings are immutable in Python, so it couldn’t work any other way.