Question

I have a string that represents an html document. I’m trying to replace text in that document, excluding the markup and attribute values ofcourse with some replacement html. I thought it would be simple, but it is incredibly tedious when you want to replace the text with markup. For example, to replace somekeyword with <a href = "link">somekeyword</a>.

from lxml.html import fragments_fromstring, fromstring, tostring
from re import compile
def markup_aware_sub(pattern, repl, text):
    exp = compile(pattern)
    root = fromstring(text)

    els = [el for el in root.getiterator() if el.text]
    els = [el for el in els if el.text.strip()]
    for el in els:
        text = exp.sub(repl, el.text)
        if text == el.text:
            continue
        parent = el.getparent()
        new_el = fromstring(text)
        new_el.tag = el.tag
        for k, v in el.attrib.items():
            new_el.attrib[k] = v
        parent.replace(el, new_el)
    return tostring(root)

markup_aware_sub('keyword', '<a>blah</a>', '<div><p>Text with keyword here</p></div>')

It works but only if the keyword is exactly two “nestings” down. There has to be a better way to do it than the above, but after googling for many hours I can’t find anything.

Answer 1

This might be the solution you are lookin for:

from HTMLParser import HTMLParser

class MyParser(HTMLParser):
    def __init__(self,link, keyword):
    HTMLParser.__init__(self)
    self.__html = []
    self.link = link
    self.keyword = keyword

    def handle_data(self, data):
    text = data.strip()
    self.__html.append(text.replace(self.keyword,'<a href="'+self.link+'>'+self.keyword+'</a>'))

    def handle_starttag(self, tag, attrs):
    self.__html.append("<"+tag+">")

    def handle_endtag(self, tag):
    self.__html.append("</"+tag+">")

    def new_html(self):
    return ''.join(self.__html).strip()


parser = MyParser("blah","keyword")
parser.feed("<div><p>Text with keyword here</p></div>")
parser.close()
print parser.new_html()

This will give you the following output

<div><p>Text with <a href="blah>keyword</a> here</p></div>

The problem with your lxml approach only seems to occur when the keywords has only a single nesting. It seems to work fine with multiple nestings. So I added an if condition to catch this exception.

from lxml.html import fragments_fromstring, fromstring, tostring
from re import compile
def markup_aware_sub(pattern, repl, text):
    exp = compile(pattern)
    root = fromstring(text)
    els = [el for el in root.getiterator() if el.text]
    els = [el for el in els if el.text.strip()]

    if len(els) == 1:
      el = els[0]
      text = exp.sub(repl, el.text)
      parent = el.getparent()
      new_el = fromstring(text)
      new_el.tag = el.tag
      for k, v in el.attrib.items():
          new_el.attrib[k] = v
      return tostring(new_el)

    for el in els:
      text = exp.sub(repl, el.text)
      if text == el.text:
        continue
      parent = el.getparent()
      new_el = fromstring(text)
      new_el.tag = el.tag
      for k, v in el.attrib.items():
          new_el.attrib[k] = v
      parent.replace(el, new_el)
    return tostring(root)

print markup_aware_sub('keyword', '<a>blah</a>', '<p>Text with keyword here</p>')

Not very elegant, but seems to work. Please check it out.