_html5lib.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338
  1. __all__ = [
  2. 'HTML5TreeBuilder',
  3. ]
  4. import warnings
  5. from bs4.builder import (
  6. PERMISSIVE,
  7. HTML,
  8. HTML_5,
  9. HTMLTreeBuilder,
  10. )
  11. from bs4.element import (
  12. NamespacedAttribute,
  13. whitespace_re,
  14. )
  15. import html5lib
  16. try:
  17. # html5lib >= 0.99999999/1.0b9
  18. from html5lib.treebuilders import base as treebuildersbase
  19. except ImportError:
  20. # html5lib <= 0.9999999/1.0b8
  21. from html5lib.treebuilders import _base as treebuildersbase
  22. from html5lib.constants import namespaces
  23. from bs4.element import (
  24. Comment,
  25. Doctype,
  26. NavigableString,
  27. Tag,
  28. )
  29. class HTML5TreeBuilder(HTMLTreeBuilder):
  30. """Use html5lib to build a tree."""
  31. NAME = "html5lib"
  32. features = [NAME, PERMISSIVE, HTML_5, HTML]
  33. def prepare_markup(self, markup, user_specified_encoding,
  34. document_declared_encoding=None, exclude_encodings=None):
  35. # Store the user-specified encoding for use later on.
  36. self.user_specified_encoding = user_specified_encoding
  37. # document_declared_encoding and exclude_encodings aren't used
  38. # ATM because the html5lib TreeBuilder doesn't use
  39. # UnicodeDammit.
  40. if exclude_encodings:
  41. warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
  42. yield (markup, None, None, False)
  43. # These methods are defined by Beautiful Soup.
  44. def feed(self, markup):
  45. if self.soup.parse_only is not None:
  46. warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
  47. parser = html5lib.HTMLParser(tree=self.create_treebuilder)
  48. doc = parser.parse(markup, encoding=self.user_specified_encoding)
  49. # Set the character encoding detected by the tokenizer.
  50. if isinstance(markup, str):
  51. # We need to special-case this because html5lib sets
  52. # charEncoding to UTF-8 if it gets Unicode input.
  53. doc.original_encoding = None
  54. else:
  55. doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
  56. def create_treebuilder(self, namespaceHTMLElements):
  57. self.underlying_builder = TreeBuilderForHtml5lib(
  58. self.soup, namespaceHTMLElements)
  59. return self.underlying_builder
  60. def test_fragment_to_document(self, fragment):
  61. """See `TreeBuilder`."""
  62. return '<html><head></head><body>%s</body></html>' % fragment
  63. class TreeBuilderForHtml5lib(treebuildersbase.TreeBuilder):
  64. def __init__(self, soup, namespaceHTMLElements):
  65. self.soup = soup
  66. super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
  67. def documentClass(self):
  68. self.soup.reset()
  69. return Element(self.soup, self.soup, None)
  70. def insertDoctype(self, token):
  71. name = token["name"]
  72. publicId = token["publicId"]
  73. systemId = token["systemId"]
  74. doctype = Doctype.for_name_and_ids(name, publicId, systemId)
  75. self.soup.object_was_parsed(doctype)
  76. def elementClass(self, name, namespace):
  77. tag = self.soup.new_tag(name, namespace)
  78. return Element(tag, self.soup, namespace)
  79. def commentClass(self, data):
  80. return TextNode(Comment(data), self.soup)
  81. def fragmentClass(self):
  82. self.soup = BeautifulSoup("")
  83. self.soup.name = "[document_fragment]"
  84. return Element(self.soup, self.soup, None)
  85. def appendChild(self, node):
  86. # XXX This code is not covered by the BS4 tests.
  87. self.soup.append(node.element)
  88. def getDocument(self):
  89. return self.soup
  90. def getFragment(self):
  91. return treebuildersbase.TreeBuilder.getFragment(self).element
  92. class AttrList(object):
  93. def __init__(self, element):
  94. self.element = element
  95. self.attrs = dict(self.element.attrs)
  96. def __iter__(self):
  97. return list(self.attrs.items()).__iter__()
  98. def __setitem__(self, name, value):
  99. # If this attribute is a multi-valued attribute for this element,
  100. # turn its value into a list.
  101. list_attr = HTML5TreeBuilder.cdata_list_attributes
  102. if (name in list_attr['*']
  103. or (self.element.name in list_attr
  104. and name in list_attr[self.element.name])):
  105. # A node that is being cloned may have already undergone
  106. # this procedure.
  107. if not isinstance(value, list):
  108. value = whitespace_re.split(value)
  109. self.element[name] = value
  110. def items(self):
  111. return list(self.attrs.items())
  112. def keys(self):
  113. return list(self.attrs.keys())
  114. def __len__(self):
  115. return len(self.attrs)
  116. def __getitem__(self, name):
  117. return self.attrs[name]
  118. def __contains__(self, name):
  119. return name in list(self.attrs.keys())
  120. class Element(treebuildersbase.Node):
  121. def __init__(self, element, soup, namespace):
  122. treebuildersbase.Node.__init__(self, element.name)
  123. self.element = element
  124. self.soup = soup
  125. self.namespace = namespace
  126. def appendChild(self, node):
  127. string_child = child = None
  128. if isinstance(node, str):
  129. # Some other piece of code decided to pass in a string
  130. # instead of creating a TextElement object to contain the
  131. # string.
  132. string_child = child = node
  133. elif isinstance(node, Tag):
  134. # Some other piece of code decided to pass in a Tag
  135. # instead of creating an Element object to contain the
  136. # Tag.
  137. child = node
  138. elif node.element.__class__ == NavigableString:
  139. string_child = child = node.element
  140. else:
  141. child = node.element
  142. if not isinstance(child, str) and child.parent is not None:
  143. node.element.extract()
  144. if (string_child and self.element.contents
  145. and self.element.contents[-1].__class__ == NavigableString):
  146. # We are appending a string onto another string.
  147. # TODO This has O(n^2) performance, for input like
  148. # "a</a>a</a>a</a>..."
  149. old_element = self.element.contents[-1]
  150. new_element = self.soup.new_string(old_element + string_child)
  151. old_element.replace_with(new_element)
  152. self.soup._most_recent_element = new_element
  153. else:
  154. if isinstance(node, str):
  155. # Create a brand new NavigableString from this string.
  156. child = self.soup.new_string(node)
  157. # Tell Beautiful Soup to act as if it parsed this element
  158. # immediately after the parent's last descendant. (Or
  159. # immediately after the parent, if it has no children.)
  160. if self.element.contents:
  161. most_recent_element = self.element._last_descendant(False)
  162. elif self.element.next_element is not None:
  163. # Something from further ahead in the parse tree is
  164. # being inserted into this earlier element. This is
  165. # very annoying because it means an expensive search
  166. # for the last element in the tree.
  167. most_recent_element = self.soup._last_descendant()
  168. else:
  169. most_recent_element = self.element
  170. self.soup.object_was_parsed(
  171. child, parent=self.element,
  172. most_recent_element=most_recent_element)
  173. def getAttributes(self):
  174. return AttrList(self.element)
  175. def setAttributes(self, attributes):
  176. if attributes is not None and len(attributes) > 0:
  177. converted_attributes = []
  178. for name, value in list(attributes.items()):
  179. if isinstance(name, tuple):
  180. new_name = NamespacedAttribute(*name)
  181. del attributes[name]
  182. attributes[new_name] = value
  183. self.soup.builder._replace_cdata_list_attribute_values(
  184. self.name, attributes)
  185. for name, value in list(attributes.items()):
  186. self.element[name] = value
  187. # The attributes may contain variables that need substitution.
  188. # Call set_up_substitutions manually.
  189. #
  190. # The Tag constructor called this method when the Tag was created,
  191. # but we just set/changed the attributes, so call it again.
  192. self.soup.builder.set_up_substitutions(self.element)
  193. attributes = property(getAttributes, setAttributes)
  194. def insertText(self, data, insertBefore=None):
  195. if insertBefore:
  196. text = TextNode(self.soup.new_string(data), self.soup)
  197. self.insertBefore(data, insertBefore)
  198. else:
  199. self.appendChild(data)
  200. def insertBefore(self, node, refNode):
  201. index = self.element.index(refNode.element)
  202. if (node.element.__class__ == NavigableString and self.element.contents
  203. and self.element.contents[index-1].__class__ == NavigableString):
  204. # (See comments in appendChild)
  205. old_node = self.element.contents[index-1]
  206. new_str = self.soup.new_string(old_node + node.element)
  207. old_node.replace_with(new_str)
  208. else:
  209. self.element.insert(index, node.element)
  210. node.parent = self
  211. def removeChild(self, node):
  212. node.element.extract()
  213. def reparentChildren(self, new_parent):
  214. """Move all of this tag's children into another tag."""
  215. # print "MOVE", self.element.contents
  216. # print "FROM", self.element
  217. # print "TO", new_parent.element
  218. element = self.element
  219. new_parent_element = new_parent.element
  220. # Determine what this tag's next_element will be once all the children
  221. # are removed.
  222. final_next_element = element.next_sibling
  223. new_parents_last_descendant = new_parent_element._last_descendant(False, False)
  224. if len(new_parent_element.contents) > 0:
  225. # The new parent already contains children. We will be
  226. # appending this tag's children to the end.
  227. new_parents_last_child = new_parent_element.contents[-1]
  228. new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
  229. else:
  230. # The new parent contains no children.
  231. new_parents_last_child = None
  232. new_parents_last_descendant_next_element = new_parent_element.next_element
  233. to_append = element.contents
  234. append_after = new_parent_element.contents
  235. if len(to_append) > 0:
  236. # Set the first child's previous_element and previous_sibling
  237. # to elements within the new parent
  238. first_child = to_append[0]
  239. if new_parents_last_descendant:
  240. first_child.previous_element = new_parents_last_descendant
  241. else:
  242. first_child.previous_element = new_parent_element
  243. first_child.previous_sibling = new_parents_last_child
  244. if new_parents_last_descendant:
  245. new_parents_last_descendant.next_element = first_child
  246. else:
  247. new_parent_element.next_element = first_child
  248. if new_parents_last_child:
  249. new_parents_last_child.next_sibling = first_child
  250. # Fix the last child's next_element and next_sibling
  251. last_child = to_append[-1]
  252. last_child.next_element = new_parents_last_descendant_next_element
  253. if new_parents_last_descendant_next_element:
  254. new_parents_last_descendant_next_element.previous_element = last_child
  255. last_child.next_sibling = None
  256. for child in to_append:
  257. child.parent = new_parent_element
  258. new_parent_element.contents.append(child)
  259. # Now that this element has no children, change its .next_element.
  260. element.contents = []
  261. element.next_element = final_next_element
  262. # print "DONE WITH MOVE"
  263. # print "FROM", self.element
  264. # print "TO", new_parent_element
  265. def cloneNode(self):
  266. tag = self.soup.new_tag(self.element.name, self.namespace)
  267. node = Element(tag, self.soup, self.namespace)
  268. for key,value in self.attributes:
  269. node.attributes[key] = value
  270. return node
  271. def hasContent(self):
  272. return self.element.contents
  273. def getNameTuple(self):
  274. if self.namespace is None:
  275. return namespaces["html"], self.name
  276. else:
  277. return self.namespace, self.name
  278. nameTuple = property(getNameTuple)
  279. class TextNode(Element):
  280. def __init__(self, element, soup):
  281. treebuildersbase.Node.__init__(self, None)
  282. self.element = element
  283. self.soup = soup
  284. def cloneNode(self):
  285. raise NotImplementedError