123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468 |
- """Beautiful Soup
- Elixir and Tonic
- "The Screen-Scraper's Friend"
- http://www.crummy.com/software/BeautifulSoup/
- Beautiful Soup uses a pluggable XML or HTML parser to parse a
- (possibly invalid) document into a tree representation. Beautiful Soup
- provides provides methods and Pythonic idioms that make it easy to
- navigate, search, and modify the parse tree.
- Beautiful Soup works with Python 2.6 and up. It works better if lxml
- and/or html5lib is installed.
- For more than you ever wanted to know about Beautiful Soup, see the
- documentation:
- http://www.crummy.com/software/BeautifulSoup/bs4/doc/
- """
- __author__ = "Leonard Richardson (leonardr@segfault.org)"
- __version__ = "4.4.1"
- __copyright__ = "Copyright (c) 2004-2015 Leonard Richardson"
- __license__ = "MIT"
- __all__ = ['BeautifulSoup']
- import os
- import re
- import warnings
- from .builder import builder_registry, ParserRejectedMarkup
- from .dammit import UnicodeDammit
- from .element import (
- CData,
- Comment,
- DEFAULT_OUTPUT_ENCODING,
- Declaration,
- Doctype,
- NavigableString,
- PageElement,
- ProcessingInstruction,
- ResultSet,
- SoupStrainer,
- Tag,
- )
- # The very first thing we do is give a useful error if someone is
- # running this code under Python 3 without converting it.
- 'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'!='You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
- class BeautifulSoup(Tag):
- """
- This class defines the basic interface called by the tree builders.
- These methods will be called by the parser:
- reset()
- feed(markup)
- The tree builder may call these methods from its feed() implementation:
- handle_starttag(name, attrs) # See note about return value
- handle_endtag(name)
- handle_data(data) # Appends to the current data node
- endData(containerClass=NavigableString) # Ends the current data node
- No matter how complicated the underlying parser is, you should be
- able to build a tree using 'start tag' events, 'end tag' events,
- 'data' events, and "done with data" events.
- If you encounter an empty-element tag (aka a self-closing tag,
- like HTML's <br> tag), call handle_starttag and then
- handle_endtag.
- """
- ROOT_TAG_NAME = '[document]'
- # If the end-user gives no indication which tree builder they
- # want, look for one with these features.
- DEFAULT_BUILDER_FEATURES = ['html', 'fast']
- ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
- NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nTo get rid of this warning, change this:\n\n BeautifulSoup([your markup])\n\nto this:\n\n BeautifulSoup([your markup], \"%(parser)s\")\n"
- def __init__(self, markup="", features=None, builder=None,
- parse_only=None, from_encoding=None, exclude_encodings=None,
- **kwargs):
- """The Soup object is initialized as the 'root tag', and the
- provided markup (which can be a string or a file-like object)
- is fed into the underlying parser."""
- if 'convertEntities' in kwargs:
- warnings.warn(
- "BS4 does not respect the convertEntities argument to the "
- "BeautifulSoup constructor. Entities are always converted "
- "to Unicode characters.")
- if 'markupMassage' in kwargs:
- del kwargs['markupMassage']
- warnings.warn(
- "BS4 does not respect the markupMassage argument to the "
- "BeautifulSoup constructor. The tree builder is responsible "
- "for any necessary markup massage.")
- if 'smartQuotesTo' in kwargs:
- del kwargs['smartQuotesTo']
- warnings.warn(
- "BS4 does not respect the smartQuotesTo argument to the "
- "BeautifulSoup constructor. Smart quotes are always converted "
- "to Unicode characters.")
- if 'selfClosingTags' in kwargs:
- del kwargs['selfClosingTags']
- warnings.warn(
- "BS4 does not respect the selfClosingTags argument to the "
- "BeautifulSoup constructor. The tree builder is responsible "
- "for understanding self-closing tags.")
- if 'isHTML' in kwargs:
- del kwargs['isHTML']
- warnings.warn(
- "BS4 does not respect the isHTML argument to the "
- "BeautifulSoup constructor. Suggest you use "
- "features='lxml' for HTML and features='lxml-xml' for "
- "XML.")
- def deprecated_argument(old_name, new_name):
- if old_name in kwargs:
- warnings.warn(
- 'The "%s" argument to the BeautifulSoup constructor '
- 'has been renamed to "%s."' % (old_name, new_name))
- value = kwargs[old_name]
- del kwargs[old_name]
- return value
- return None
- parse_only = parse_only or deprecated_argument(
- "parseOnlyThese", "parse_only")
- from_encoding = from_encoding or deprecated_argument(
- "fromEncoding", "from_encoding")
- if len(kwargs) > 0:
- arg = list(kwargs.keys()).pop()
- raise TypeError(
- "__init__() got an unexpected keyword argument '%s'" % arg)
- if builder is None:
- original_features = features
- if isinstance(features, str):
- features = [features]
- if features is None or len(features) == 0:
- features = self.DEFAULT_BUILDER_FEATURES
- builder_class = builder_registry.lookup(*features)
- if builder_class is None:
- raise FeatureNotFound(
- "Couldn't find a tree builder with the features you "
- "requested: %s. Do you need to install a parser library?"
- % ",".join(features))
- builder = builder_class()
- if not (original_features == builder.NAME or
- original_features in builder.ALTERNATE_NAMES):
- if builder.is_xml:
- markup_type = "XML"
- else:
- markup_type = "HTML"
- warnings.warn(self.NO_PARSER_SPECIFIED_WARNING % dict(
- parser=builder.NAME,
- markup_type=markup_type))
- self.builder = builder
- self.is_xml = builder.is_xml
- self.builder.soup = self
- self.parse_only = parse_only
- if hasattr(markup, 'read'): # It's a file-type object.
- markup = markup.read()
- elif len(markup) <= 256:
- # Print out warnings for a couple beginner problems
- # involving passing non-markup to Beautiful Soup.
- # Beautiful Soup will still parse the input as markup,
- # just in case that's what the user really wants.
- if (isinstance(markup, str)
- and not os.path.supports_unicode_filenames):
- possible_filename = markup.encode("utf8")
- else:
- possible_filename = markup
- is_file = False
- try:
- is_file = os.path.exists(possible_filename)
- except Exception as e:
- # This is almost certainly a problem involving
- # characters not valid in filenames on this
- # system. Just let it go.
- pass
- if is_file:
- if isinstance(markup, str):
- markup = markup.encode("utf8")
- warnings.warn(
- '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup)
- if markup[:5] == "http:" or markup[:6] == "https:":
- # TODO: This is ugly but I couldn't get it to work in
- # Python 3 otherwise.
- if ((isinstance(markup, bytes) and not b' ' in markup)
- or (isinstance(markup, str) and not ' ' in markup)):
- if isinstance(markup, str):
- markup = markup.encode("utf8")
- warnings.warn(
- '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)
- for (self.markup, self.original_encoding, self.declared_html_encoding,
- self.contains_replacement_characters) in (
- self.builder.prepare_markup(
- markup, from_encoding, exclude_encodings=exclude_encodings)):
- self.reset()
- try:
- self._feed()
- break
- except ParserRejectedMarkup:
- pass
- # Clear out the markup and remove the builder's circular
- # reference to this object.
- self.markup = None
- self.builder.soup = None
- def __copy__(self):
- return type(self)(self.encode(), builder=self.builder)
- def __getstate__(self):
- # Frequently a tree builder can't be pickled.
- d = dict(self.__dict__)
- if 'builder' in d and not self.builder.picklable:
- del d['builder']
- return d
- def _feed(self):
- # Convert the document to Unicode.
- self.builder.reset()
- self.builder.feed(self.markup)
- # Close out any unfinished strings and close all the open tags.
- self.endData()
- while self.currentTag.name != self.ROOT_TAG_NAME:
- self.popTag()
- def reset(self):
- Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
- self.hidden = 1
- self.builder.reset()
- self.current_data = []
- self.currentTag = None
- self.tagStack = []
- self.preserve_whitespace_tag_stack = []
- self.pushTag(self)
- def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
- """Create a new tag associated with this soup."""
- return Tag(None, self.builder, name, namespace, nsprefix, attrs)
- def new_string(self, s, subclass=NavigableString):
- """Create a new NavigableString associated with this soup."""
- return subclass(s)
- def insert_before(self, successor):
- raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
- def insert_after(self, successor):
- raise NotImplementedError("BeautifulSoup objects don't support insert_after().")
- def popTag(self):
- tag = self.tagStack.pop()
- if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]:
- self.preserve_whitespace_tag_stack.pop()
- #print "Pop", tag.name
- if self.tagStack:
- self.currentTag = self.tagStack[-1]
- return self.currentTag
- def pushTag(self, tag):
- #print "Push", tag.name
- if self.currentTag:
- self.currentTag.contents.append(tag)
- self.tagStack.append(tag)
- self.currentTag = self.tagStack[-1]
- if tag.name in self.builder.preserve_whitespace_tags:
- self.preserve_whitespace_tag_stack.append(tag)
- def endData(self, containerClass=NavigableString):
- if self.current_data:
- current_data = ''.join(self.current_data)
- # If whitespace is not preserved, and this string contains
- # nothing but ASCII spaces, replace it with a single space
- # or newline.
- if not self.preserve_whitespace_tag_stack:
- strippable = True
- for i in current_data:
- if i not in self.ASCII_SPACES:
- strippable = False
- break
- if strippable:
- if '\n' in current_data:
- current_data = '\n'
- else:
- current_data = ' '
- # Reset the data collector.
- self.current_data = []
- # Should we add this string to the tree at all?
- if self.parse_only and len(self.tagStack) <= 1 and \
- (not self.parse_only.text or \
- not self.parse_only.search(current_data)):
- return
- o = containerClass(current_data)
- self.object_was_parsed(o)
- def object_was_parsed(self, o, parent=None, most_recent_element=None):
- """Add an object to the parse tree."""
- parent = parent or self.currentTag
- previous_element = most_recent_element or self._most_recent_element
- next_element = previous_sibling = next_sibling = None
- if isinstance(o, Tag):
- next_element = o.next_element
- next_sibling = o.next_sibling
- previous_sibling = o.previous_sibling
- if not previous_element:
- previous_element = o.previous_element
- o.setup(parent, previous_element, next_element, previous_sibling, next_sibling)
- self._most_recent_element = o
- parent.contents.append(o)
- if parent.next_sibling:
- # This node is being inserted into an element that has
- # already been parsed. Deal with any dangling references.
- index = parent.contents.index(o)
- if index == 0:
- previous_element = parent
- previous_sibling = None
- else:
- previous_element = previous_sibling = parent.contents[index-1]
- if index == len(parent.contents)-1:
- next_element = parent.next_sibling
- next_sibling = None
- else:
- next_element = next_sibling = parent.contents[index+1]
- o.previous_element = previous_element
- if previous_element:
- previous_element.next_element = o
- o.next_element = next_element
- if next_element:
- next_element.previous_element = o
- o.next_sibling = next_sibling
- if next_sibling:
- next_sibling.previous_sibling = o
- o.previous_sibling = previous_sibling
- if previous_sibling:
- previous_sibling.next_sibling = o
- def _popToTag(self, name, nsprefix=None, inclusivePop=True):
- """Pops the tag stack up to and including the most recent
- instance of the given tag. If inclusivePop is false, pops the tag
- stack up to but *not* including the most recent instqance of
- the given tag."""
- #print "Popping to %s" % name
- if name == self.ROOT_TAG_NAME:
- # The BeautifulSoup object itself can never be popped.
- return
- most_recently_popped = None
- stack_size = len(self.tagStack)
- for i in range(stack_size - 1, 0, -1):
- t = self.tagStack[i]
- if (name == t.name and nsprefix == t.prefix):
- if inclusivePop:
- most_recently_popped = self.popTag()
- break
- most_recently_popped = self.popTag()
- return most_recently_popped
- def handle_starttag(self, name, namespace, nsprefix, attrs):
- """Push a start tag on to the stack.
- If this method returns None, the tag was rejected by the
- SoupStrainer. You should proceed as if the tag had not occured
- in the document. For instance, if this was a self-closing tag,
- don't call handle_endtag.
- """
- # print "Start tag %s: %s" % (name, attrs)
- self.endData()
- if (self.parse_only and len(self.tagStack) <= 1
- and (self.parse_only.text
- or not self.parse_only.search_tag(name, attrs))):
- return None
- tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
- self.currentTag, self._most_recent_element)
- if tag is None:
- return tag
- if self._most_recent_element:
- self._most_recent_element.next_element = tag
- self._most_recent_element = tag
- self.pushTag(tag)
- return tag
- def handle_endtag(self, name, nsprefix=None):
- #print "End tag: " + name
- self.endData()
- self._popToTag(name, nsprefix)
- def handle_data(self, data):
- self.current_data.append(data)
- def decode(self, pretty_print=False,
- eventual_encoding=DEFAULT_OUTPUT_ENCODING,
- formatter="minimal"):
- """Returns a string or Unicode representation of this document.
- To get Unicode, pass None for encoding."""
- if self.is_xml:
- # Print the XML declaration
- encoding_part = ''
- if eventual_encoding is not None:
- encoding_part = ' encoding="%s"' % eventual_encoding
- prefix = '<?xml version="1.0"%s?>\n' % encoding_part
- else:
- prefix = ''
- if not pretty_print:
- indent_level = None
- else:
- indent_level = 0
- return prefix + super(BeautifulSoup, self).decode(
- indent_level, eventual_encoding, formatter)
- # Alias to make it easier to type import: 'from bs4 import _soup'
- _s = BeautifulSoup
- _soup = BeautifulSoup
- class BeautifulStoneSoup(BeautifulSoup):
- """Deprecated interface to an XML parser."""
- def __init__(self, *args, **kwargs):
- kwargs['features'] = 'xml'
- warnings.warn(
- 'The BeautifulStoneSoup class is deprecated. Instead of using '
- 'it, pass features="xml" into the BeautifulSoup constructor.')
- super(BeautifulStoneSoup, self).__init__(*args, **kwargs)
- class StopParsing(Exception):
- pass
- class FeatureNotFound(ValueError):
- pass
- #By default, act as an HTML pretty-printer.
- if __name__ == '__main__':
- import sys
- soup = BeautifulSoup(sys.stdin)
- print(soup.prettify())
|