Source code for linkcheck.parser.sitemap

# Copyright (C) 2000-2014 Bastian Kleineidam
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Main functions for link parsing
from xml.parsers.expat import ParserCreate
from xml.parsers.expat import ExpatError
from ..checker.const import WARN_XML_PARSE_ERROR

[docs]class XmlTagUrlParser: """Parse XML files and find URLs in text content of a tag name.""" def __init__(self, tag): """Initialize the parser.""" self.tag = tag self.parser = ParserCreate() self.parser.buffer_text = True self.parser.StartElementHandler = self.start_element self.parser.EndElementHandler = self.end_element self.parser.CharacterDataHandler = self.char_data
[docs] def parse(self, url_data): """Parse XML URL data.""" self.url_data = url_data self.in_tag = False self.url = "" data = url_data.get_raw_content() isfinal = True try: self.parser.Parse(data, isfinal) except ExpatError as expaterr: self.url_data.add_warning(expaterr.message, tag=WARN_XML_PARSE_ERROR)
[docs] def start_element(self, name, attrs): """Set tag status for start element.""" self.in_tag = name == self.tag self.url = ""
[docs] def end_element(self, name): """If end tag is our tag, call add_url().""" self.in_tag = False if name == self.tag: self.add_url()
[docs] def add_url(self): """Add non-empty URLs to the queue.""" if self.url: self.url_data.add_url( self.url, line=self.parser.CurrentLineNumber, column=self.parser.CurrentColumnNumber, ) self.url = ""
[docs] def char_data(self, data): """If inside the wanted tag, append data to URL.""" if self.in_tag: self.url += data
[docs]def parse_sitemap(url_data): """Parse XML sitemap data.""" XmlTagUrlParser("loc").parse(url_data)
[docs]def parse_sitemapindex(url_data): """Parse XML sitemap index data.""" XmlTagUrlParser("loc").parse(url_data)