Source code for linkcheck.checker.fileurl

# Copyright (C) 2000-2014 Bastian Kleineidam
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Handle local file: links.

import re
import os
import urllib.parse
import urllib.request
from datetime import datetime

from . import urlbase, get_index_html
from .. import log, LOG_CHECK, fileutil, mimeutil, LinkCheckerError, url as urlutil
from ..bookmarks import firefox

[docs]def get_files(dirname): """Get iterator of entries in directory. Only allows regular files and directories, no symlinks.""" for entry in os.listdir(dirname): fullentry = os.path.join(dirname, entry) if os.path.islink(fullentry): continue if os.path.isfile(fullentry): yield entry elif os.path.isdir(fullentry): yield entry + "/"
[docs]def prepare_urlpath_for_nt(path): """ URLs like *file://server/path/* result in a path named */server/path*. However urllib.url2pathname expects *////server/path*. """ if '|' not in path: return "////" + path.lstrip("/") return path
[docs]def get_nt_filename(path): """Return case sensitive filename for NT path.""" unc, rest = os.path.splitdrive(path) head, tail = os.path.split(rest) if not tail: return path for fname in os.listdir(unc + head): if fname.lower() == tail.lower(): return os.path.join(get_nt_filename(unc + head), fname) log.error(LOG_CHECK, "could not find %r in %r", tail, head) return path
[docs]def get_os_filename(path): """Return filesystem path for given URL path.""" if == 'nt': path = prepare_urlpath_for_nt(path) res = urllib.request.url2pathname(fileutil.path_safe(path)) if == 'nt' and res.endswith(':') and len(res) == 2: # Work around res += os.sep return res
[docs]def is_absolute_path(path): """Check if given path is absolute. On Windows absolute paths start with a drive letter. On all other systems absolute paths start with a slash.""" if == 'nt': if"^[a-zA-Z]:", path): return True path = path.replace("\\", "/") return path.startswith("/")
[docs]class FileUrl(urlbase.UrlBase): """ Url link with file scheme. """
[docs] def init( self, base_ref, base_url, parent_url, recursion_level, aggregate, line, column, page, name, url_encoding, extern, ): """Initialize the scheme.""" super().init( base_ref, base_url, parent_url, recursion_level, aggregate, line, column, page, name, url_encoding, extern, ) self.scheme = 'file'
[docs] def build_base_url(self): """The URL is normed according to the platform: - the base URL is made an absolute *file://* URL - under Windows platform the drive specifier is normed """ if self.base_url is None: return base_url = self.base_url if not (self.parent_url or self.base_ref or base_url.startswith("file:")): base_url = os.path.expanduser(base_url) if not is_absolute_path(base_url): try: base_url = os.getcwd() + "/" + base_url except OSError as msg: # occurs on stale remote filesystems (eg. NFS) errmsg = _( "Could not get current working directory: %(msg)s" ) % dict(msg=msg) raise LinkCheckerError(errmsg) if os.path.isdir(base_url): base_url += "/" base_url = "file://" + base_url if == "nt": base_url = base_url.replace("\\", "/") # transform c:/windows into /c|/windows base_url = re.sub("^file://(/?)([a-zA-Z]):", r"file:///\2|", base_url) # transform file://path into file:///path base_url = re.sub("^file://([^/])", r"file:///\1", base_url) self.base_url = base_url
[docs] def build_url(self): """ Calls super.build_url() and adds a trailing slash to directories. """ self.build_base_url() if self.parent_url is not None: # URL joining with the parent URL only works if the query # of the base URL are removed first. # Otherwise the join function thinks the query is part of # the file name. from .urlbase import url_norm # norm base url - can raise UnicodeError from url.idna_encode() base_url, is_idn = url_norm(self.base_url, self.encoding) urlparts = list(urllib.parse.urlsplit(base_url)) # ignore query part for filesystem urls urlparts[3] = '' self.base_url = urlutil.urlunsplit(urlparts) super().build_url() # ignore query and fragment url parts for filesystem urls self.urlparts[3] = self.urlparts[4] = '' if self.is_directory() and not self.urlparts[2].endswith('/'): self.add_warning( _("Added trailing slash to directory."), tag=WARN_FILE_MISSING_SLASH ) self.urlparts[2] += '/' self.url = urlutil.urlunsplit(self.urlparts)
[docs] def add_size_info(self): """Get size of file content and modification time from filename path.""" if self.is_directory(): # Directory size always differs from the customer index.html # that is generated. So return without calculating any size. return filename = self.get_os_filename() self.size = fileutil.get_size(filename) self.modified = datetime.utcfromtimestamp(fileutil.get_mtime(filename))
[docs] def check_connection(self): """ Try to open the local file. Under NT systems the case sensitivity is checked. """ if self.parent_url is not None and not self.parent_url.startswith("file:"): msg = _( "local files are only checked without parent URL or when" " the parent URL is also a file" ) raise LinkCheckerError(msg) if self.is_directory(): self.set_result(_("directory")) else: url = fileutil.path_safe(self.url) self.url_connection = urllib.request.urlopen(url) self.check_case_sensitivity()
[docs] def check_case_sensitivity(self): """ Check if url and windows path name match cases else there might be problems when copying such files on web servers that are case sensitive. """ if != 'nt': return path = self.get_os_filename() realpath = get_nt_filename(path) if path != realpath: self.add_warning( _( "The URL path %(path)r is not the same as the" " system path %(realpath)r. You should always use" " the system path in URLs." ) % {"path": path, "realpath": realpath}, tag=WARN_FILE_SYSTEM_PATH, )
[docs] def read_content(self): """Return file content, or in case of directories a dummy HTML file with links to the files.""" if self.is_directory(): data = get_index_html(get_files(self.get_os_filename())) data = data.encode("iso8859-1", "ignore") else: data = super().read_content() return data
[docs] def get_os_filename(self): """ Construct os specific file path out of the *file://* URL. @return: file name @rtype: string """ return get_os_filename(self.urlparts[2])
[docs] def get_temp_filename(self): """Get filename for content to parse.""" return self.get_os_filename()
[docs] def is_directory(self): """ Check if file is a directory. @return: True iff file is a directory @rtype: bool """ filename = self.get_os_filename() return os.path.isdir(filename) and not os.path.islink(filename)
[docs] def is_parseable(self): """Check if content is parseable for recursion. @return: True if content is parseable @rtype: bool """ if self.is_directory(): return True if firefox.has_sqlite and return True if self.content_type in self.ContentMimetypes: return True log.debug( LOG_CHECK, "File with content type %r is not parseable.", self.content_type ) return False
[docs] def set_content_type(self): """Return URL content type, or an empty string if content type could not be found.""" if self.url: self.content_type = mimeutil.guess_mimetype(self.url, read=self.get_content) else: self.content_type = ""
[docs] def get_intern_pattern(self, url=None): """Get pattern for intern URL matching. @return non-empty regex pattern or None @rtype String or None """ if url is None: url = self.url if not url: return None if url.startswith('file://'): i = url.rindex('/') if i > 6: # remove last filename to make directory internal url = url[: i + 1] return re.escape(url)
[docs] def add_url(self, url, line=0, column=0, page=0, name="", base=None): """If a local webroot directory is configured, replace absolute URLs with it. After that queue the URL data for checking.""" webroot = self.aggregate.config["localwebroot"] if webroot and url and url.startswith("/"): url = webroot + url[1:] log.debug(LOG_CHECK, "Applied local webroot `%s' to `%s'.", webroot, url) super().add_url(url, line=line, column=column, page=page, name=name, base=base)