resources.py

Go to the documentation of this file.

# -*- coding: utf-8 -*-
#
# Copyright (C) 2013-2017 Vinay Sajip.
# Licensed to the Python Software Foundation under a contributor agreement.
# See LICENSE.txt and CONTRIBUTORS.txt.
#
from __future__ import unicode_literals
 
import bisect
import io
import logging
import os
import pkgutil
import sys
import types
import zipimport
 
from . import DistlibException
from .util import cached_property, get_cache_base, Cache
 
logger = logging.getLogger(__name__)
 
 
cache = None    # created when needed
 
 
class ResourceCache(Cache):
    def __init__(self, base=None):
        if base is None:
            # Use native string to avoid issues on 2.x: see Python #20140.
            base = os.path.join(get_cache_base(), str('resource-cache'))
        super(ResourceCache, self).__init__(base)
 
    def is_stale(self, resource, path):
        """
        Is the cache stale for the given resource?
 
        :param resource: The :class:`Resource` being cached.
        :param path: The path of the resource in the cache.
        :return: True if the cache is stale.
        """
        # Cache invalidation is a hard problem :-)
        return True
 
    def get(self, resource):
        """
        Get a resource into the cache,
 
        :param resource: A :class:`Resource` instance.
        :return: The pathname of the resource in the cache.
        """
        prefix, path = resource.finder.get_cache_info(resource)
        if prefix is None:
            result = path
        else:
            result = os.path.join(self.base, self.prefix_to_dir(prefix), path)
            dirname = os.path.dirname(result)
            if not os.path.isdir(dirname):
                os.makedirs(dirname)
            if not os.path.exists(result):
                stale = True
            else:
                stale = self.is_stale(resource, path)
            if stale:
                # write the bytes of the resource to the cache location
                with open(result, 'wb') as f:
                    f.write(resource.bytes)
        return result
 
 
class ResourceBase(object):
    def __init__(self, finder, name):
        self.finder = finder
        self.name = name
 
 
class Resource(ResourceBase):
    """
    A class representing an in-package resource, such as a data file. This is
    not normally instantiated by user code, but rather by a
    :class:`ResourceFinder` which manages the resource.
    """
    is_container = False        # Backwards compatibility
 
    def as_stream(self):
        """
        Get the resource as a stream.
 
        This is not a property to make it obvious that it returns a new stream
        each time.
        """
        return self.finder.get_stream(self)
 
    @cached_property
    def file_path(self):
        global cache
        if cache is None:
            cache = ResourceCache()
        return cache.get(self)
 
    @cached_property
    def bytes(self):
        return self.finder.get_bytes(self)
 
    @cached_property
    def size(self):
        return self.finder.get_size(self)
 
 
class ResourceContainer(ResourceBase):
    is_container = True     # Backwards compatibility
 
    @cached_property
    def resources(self):
        return self.finder.get_resources(self)
 
 
class ResourceFinder(object):
    """
    Resource finder for file system resources.
    """
 
    if sys.platform.startswith('java'):
        skipped_extensions = ('.pyc', '.pyo', '.class')
    else:
        skipped_extensions = ('.pyc', '.pyo')
 
    def __init__(self, module):
        self.module = module
        self.loader = getattr(module, '__loader__', None)
        self.base = os.path.dirname(getattr(module, '__file__', ''))
 
    def _adjust_path(self, path):
        return os.path.realpath(path)
 
    def _make_path(self, resource_name):
        # Issue #50: need to preserve type of path on Python 2.x
        # like os.path._get_sep
        if isinstance(resource_name, bytes):    # should only happen on 2.x
            sep = b'/'
        else:
            sep = '/'
        parts = resource_name.split(sep)
        parts.insert(0, self.base)
        result = os.path.join(*parts)
        return self._adjust_path(result)
 
    def _find(self, path):
        return os.path.exists(path)
 
    def get_cache_info(self, resource):
        return None, resource.path
 
    def find(self, resource_name):
        path = self._make_path(resource_name)
        if not self._find(path):
            result = None
        else:
            if self._is_directory(path):
                result = ResourceContainer(self, resource_name)
            else:
                result = Resource(self, resource_name)
            result.path = path
        return result
 
    def get_stream(self, resource):
        return open(resource.path, 'rb')
 
    def get_bytes(self, resource):
        with open(resource.path, 'rb') as f:
            return f.read()
 
    def get_size(self, resource):
        return os.path.getsize(resource.path)
 
    def get_resources(self, resource):
        def allowed(f):
            return (f != '__pycache__' and not
                    f.endswith(self.skipped_extensionsskipped_extensions))
        return set([f for f in os.listdir(resource.path) if allowed(f)])
 
    def is_container(self, resource):
        return self._is_directory(resource.path)
 
    _is_directory = staticmethod(os.path.isdir)
 
    def iterator(self, resource_name):
        resource = self.find(resource_name)
        if resource is not None:
            todo = [resource]
            while todo:
                resource = todo.pop(0)
                yield resource
                if resource.is_container:
                    rname = resource.name
                    for name in resource.resources:
                        if not rname:
                            new_name = name
                        else:
                            new_name = '/'.join([rname, name])
                        child = self.find(new_name)
                        if child.is_container:
                            todo.append(child)
                        else:
                            yield child
 
 
class ZipResourceFinder(ResourceFinder):
    """
    Resource finder for resources in .zip files.
    """
    def __init__(self, module):
        super(ZipResourceFinder, self).__init__(module)
        archive = self.loaderloader.archive
        self.prefix_len = 1 + len(archive)
        # PyPy doesn't have a _files attr on zipimporter, and you can't set one
        if hasattr(self.loaderloader, '_files'):
            self._files = self.loaderloader._files
        else:
            self._files = zipimport._zip_directory_cache[archive]
        self.index = sorted(self._files)
 
    def _adjust_path(self, path):
        return path
 
    def _find(self, path):
        path = path[self.prefix_len:]
        if path in self._files:
            result = True
        else:
            if path and path[-1] != os.sep:
                path = path + os.sep
            i = bisect.bisect(self.index, path)
            try:
                result = self.index[i].startswith(path)
            except IndexError:
                result = False
        if not result:
            logger.debug('_find failed: %r %r', path, self.loaderloader.prefix)
        else:
            logger.debug('_find worked: %r %r', path, self.loaderloader.prefix)
        return result
 
    def get_cache_info(self, resource):
        prefix = self.loaderloader.archive
        path = resource.path[1 + len(prefix):]
        return prefix, path
 
    def get_bytes(self, resource):
        return self.loaderloader.get_data(resource.path)
 
    def get_stream(self, resource):
        return io.BytesIO(self.get_bytesget_bytes(resource))
 
    def get_size(self, resource):
        path = resource.path[self.prefix_len:]
        return self._files[path][3]
 
    def get_resources(self, resource):
        path = resource.path[self.prefix_len:]
        if path and path[-1] != os.sep:
            path += os.sep
        plen = len(path)
        result = set()
        i = bisect.bisect(self.index, path)
        while i < len(self.index):
            if not self.index[i].startswith(path):
                break
            s = self.index[i][plen:]
            result.add(s.split(os.sep, 1)[0])   # only immediate children
            i += 1
        return result
 
    def _is_directory(self, path):
        path = path[self.prefix_len:]
        if path and path[-1] != os.sep:
            path += os.sep
        i = bisect.bisect(self.index, path)
        try:
            result = self.index[i].startswith(path)
        except IndexError:
            result = False
        return result
 
 
_finder_registry = {
    type(None): ResourceFinder,
    zipimport.zipimporter: ZipResourceFinder
}
 
try:
    # In Python 3.6, _frozen_importlib -> _frozen_importlib_external
    try:
        import _frozen_importlib_external as _fi
    except ImportError:
        import _frozen_importlib as _fi
    _finder_registry[_fi.SourceFileLoader] = ResourceFinder
    _finder_registry[_fi.FileFinder] = ResourceFinder
    # See issue #146
    _finder_registry[_fi.SourcelessFileLoader] = ResourceFinder
    del _fi
except (ImportError, AttributeError):
    pass
 
 
def register_finder(loader, finder_maker):
    _finder_registry[type(loader)] = finder_maker
 
 
_finder_cache = {}
 
 
def finder(package):
    """
    Return a resource finder for a package.
    :param package: The name of the package.
    :return: A :class:`ResourceFinder` instance for the package.
    """
    if package in _finder_cache:
        result = _finder_cache[package]
    else:
        if package not in sys.modules:
            __import__(package)
        module = sys.modules[package]
        path = getattr(module, '__path__', None)
        if path is None:
            raise DistlibException('You cannot get a finder for a module, '
                                   'only for a package')
        loader = getattr(module, '__loader__', None)
        finder_maker = _finder_registry.get(type(loader))
        if finder_maker is None:
            raise DistlibException('Unable to locate finder for %r' % package)
        result = finder_maker(module)
        _finder_cache[package] = result
    return result
 
 
_dummy_module = types.ModuleType(str('__dummy__'))
 
 
def finder_for_path(path):
    """
    Return a resource finder for a path, which should represent a container.
 
    :param path: The path.
    :return: A :class:`ResourceFinder` instance for the path.
    """
    result = None
    # calls any path hooks, gets importer into cache
    pkgutil.get_importer(path)
    loader = sys.path_importer_cache.get(path)
    finder = _finder_registry.get(type(loader))
    if finder:
        module = _dummy_module
        module.__file__ = os.path.join(path, '')
        module.__loader__ = loader
        result = finder(module)
    return result