123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700 |
- from __future__ import absolute_import, unicode_literals
- import ast
- import io
- import os
- import sys
- import warnings
- import functools
- import importlib
- from collections import defaultdict
- from functools import partial
- from functools import wraps
- import contextlib
- from distutils.errors import DistutilsOptionError, DistutilsFileError
- from setuptools.extern.packaging.version import LegacyVersion, parse
- from setuptools.extern.packaging.specifiers import SpecifierSet
- from setuptools.extern.six import string_types, PY3
- __metaclass__ = type
- class StaticModule:
- """
- Attempt to load the module by the name
- """
- def __init__(self, name):
- spec = importlib.util.find_spec(name)
- with open(spec.origin) as strm:
- src = strm.read()
- module = ast.parse(src)
- vars(self).update(locals())
- del self.self
- def __getattr__(self, attr):
- try:
- return next(
- ast.literal_eval(statement.value)
- for statement in self.module.body
- if isinstance(statement, ast.Assign)
- for target in statement.targets
- if isinstance(target, ast.Name) and target.id == attr
- )
- except Exception:
- raise AttributeError(
- "{self.name} has no attribute {attr}".format(**locals()))
- @contextlib.contextmanager
- def patch_path(path):
- """
- Add path to front of sys.path for the duration of the context.
- """
- try:
- sys.path.insert(0, path)
- yield
- finally:
- sys.path.remove(path)
- def read_configuration(
- filepath, find_others=False, ignore_option_errors=False):
- """Read given configuration file and returns options from it as a dict.
- :param str|unicode filepath: Path to configuration file
- to get options from.
- :param bool find_others: Whether to search for other configuration files
- which could be on in various places.
- :param bool ignore_option_errors: Whether to silently ignore
- options, values of which could not be resolved (e.g. due to exceptions
- in directives such as file:, attr:, etc.).
- If False exceptions are propagated as expected.
- :rtype: dict
- """
- from setuptools.dist import Distribution, _Distribution
- filepath = os.path.abspath(filepath)
- if not os.path.isfile(filepath):
- raise DistutilsFileError(
- 'Configuration file %s does not exist.' % filepath)
- current_directory = os.getcwd()
- os.chdir(os.path.dirname(filepath))
- try:
- dist = Distribution()
- filenames = dist.find_config_files() if find_others else []
- if filepath not in filenames:
- filenames.append(filepath)
- _Distribution.parse_config_files(dist, filenames=filenames)
- handlers = parse_configuration(
- dist, dist.command_options,
- ignore_option_errors=ignore_option_errors)
- finally:
- os.chdir(current_directory)
- return configuration_to_dict(handlers)
- def _get_option(target_obj, key):
- """
- Given a target object and option key, get that option from
- the target object, either through a get_{key} method or
- from an attribute directly.
- """
- getter_name = 'get_{key}'.format(**locals())
- by_attribute = functools.partial(getattr, target_obj, key)
- getter = getattr(target_obj, getter_name, by_attribute)
- return getter()
- def configuration_to_dict(handlers):
- """Returns configuration data gathered by given handlers as a dict.
- :param list[ConfigHandler] handlers: Handlers list,
- usually from parse_configuration()
- :rtype: dict
- """
- config_dict = defaultdict(dict)
- for handler in handlers:
- for option in handler.set_options:
- value = _get_option(handler.target_obj, option)
- config_dict[handler.section_prefix][option] = value
- return config_dict
- def parse_configuration(
- distribution, command_options, ignore_option_errors=False):
- """Performs additional parsing of configuration options
- for a distribution.
- Returns a list of used option handlers.
- :param Distribution distribution:
- :param dict command_options:
- :param bool ignore_option_errors: Whether to silently ignore
- options, values of which could not be resolved (e.g. due to exceptions
- in directives such as file:, attr:, etc.).
- If False exceptions are propagated as expected.
- :rtype: list
- """
- options = ConfigOptionsHandler(
- distribution, command_options, ignore_option_errors)
- options.parse()
- meta = ConfigMetadataHandler(
- distribution.metadata, command_options, ignore_option_errors,
- distribution.package_dir)
- meta.parse()
- return meta, options
- class ConfigHandler:
- """Handles metadata supplied in configuration files."""
- section_prefix = None
- """Prefix for config sections handled by this handler.
- Must be provided by class heirs.
- """
- aliases = {}
- """Options aliases.
- For compatibility with various packages. E.g.: d2to1 and pbr.
- Note: `-` in keys is replaced with `_` by config parser.
- """
- def __init__(self, target_obj, options, ignore_option_errors=False):
- sections = {}
- section_prefix = self.section_prefix
- for section_name, section_options in options.items():
- if not section_name.startswith(section_prefix):
- continue
- section_name = section_name.replace(section_prefix, '').strip('.')
- sections[section_name] = section_options
- self.ignore_option_errors = ignore_option_errors
- self.target_obj = target_obj
- self.sections = sections
- self.set_options = []
- @property
- def parsers(self):
- """Metadata item name to parser function mapping."""
- raise NotImplementedError(
- '%s must provide .parsers property' % self.__class__.__name__)
- def __setitem__(self, option_name, value):
- unknown = tuple()
- target_obj = self.target_obj
- # Translate alias into real name.
- option_name = self.aliases.get(option_name, option_name)
- current_value = getattr(target_obj, option_name, unknown)
- if current_value is unknown:
- raise KeyError(option_name)
- if current_value:
- # Already inhabited. Skipping.
- return
- skip_option = False
- parser = self.parsers.get(option_name)
- if parser:
- try:
- value = parser(value)
- except Exception:
- skip_option = True
- if not self.ignore_option_errors:
- raise
- if skip_option:
- return
- setter = getattr(target_obj, 'set_%s' % option_name, None)
- if setter is None:
- setattr(target_obj, option_name, value)
- else:
- setter(value)
- self.set_options.append(option_name)
- @classmethod
- def _parse_list(cls, value, separator=','):
- """Represents value as a list.
- Value is split either by separator (defaults to comma) or by lines.
- :param value:
- :param separator: List items separator character.
- :rtype: list
- """
- if isinstance(value, list): # _get_parser_compound case
- return value
- if '\n' in value:
- value = value.splitlines()
- else:
- value = value.split(separator)
- return [chunk.strip() for chunk in value if chunk.strip()]
- @classmethod
- def _parse_dict(cls, value):
- """Represents value as a dict.
- :param value:
- :rtype: dict
- """
- separator = '='
- result = {}
- for line in cls._parse_list(value):
- key, sep, val = line.partition(separator)
- if sep != separator:
- raise DistutilsOptionError(
- 'Unable to parse option value to dict: %s' % value)
- result[key.strip()] = val.strip()
- return result
- @classmethod
- def _parse_bool(cls, value):
- """Represents value as boolean.
- :param value:
- :rtype: bool
- """
- value = value.lower()
- return value in ('1', 'true', 'yes')
- @classmethod
- def _exclude_files_parser(cls, key):
- """Returns a parser function to make sure field inputs
- are not files.
- Parses a value after getting the key so error messages are
- more informative.
- :param key:
- :rtype: callable
- """
- def parser(value):
- exclude_directive = 'file:'
- if value.startswith(exclude_directive):
- raise ValueError(
- 'Only strings are accepted for the {0} field, '
- 'files are not accepted'.format(key))
- return value
- return parser
- @classmethod
- def _parse_file(cls, value):
- """Represents value as a string, allowing including text
- from nearest files using `file:` directive.
- Directive is sandboxed and won't reach anything outside
- directory with setup.py.
- Examples:
- file: README.rst, CHANGELOG.md, src/file.txt
- :param str value:
- :rtype: str
- """
- include_directive = 'file:'
- if not isinstance(value, string_types):
- return value
- if not value.startswith(include_directive):
- return value
- spec = value[len(include_directive):]
- filepaths = (os.path.abspath(path.strip()) for path in spec.split(','))
- return '\n'.join(
- cls._read_file(path)
- for path in filepaths
- if (cls._assert_local(path) or True)
- and os.path.isfile(path)
- )
- @staticmethod
- def _assert_local(filepath):
- if not filepath.startswith(os.getcwd()):
- raise DistutilsOptionError(
- '`file:` directive can not access %s' % filepath)
- @staticmethod
- def _read_file(filepath):
- with io.open(filepath, encoding='utf-8') as f:
- return f.read()
- @classmethod
- def _parse_attr(cls, value, package_dir=None):
- """Represents value as a module attribute.
- Examples:
- attr: package.attr
- attr: package.module.attr
- :param str value:
- :rtype: str
- """
- attr_directive = 'attr:'
- if not value.startswith(attr_directive):
- return value
- attrs_path = value.replace(attr_directive, '').strip().split('.')
- attr_name = attrs_path.pop()
- module_name = '.'.join(attrs_path)
- module_name = module_name or '__init__'
- parent_path = os.getcwd()
- if package_dir:
- if attrs_path[0] in package_dir:
- # A custom path was specified for the module we want to import
- custom_path = package_dir[attrs_path[0]]
- parts = custom_path.rsplit('/', 1)
- if len(parts) > 1:
- parent_path = os.path.join(os.getcwd(), parts[0])
- module_name = parts[1]
- else:
- module_name = custom_path
- elif '' in package_dir:
- # A custom parent directory was specified for all root modules
- parent_path = os.path.join(os.getcwd(), package_dir[''])
- with patch_path(parent_path):
- try:
- # attempt to load value statically
- return getattr(StaticModule(module_name), attr_name)
- except Exception:
- # fallback to simple import
- module = importlib.import_module(module_name)
- return getattr(module, attr_name)
- @classmethod
- def _get_parser_compound(cls, *parse_methods):
- """Returns parser function to represents value as a list.
- Parses a value applying given methods one after another.
- :param parse_methods:
- :rtype: callable
- """
- def parse(value):
- parsed = value
- for method in parse_methods:
- parsed = method(parsed)
- return parsed
- return parse
- @classmethod
- def _parse_section_to_dict(cls, section_options, values_parser=None):
- """Parses section options into a dictionary.
- Optionally applies a given parser to values.
- :param dict section_options:
- :param callable values_parser:
- :rtype: dict
- """
- value = {}
- values_parser = values_parser or (lambda val: val)
- for key, (_, val) in section_options.items():
- value[key] = values_parser(val)
- return value
- def parse_section(self, section_options):
- """Parses configuration file section.
- :param dict section_options:
- """
- for (name, (_, value)) in section_options.items():
- try:
- self[name] = value
- except KeyError:
- pass # Keep silent for a new option may appear anytime.
- def parse(self):
- """Parses configuration file items from one
- or more related sections.
- """
- for section_name, section_options in self.sections.items():
- method_postfix = ''
- if section_name: # [section.option] variant
- method_postfix = '_%s' % section_name
- section_parser_method = getattr(
- self,
- # Dots in section names are translated into dunderscores.
- ('parse_section%s' % method_postfix).replace('.', '__'),
- None)
- if section_parser_method is None:
- raise DistutilsOptionError(
- 'Unsupported distribution option section: [%s.%s]' % (
- self.section_prefix, section_name))
- section_parser_method(section_options)
- def _deprecated_config_handler(self, func, msg, warning_class):
- """ this function will wrap around parameters that are deprecated
- :param msg: deprecation message
- :param warning_class: class of warning exception to be raised
- :param func: function to be wrapped around
- """
- @wraps(func)
- def config_handler(*args, **kwargs):
- warnings.warn(msg, warning_class)
- return func(*args, **kwargs)
- return config_handler
- class ConfigMetadataHandler(ConfigHandler):
- section_prefix = 'metadata'
- aliases = {
- 'home_page': 'url',
- 'summary': 'description',
- 'classifier': 'classifiers',
- 'platform': 'platforms',
- }
- strict_mode = False
- """We need to keep it loose, to be partially compatible with
- `pbr` and `d2to1` packages which also uses `metadata` section.
- """
- def __init__(self, target_obj, options, ignore_option_errors=False,
- package_dir=None):
- super(ConfigMetadataHandler, self).__init__(target_obj, options,
- ignore_option_errors)
- self.package_dir = package_dir
- @property
- def parsers(self):
- """Metadata item name to parser function mapping."""
- parse_list = self._parse_list
- parse_file = self._parse_file
- parse_dict = self._parse_dict
- exclude_files_parser = self._exclude_files_parser
- return {
- 'platforms': parse_list,
- 'keywords': parse_list,
- 'provides': parse_list,
- 'requires': self._deprecated_config_handler(
- parse_list,
- "The requires parameter is deprecated, please use "
- "install_requires for runtime dependencies.",
- DeprecationWarning),
- 'obsoletes': parse_list,
- 'classifiers': self._get_parser_compound(parse_file, parse_list),
- 'license': exclude_files_parser('license'),
- 'license_files': parse_list,
- 'description': parse_file,
- 'long_description': parse_file,
- 'version': self._parse_version,
- 'project_urls': parse_dict,
- }
- def _parse_version(self, value):
- """Parses `version` option value.
- :param value:
- :rtype: str
- """
- version = self._parse_file(value)
- if version != value:
- version = version.strip()
- # Be strict about versions loaded from file because it's easy to
- # accidentally include newlines and other unintended content
- if isinstance(parse(version), LegacyVersion):
- tmpl = (
- 'Version loaded from {value} does not '
- 'comply with PEP 440: {version}'
- )
- raise DistutilsOptionError(tmpl.format(**locals()))
- return version
- version = self._parse_attr(value, self.package_dir)
- if callable(version):
- version = version()
- if not isinstance(version, string_types):
- if hasattr(version, '__iter__'):
- version = '.'.join(map(str, version))
- else:
- version = '%s' % version
- return version
- class ConfigOptionsHandler(ConfigHandler):
- section_prefix = 'options'
- @property
- def parsers(self):
- """Metadata item name to parser function mapping."""
- parse_list = self._parse_list
- parse_list_semicolon = partial(self._parse_list, separator=';')
- parse_bool = self._parse_bool
- parse_dict = self._parse_dict
- return {
- 'zip_safe': parse_bool,
- 'use_2to3': parse_bool,
- 'include_package_data': parse_bool,
- 'package_dir': parse_dict,
- 'use_2to3_fixers': parse_list,
- 'use_2to3_exclude_fixers': parse_list,
- 'convert_2to3_doctests': parse_list,
- 'scripts': parse_list,
- 'eager_resources': parse_list,
- 'dependency_links': parse_list,
- 'namespace_packages': parse_list,
- 'install_requires': parse_list_semicolon,
- 'setup_requires': parse_list_semicolon,
- 'tests_require': parse_list_semicolon,
- 'packages': self._parse_packages,
- 'entry_points': self._parse_file,
- 'py_modules': parse_list,
- 'python_requires': SpecifierSet,
- }
- def _parse_packages(self, value):
- """Parses `packages` option value.
- :param value:
- :rtype: list
- """
- find_directives = ['find:', 'find_namespace:']
- trimmed_value = value.strip()
- if trimmed_value not in find_directives:
- return self._parse_list(value)
- findns = trimmed_value == find_directives[1]
- if findns and not PY3:
- raise DistutilsOptionError(
- 'find_namespace: directive is unsupported on Python < 3.3')
- # Read function arguments from a dedicated section.
- find_kwargs = self.parse_section_packages__find(
- self.sections.get('packages.find', {}))
- if findns:
- from setuptools import find_namespace_packages as find_packages
- else:
- from setuptools import find_packages
- return find_packages(**find_kwargs)
- def parse_section_packages__find(self, section_options):
- """Parses `packages.find` configuration file section.
- To be used in conjunction with _parse_packages().
- :param dict section_options:
- """
- section_data = self._parse_section_to_dict(
- section_options, self._parse_list)
- valid_keys = ['where', 'include', 'exclude']
- find_kwargs = dict(
- [(k, v) for k, v in section_data.items() if k in valid_keys and v])
- where = find_kwargs.get('where')
- if where is not None:
- find_kwargs['where'] = where[0] # cast list to single val
- return find_kwargs
- def parse_section_entry_points(self, section_options):
- """Parses `entry_points` configuration file section.
- :param dict section_options:
- """
- parsed = self._parse_section_to_dict(section_options, self._parse_list)
- self['entry_points'] = parsed
- def _parse_package_data(self, section_options):
- parsed = self._parse_section_to_dict(section_options, self._parse_list)
- root = parsed.get('*')
- if root:
- parsed[''] = root
- del parsed['*']
- return parsed
- def parse_section_package_data(self, section_options):
- """Parses `package_data` configuration file section.
- :param dict section_options:
- """
- self['package_data'] = self._parse_package_data(section_options)
- def parse_section_exclude_package_data(self, section_options):
- """Parses `exclude_package_data` configuration file section.
- :param dict section_options:
- """
- self['exclude_package_data'] = self._parse_package_data(
- section_options)
- def parse_section_extras_require(self, section_options):
- """Parses `extras_require` configuration file section.
- :param dict section_options:
- """
- parse_list = partial(self._parse_list, separator=';')
- self['extras_require'] = self._parse_section_to_dict(
- section_options, parse_list)
- def parse_section_data_files(self, section_options):
- """Parses `data_files` configuration file section.
- :param dict section_options:
- """
- parsed = self._parse_section_to_dict(section_options, self._parse_list)
- self['data_files'] = [(k, v) for k, v in parsed.items()]
|