configuration.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426
  1. """Configuration management setup
  2. Some terminology:
  3. - name
  4. As written in config files.
  5. - value
  6. Value associated with a name
  7. - key
  8. Name combined with it's section (section.name)
  9. - variant
  10. A single word describing where the configuration key-value pair came from
  11. """
  12. # The following comment should be removed at some point in the future.
  13. # mypy: strict-optional=False
  14. import locale
  15. import logging
  16. import os
  17. import sys
  18. from pip._vendor.six.moves import configparser
  19. from pip._internal.exceptions import (
  20. ConfigurationError,
  21. ConfigurationFileCouldNotBeLoaded,
  22. )
  23. from pip._internal.utils import appdirs
  24. from pip._internal.utils.compat import WINDOWS, expanduser
  25. from pip._internal.utils.misc import ensure_dir, enum
  26. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  27. if MYPY_CHECK_RUNNING:
  28. from typing import (
  29. Any, Dict, Iterable, List, NewType, Optional, Tuple
  30. )
  31. RawConfigParser = configparser.RawConfigParser # Shorthand
  32. Kind = NewType("Kind", str)
  33. logger = logging.getLogger(__name__)
  34. # NOTE: Maybe use the optionx attribute to normalize keynames.
  35. def _normalize_name(name):
  36. # type: (str) -> str
  37. """Make a name consistent regardless of source (environment or file)
  38. """
  39. name = name.lower().replace('_', '-')
  40. if name.startswith('--'):
  41. name = name[2:] # only prefer long opts
  42. return name
  43. def _disassemble_key(name):
  44. # type: (str) -> List[str]
  45. if "." not in name:
  46. error_message = (
  47. "Key does not contain dot separated section and key. "
  48. "Perhaps you wanted to use 'global.{}' instead?"
  49. ).format(name)
  50. raise ConfigurationError(error_message)
  51. return name.split(".", 1)
  52. # The kinds of configurations there are.
  53. kinds = enum(
  54. USER="user", # User Specific
  55. GLOBAL="global", # System Wide
  56. SITE="site", # [Virtual] Environment Specific
  57. ENV="env", # from PIP_CONFIG_FILE
  58. ENV_VAR="env-var", # from Environment Variables
  59. )
  60. CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf'
  61. def get_configuration_files():
  62. # type: () -> Dict[Kind, List[str]]
  63. global_config_files = [
  64. os.path.join(path, CONFIG_BASENAME)
  65. for path in appdirs.site_config_dirs('pip')
  66. ]
  67. site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME)
  68. legacy_config_file = os.path.join(
  69. expanduser('~'),
  70. 'pip' if WINDOWS else '.pip',
  71. CONFIG_BASENAME,
  72. )
  73. new_config_file = os.path.join(
  74. appdirs.user_config_dir("pip"), CONFIG_BASENAME
  75. )
  76. return {
  77. kinds.GLOBAL: global_config_files,
  78. kinds.SITE: [site_config_file],
  79. kinds.USER: [legacy_config_file, new_config_file],
  80. }
  81. class Configuration(object):
  82. """Handles management of configuration.
  83. Provides an interface to accessing and managing configuration files.
  84. This class converts provides an API that takes "section.key-name" style
  85. keys and stores the value associated with it as "key-name" under the
  86. section "section".
  87. This allows for a clean interface wherein the both the section and the
  88. key-name are preserved in an easy to manage form in the configuration files
  89. and the data stored is also nice.
  90. """
  91. def __init__(self, isolated, load_only=None):
  92. # type: (bool, Kind) -> None
  93. super(Configuration, self).__init__()
  94. _valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.SITE, None]
  95. if load_only not in _valid_load_only:
  96. raise ConfigurationError(
  97. "Got invalid value for load_only - should be one of {}".format(
  98. ", ".join(map(repr, _valid_load_only[:-1]))
  99. )
  100. )
  101. self.isolated = isolated # type: bool
  102. self.load_only = load_only # type: Optional[Kind]
  103. # The order here determines the override order.
  104. self._override_order = [
  105. kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR
  106. ]
  107. self._ignore_env_names = ["version", "help"]
  108. # Because we keep track of where we got the data from
  109. self._parsers = {
  110. variant: [] for variant in self._override_order
  111. } # type: Dict[Kind, List[Tuple[str, RawConfigParser]]]
  112. self._config = {
  113. variant: {} for variant in self._override_order
  114. } # type: Dict[Kind, Dict[str, Any]]
  115. self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]]
  116. def load(self):
  117. # type: () -> None
  118. """Loads configuration from configuration files and environment
  119. """
  120. self._load_config_files()
  121. if not self.isolated:
  122. self._load_environment_vars()
  123. def get_file_to_edit(self):
  124. # type: () -> Optional[str]
  125. """Returns the file with highest priority in configuration
  126. """
  127. assert self.load_only is not None, \
  128. "Need to be specified a file to be editing"
  129. try:
  130. return self._get_parser_to_modify()[0]
  131. except IndexError:
  132. return None
  133. def items(self):
  134. # type: () -> Iterable[Tuple[str, Any]]
  135. """Returns key-value pairs like dict.items() representing the loaded
  136. configuration
  137. """
  138. return self._dictionary.items()
  139. def get_value(self, key):
  140. # type: (str) -> Any
  141. """Get a value from the configuration.
  142. """
  143. try:
  144. return self._dictionary[key]
  145. except KeyError:
  146. raise ConfigurationError("No such key - {}".format(key))
  147. def set_value(self, key, value):
  148. # type: (str, Any) -> None
  149. """Modify a value in the configuration.
  150. """
  151. self._ensure_have_load_only()
  152. fname, parser = self._get_parser_to_modify()
  153. if parser is not None:
  154. section, name = _disassemble_key(key)
  155. # Modify the parser and the configuration
  156. if not parser.has_section(section):
  157. parser.add_section(section)
  158. parser.set(section, name, value)
  159. self._config[self.load_only][key] = value
  160. self._mark_as_modified(fname, parser)
  161. def unset_value(self, key):
  162. # type: (str) -> None
  163. """Unset a value in the configuration.
  164. """
  165. self._ensure_have_load_only()
  166. if key not in self._config[self.load_only]:
  167. raise ConfigurationError("No such key - {}".format(key))
  168. fname, parser = self._get_parser_to_modify()
  169. if parser is not None:
  170. section, name = _disassemble_key(key)
  171. # Remove the key in the parser
  172. modified_something = False
  173. if parser.has_section(section):
  174. # Returns whether the option was removed or not
  175. modified_something = parser.remove_option(section, name)
  176. if modified_something:
  177. # name removed from parser, section may now be empty
  178. section_iter = iter(parser.items(section))
  179. try:
  180. val = next(section_iter)
  181. except StopIteration:
  182. val = None
  183. if val is None:
  184. parser.remove_section(section)
  185. self._mark_as_modified(fname, parser)
  186. else:
  187. raise ConfigurationError(
  188. "Fatal Internal error [id=1]. Please report as a bug."
  189. )
  190. del self._config[self.load_only][key]
  191. def save(self):
  192. # type: () -> None
  193. """Save the current in-memory state.
  194. """
  195. self._ensure_have_load_only()
  196. for fname, parser in self._modified_parsers:
  197. logger.info("Writing to %s", fname)
  198. # Ensure directory exists.
  199. ensure_dir(os.path.dirname(fname))
  200. with open(fname, "w") as f:
  201. parser.write(f)
  202. #
  203. # Private routines
  204. #
  205. def _ensure_have_load_only(self):
  206. # type: () -> None
  207. if self.load_only is None:
  208. raise ConfigurationError("Needed a specific file to be modifying.")
  209. logger.debug("Will be working with %s variant only", self.load_only)
  210. @property
  211. def _dictionary(self):
  212. # type: () -> Dict[str, Any]
  213. """A dictionary representing the loaded configuration.
  214. """
  215. # NOTE: Dictionaries are not populated if not loaded. So, conditionals
  216. # are not needed here.
  217. retval = {}
  218. for variant in self._override_order:
  219. retval.update(self._config[variant])
  220. return retval
  221. def _load_config_files(self):
  222. # type: () -> None
  223. """Loads configuration from configuration files
  224. """
  225. config_files = dict(self._iter_config_files())
  226. if config_files[kinds.ENV][0:1] == [os.devnull]:
  227. logger.debug(
  228. "Skipping loading configuration files due to "
  229. "environment's PIP_CONFIG_FILE being os.devnull"
  230. )
  231. return
  232. for variant, files in config_files.items():
  233. for fname in files:
  234. # If there's specific variant set in `load_only`, load only
  235. # that variant, not the others.
  236. if self.load_only is not None and variant != self.load_only:
  237. logger.debug(
  238. "Skipping file '%s' (variant: %s)", fname, variant
  239. )
  240. continue
  241. parser = self._load_file(variant, fname)
  242. # Keeping track of the parsers used
  243. self._parsers[variant].append((fname, parser))
  244. def _load_file(self, variant, fname):
  245. # type: (Kind, str) -> RawConfigParser
  246. logger.debug("For variant '%s', will try loading '%s'", variant, fname)
  247. parser = self._construct_parser(fname)
  248. for section in parser.sections():
  249. items = parser.items(section)
  250. self._config[variant].update(self._normalized_keys(section, items))
  251. return parser
  252. def _construct_parser(self, fname):
  253. # type: (str) -> RawConfigParser
  254. parser = configparser.RawConfigParser()
  255. # If there is no such file, don't bother reading it but create the
  256. # parser anyway, to hold the data.
  257. # Doing this is useful when modifying and saving files, where we don't
  258. # need to construct a parser.
  259. if os.path.exists(fname):
  260. try:
  261. parser.read(fname)
  262. except UnicodeDecodeError:
  263. # See https://github.com/pypa/pip/issues/4963
  264. raise ConfigurationFileCouldNotBeLoaded(
  265. reason="contains invalid {} characters".format(
  266. locale.getpreferredencoding(False)
  267. ),
  268. fname=fname,
  269. )
  270. except configparser.Error as error:
  271. # See https://github.com/pypa/pip/issues/4893
  272. raise ConfigurationFileCouldNotBeLoaded(error=error)
  273. return parser
  274. def _load_environment_vars(self):
  275. # type: () -> None
  276. """Loads configuration from environment variables
  277. """
  278. self._config[kinds.ENV_VAR].update(
  279. self._normalized_keys(":env:", self._get_environ_vars())
  280. )
  281. def _normalized_keys(self, section, items):
  282. # type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any]
  283. """Normalizes items to construct a dictionary with normalized keys.
  284. This routine is where the names become keys and are made the same
  285. regardless of source - configuration files or environment.
  286. """
  287. normalized = {}
  288. for name, val in items:
  289. key = section + "." + _normalize_name(name)
  290. normalized[key] = val
  291. return normalized
  292. def _get_environ_vars(self):
  293. # type: () -> Iterable[Tuple[str, str]]
  294. """Returns a generator with all environmental vars with prefix PIP_"""
  295. for key, val in os.environ.items():
  296. should_be_yielded = (
  297. key.startswith("PIP_") and
  298. key[4:].lower() not in self._ignore_env_names
  299. )
  300. if should_be_yielded:
  301. yield key[4:].lower(), val
  302. # XXX: This is patched in the tests.
  303. def _iter_config_files(self):
  304. # type: () -> Iterable[Tuple[Kind, List[str]]]
  305. """Yields variant and configuration files associated with it.
  306. This should be treated like items of a dictionary.
  307. """
  308. # SMELL: Move the conditions out of this function
  309. # environment variables have the lowest priority
  310. config_file = os.environ.get('PIP_CONFIG_FILE', None)
  311. if config_file is not None:
  312. yield kinds.ENV, [config_file]
  313. else:
  314. yield kinds.ENV, []
  315. config_files = get_configuration_files()
  316. # at the base we have any global configuration
  317. yield kinds.GLOBAL, config_files[kinds.GLOBAL]
  318. # per-user configuration next
  319. should_load_user_config = not self.isolated and not (
  320. config_file and os.path.exists(config_file)
  321. )
  322. if should_load_user_config:
  323. # The legacy config file is overridden by the new config file
  324. yield kinds.USER, config_files[kinds.USER]
  325. # finally virtualenv configuration first trumping others
  326. yield kinds.SITE, config_files[kinds.SITE]
  327. def _get_parser_to_modify(self):
  328. # type: () -> Tuple[str, RawConfigParser]
  329. # Determine which parser to modify
  330. parsers = self._parsers[self.load_only]
  331. if not parsers:
  332. # This should not happen if everything works correctly.
  333. raise ConfigurationError(
  334. "Fatal Internal error [id=2]. Please report as a bug."
  335. )
  336. # Use the highest priority parser.
  337. return parsers[-1]
  338. # XXX: This is patched in the tests.
  339. def _mark_as_modified(self, fname, parser):
  340. # type: (str, RawConfigParser) -> None
  341. file_parser_tuple = (fname, parser)
  342. if file_parser_tuple not in self._modified_parsers:
  343. self._modified_parsers.append(file_parser_tuple)
  344. def __repr__(self):
  345. # type: () -> str
  346. return "{}({!r})".format(self.__class__.__name__, self._dictionary)