misc.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942
  1. # The following comment should be removed at some point in the future.
  2. # mypy: strict-optional=False
  3. # mypy: disallow-untyped-defs=False
  4. from __future__ import absolute_import
  5. import contextlib
  6. import errno
  7. import getpass
  8. import hashlib
  9. import io
  10. import logging
  11. import os
  12. import posixpath
  13. import shutil
  14. import stat
  15. import sys
  16. from collections import deque
  17. from itertools import tee
  18. from pip._vendor import pkg_resources
  19. from pip._vendor.packaging.utils import canonicalize_name
  20. # NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is
  21. # why we ignore the type on this import.
  22. from pip._vendor.retrying import retry # type: ignore
  23. from pip._vendor.six import PY2, text_type
  24. from pip._vendor.six.moves import filter, filterfalse, input, map, zip_longest
  25. from pip._vendor.six.moves.urllib import parse as urllib_parse
  26. from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote
  27. from pip import __version__
  28. from pip._internal.exceptions import CommandError
  29. from pip._internal.locations import (
  30. get_major_minor_version,
  31. site_packages,
  32. user_site,
  33. )
  34. from pip._internal.utils.compat import (
  35. WINDOWS,
  36. expanduser,
  37. stdlib_pkgs,
  38. str_to_display,
  39. )
  40. from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast
  41. from pip._internal.utils.virtualenv import (
  42. running_under_virtualenv,
  43. virtualenv_no_global,
  44. )
  45. if PY2:
  46. from io import BytesIO as StringIO
  47. else:
  48. from io import StringIO
  49. if MYPY_CHECK_RUNNING:
  50. from typing import (
  51. Any, AnyStr, Callable, Container, Iterable, Iterator, List, Optional,
  52. Text, Tuple, TypeVar, Union,
  53. )
  54. from pip._vendor.pkg_resources import Distribution
  55. VersionInfo = Tuple[int, int, int]
  56. T = TypeVar("T")
  57. __all__ = ['rmtree', 'display_path', 'backup_dir',
  58. 'ask', 'splitext',
  59. 'format_size', 'is_installable_dir',
  60. 'normalize_path',
  61. 'renames', 'get_prog',
  62. 'captured_stdout', 'ensure_dir',
  63. 'get_installed_version', 'remove_auth_from_url']
  64. logger = logging.getLogger(__name__)
  65. def get_pip_version():
  66. # type: () -> str
  67. pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..")
  68. pip_pkg_dir = os.path.abspath(pip_pkg_dir)
  69. return (
  70. 'pip {} from {} (python {})'.format(
  71. __version__, pip_pkg_dir, get_major_minor_version(),
  72. )
  73. )
  74. def normalize_version_info(py_version_info):
  75. # type: (Tuple[int, ...]) -> Tuple[int, int, int]
  76. """
  77. Convert a tuple of ints representing a Python version to one of length
  78. three.
  79. :param py_version_info: a tuple of ints representing a Python version,
  80. or None to specify no version. The tuple can have any length.
  81. :return: a tuple of length three if `py_version_info` is non-None.
  82. Otherwise, return `py_version_info` unchanged (i.e. None).
  83. """
  84. if len(py_version_info) < 3:
  85. py_version_info += (3 - len(py_version_info)) * (0,)
  86. elif len(py_version_info) > 3:
  87. py_version_info = py_version_info[:3]
  88. return cast('VersionInfo', py_version_info)
  89. def ensure_dir(path):
  90. # type: (AnyStr) -> None
  91. """os.path.makedirs without EEXIST."""
  92. try:
  93. os.makedirs(path)
  94. except OSError as e:
  95. # Windows can raise spurious ENOTEMPTY errors. See #6426.
  96. if e.errno != errno.EEXIST and e.errno != errno.ENOTEMPTY:
  97. raise
  98. def get_prog():
  99. # type: () -> str
  100. try:
  101. prog = os.path.basename(sys.argv[0])
  102. if prog in ('__main__.py', '-c'):
  103. return "{} -m pip".format(sys.executable)
  104. else:
  105. return prog
  106. except (AttributeError, TypeError, IndexError):
  107. pass
  108. return 'pip'
  109. # Retry every half second for up to 3 seconds
  110. @retry(stop_max_delay=3000, wait_fixed=500)
  111. def rmtree(dir, ignore_errors=False):
  112. # type: (Text, bool) -> None
  113. shutil.rmtree(dir, ignore_errors=ignore_errors,
  114. onerror=rmtree_errorhandler)
  115. def rmtree_errorhandler(func, path, exc_info):
  116. """On Windows, the files in .svn are read-only, so when rmtree() tries to
  117. remove them, an exception is thrown. We catch that here, remove the
  118. read-only attribute, and hopefully continue without problems."""
  119. try:
  120. has_attr_readonly = not (os.stat(path).st_mode & stat.S_IWRITE)
  121. except (IOError, OSError):
  122. # it's equivalent to os.path.exists
  123. return
  124. if has_attr_readonly:
  125. # convert to read/write
  126. os.chmod(path, stat.S_IWRITE)
  127. # use the original function to repeat the operation
  128. func(path)
  129. return
  130. else:
  131. raise
  132. def path_to_display(path):
  133. # type: (Optional[Union[str, Text]]) -> Optional[Text]
  134. """
  135. Convert a bytes (or text) path to text (unicode in Python 2) for display
  136. and logging purposes.
  137. This function should never error out. Also, this function is mainly needed
  138. for Python 2 since in Python 3 str paths are already text.
  139. """
  140. if path is None:
  141. return None
  142. if isinstance(path, text_type):
  143. return path
  144. # Otherwise, path is a bytes object (str in Python 2).
  145. try:
  146. display_path = path.decode(sys.getfilesystemencoding(), 'strict')
  147. except UnicodeDecodeError:
  148. # Include the full bytes to make troubleshooting easier, even though
  149. # it may not be very human readable.
  150. if PY2:
  151. # Convert the bytes to a readable str representation using
  152. # repr(), and then convert the str to unicode.
  153. # Also, we add the prefix "b" to the repr() return value both
  154. # to make the Python 2 output look like the Python 3 output, and
  155. # to signal to the user that this is a bytes representation.
  156. display_path = str_to_display('b{!r}'.format(path))
  157. else:
  158. # Silence the "F821 undefined name 'ascii'" flake8 error since
  159. # in Python 3 ascii() is a built-in.
  160. display_path = ascii(path) # noqa: F821
  161. return display_path
  162. def display_path(path):
  163. # type: (Union[str, Text]) -> str
  164. """Gives the display value for a given path, making it relative to cwd
  165. if possible."""
  166. path = os.path.normcase(os.path.abspath(path))
  167. if sys.version_info[0] == 2:
  168. path = path.decode(sys.getfilesystemencoding(), 'replace')
  169. path = path.encode(sys.getdefaultencoding(), 'replace')
  170. if path.startswith(os.getcwd() + os.path.sep):
  171. path = '.' + path[len(os.getcwd()):]
  172. return path
  173. def backup_dir(dir, ext='.bak'):
  174. # type: (str, str) -> str
  175. """Figure out the name of a directory to back up the given dir to
  176. (adding .bak, .bak2, etc)"""
  177. n = 1
  178. extension = ext
  179. while os.path.exists(dir + extension):
  180. n += 1
  181. extension = ext + str(n)
  182. return dir + extension
  183. def ask_path_exists(message, options):
  184. # type: (str, Iterable[str]) -> str
  185. for action in os.environ.get('PIP_EXISTS_ACTION', '').split():
  186. if action in options:
  187. return action
  188. return ask(message, options)
  189. def _check_no_input(message):
  190. # type: (str) -> None
  191. """Raise an error if no input is allowed."""
  192. if os.environ.get('PIP_NO_INPUT'):
  193. raise Exception(
  194. 'No input was expected ($PIP_NO_INPUT set); question: {}'.format(
  195. message)
  196. )
  197. def ask(message, options):
  198. # type: (str, Iterable[str]) -> str
  199. """Ask the message interactively, with the given possible responses"""
  200. while 1:
  201. _check_no_input(message)
  202. response = input(message)
  203. response = response.strip().lower()
  204. if response not in options:
  205. print(
  206. 'Your response ({!r}) was not one of the expected responses: '
  207. '{}'.format(response, ', '.join(options))
  208. )
  209. else:
  210. return response
  211. def ask_input(message):
  212. # type: (str) -> str
  213. """Ask for input interactively."""
  214. _check_no_input(message)
  215. return input(message)
  216. def ask_password(message):
  217. # type: (str) -> str
  218. """Ask for a password interactively."""
  219. _check_no_input(message)
  220. return getpass.getpass(message)
  221. def format_size(bytes):
  222. # type: (float) -> str
  223. if bytes > 1000 * 1000:
  224. return '{:.1f} MB'.format(bytes / 1000.0 / 1000)
  225. elif bytes > 10 * 1000:
  226. return '{} kB'.format(int(bytes / 1000))
  227. elif bytes > 1000:
  228. return '{:.1f} kB'.format(bytes / 1000.0)
  229. else:
  230. return '{} bytes'.format(int(bytes))
  231. def tabulate(rows):
  232. # type: (Iterable[Iterable[Any]]) -> Tuple[List[str], List[int]]
  233. """Return a list of formatted rows and a list of column sizes.
  234. For example::
  235. >>> tabulate([['foobar', 2000], [0xdeadbeef]])
  236. (['foobar 2000', '3735928559'], [10, 4])
  237. """
  238. rows = [tuple(map(str, row)) for row in rows]
  239. sizes = [max(map(len, col)) for col in zip_longest(*rows, fillvalue='')]
  240. table = [" ".join(map(str.ljust, row, sizes)).rstrip() for row in rows]
  241. return table, sizes
  242. def is_installable_dir(path):
  243. # type: (str) -> bool
  244. """Is path is a directory containing setup.py or pyproject.toml?
  245. """
  246. if not os.path.isdir(path):
  247. return False
  248. setup_py = os.path.join(path, 'setup.py')
  249. if os.path.isfile(setup_py):
  250. return True
  251. pyproject_toml = os.path.join(path, 'pyproject.toml')
  252. if os.path.isfile(pyproject_toml):
  253. return True
  254. return False
  255. def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE):
  256. """Yield pieces of data from a file-like object until EOF."""
  257. while True:
  258. chunk = file.read(size)
  259. if not chunk:
  260. break
  261. yield chunk
  262. def normalize_path(path, resolve_symlinks=True):
  263. # type: (str, bool) -> str
  264. """
  265. Convert a path to its canonical, case-normalized, absolute version.
  266. """
  267. path = expanduser(path)
  268. if resolve_symlinks:
  269. path = os.path.realpath(path)
  270. else:
  271. path = os.path.abspath(path)
  272. return os.path.normcase(path)
  273. def splitext(path):
  274. # type: (str) -> Tuple[str, str]
  275. """Like os.path.splitext, but take off .tar too"""
  276. base, ext = posixpath.splitext(path)
  277. if base.lower().endswith('.tar'):
  278. ext = base[-4:] + ext
  279. base = base[:-4]
  280. return base, ext
  281. def renames(old, new):
  282. # type: (str, str) -> None
  283. """Like os.renames(), but handles renaming across devices."""
  284. # Implementation borrowed from os.renames().
  285. head, tail = os.path.split(new)
  286. if head and tail and not os.path.exists(head):
  287. os.makedirs(head)
  288. shutil.move(old, new)
  289. head, tail = os.path.split(old)
  290. if head and tail:
  291. try:
  292. os.removedirs(head)
  293. except OSError:
  294. pass
  295. def is_local(path):
  296. # type: (str) -> bool
  297. """
  298. Return True if path is within sys.prefix, if we're running in a virtualenv.
  299. If we're not in a virtualenv, all paths are considered "local."
  300. Caution: this function assumes the head of path has been normalized
  301. with normalize_path.
  302. """
  303. if not running_under_virtualenv():
  304. return True
  305. return path.startswith(normalize_path(sys.prefix))
  306. def dist_is_local(dist):
  307. # type: (Distribution) -> bool
  308. """
  309. Return True if given Distribution object is installed locally
  310. (i.e. within current virtualenv).
  311. Always True if we're not in a virtualenv.
  312. """
  313. return is_local(dist_location(dist))
  314. def dist_in_usersite(dist):
  315. # type: (Distribution) -> bool
  316. """
  317. Return True if given Distribution is installed in user site.
  318. """
  319. return dist_location(dist).startswith(normalize_path(user_site))
  320. def dist_in_site_packages(dist):
  321. # type: (Distribution) -> bool
  322. """
  323. Return True if given Distribution is installed in
  324. sysconfig.get_python_lib().
  325. """
  326. return dist_location(dist).startswith(normalize_path(site_packages))
  327. def dist_is_editable(dist):
  328. # type: (Distribution) -> bool
  329. """
  330. Return True if given Distribution is an editable install.
  331. """
  332. for path_item in sys.path:
  333. egg_link = os.path.join(path_item, dist.project_name + '.egg-link')
  334. if os.path.isfile(egg_link):
  335. return True
  336. return False
  337. def get_installed_distributions(
  338. local_only=True, # type: bool
  339. skip=stdlib_pkgs, # type: Container[str]
  340. include_editables=True, # type: bool
  341. editables_only=False, # type: bool
  342. user_only=False, # type: bool
  343. paths=None # type: Optional[List[str]]
  344. ):
  345. # type: (...) -> List[Distribution]
  346. """
  347. Return a list of installed Distribution objects.
  348. If ``local_only`` is True (default), only return installations
  349. local to the current virtualenv, if in a virtualenv.
  350. ``skip`` argument is an iterable of lower-case project names to
  351. ignore; defaults to stdlib_pkgs
  352. If ``include_editables`` is False, don't report editables.
  353. If ``editables_only`` is True , only report editables.
  354. If ``user_only`` is True , only report installations in the user
  355. site directory.
  356. If ``paths`` is set, only report the distributions present at the
  357. specified list of locations.
  358. """
  359. if paths:
  360. working_set = pkg_resources.WorkingSet(paths)
  361. else:
  362. working_set = pkg_resources.working_set
  363. if local_only:
  364. local_test = dist_is_local
  365. else:
  366. def local_test(d):
  367. return True
  368. if include_editables:
  369. def editable_test(d):
  370. return True
  371. else:
  372. def editable_test(d):
  373. return not dist_is_editable(d)
  374. if editables_only:
  375. def editables_only_test(d):
  376. return dist_is_editable(d)
  377. else:
  378. def editables_only_test(d):
  379. return True
  380. if user_only:
  381. user_test = dist_in_usersite
  382. else:
  383. def user_test(d):
  384. return True
  385. return [d for d in working_set
  386. if local_test(d) and
  387. d.key not in skip and
  388. editable_test(d) and
  389. editables_only_test(d) and
  390. user_test(d)
  391. ]
  392. def search_distribution(req_name):
  393. # Canonicalize the name before searching in the list of
  394. # installed distributions and also while creating the package
  395. # dictionary to get the Distribution object
  396. req_name = canonicalize_name(req_name)
  397. packages = get_installed_distributions(skip=())
  398. pkg_dict = {canonicalize_name(p.key): p for p in packages}
  399. return pkg_dict.get(req_name)
  400. def get_distribution(req_name):
  401. """Given a requirement name, return the installed Distribution object"""
  402. # Search the distribution by looking through the working set
  403. dist = search_distribution(req_name)
  404. # If distribution could not be found, call working_set.require
  405. # to update the working set, and try to find the distribution
  406. # again.
  407. # This might happen for e.g. when you install a package
  408. # twice, once using setup.py develop and again using setup.py install.
  409. # Now when run pip uninstall twice, the package gets removed
  410. # from the working set in the first uninstall, so we have to populate
  411. # the working set again so that pip knows about it and the packages
  412. # gets picked up and is successfully uninstalled the second time too.
  413. if not dist:
  414. try:
  415. pkg_resources.working_set.require(req_name)
  416. except pkg_resources.DistributionNotFound:
  417. return None
  418. return search_distribution(req_name)
  419. def egg_link_path(dist):
  420. # type: (Distribution) -> Optional[str]
  421. """
  422. Return the path for the .egg-link file if it exists, otherwise, None.
  423. There's 3 scenarios:
  424. 1) not in a virtualenv
  425. try to find in site.USER_SITE, then site_packages
  426. 2) in a no-global virtualenv
  427. try to find in site_packages
  428. 3) in a yes-global virtualenv
  429. try to find in site_packages, then site.USER_SITE
  430. (don't look in global location)
  431. For #1 and #3, there could be odd cases, where there's an egg-link in 2
  432. locations.
  433. This method will just return the first one found.
  434. """
  435. sites = []
  436. if running_under_virtualenv():
  437. sites.append(site_packages)
  438. if not virtualenv_no_global() and user_site:
  439. sites.append(user_site)
  440. else:
  441. if user_site:
  442. sites.append(user_site)
  443. sites.append(site_packages)
  444. for site in sites:
  445. egglink = os.path.join(site, dist.project_name) + '.egg-link'
  446. if os.path.isfile(egglink):
  447. return egglink
  448. return None
  449. def dist_location(dist):
  450. # type: (Distribution) -> str
  451. """
  452. Get the site-packages location of this distribution. Generally
  453. this is dist.location, except in the case of develop-installed
  454. packages, where dist.location is the source code location, and we
  455. want to know where the egg-link file is.
  456. The returned location is normalized (in particular, with symlinks removed).
  457. """
  458. egg_link = egg_link_path(dist)
  459. if egg_link:
  460. return normalize_path(egg_link)
  461. return normalize_path(dist.location)
  462. def write_output(msg, *args):
  463. # type: (Any, Any) -> None
  464. logger.info(msg, *args)
  465. class FakeFile(object):
  466. """Wrap a list of lines in an object with readline() to make
  467. ConfigParser happy."""
  468. def __init__(self, lines):
  469. self._gen = iter(lines)
  470. def readline(self):
  471. try:
  472. return next(self._gen)
  473. except StopIteration:
  474. return ''
  475. def __iter__(self):
  476. return self._gen
  477. class StreamWrapper(StringIO):
  478. @classmethod
  479. def from_stream(cls, orig_stream):
  480. cls.orig_stream = orig_stream
  481. return cls()
  482. # compileall.compile_dir() needs stdout.encoding to print to stdout
  483. @property
  484. def encoding(self):
  485. return self.orig_stream.encoding
  486. @contextlib.contextmanager
  487. def captured_output(stream_name):
  488. """Return a context manager used by captured_stdout/stdin/stderr
  489. that temporarily replaces the sys stream *stream_name* with a StringIO.
  490. Taken from Lib/support/__init__.py in the CPython repo.
  491. """
  492. orig_stdout = getattr(sys, stream_name)
  493. setattr(sys, stream_name, StreamWrapper.from_stream(orig_stdout))
  494. try:
  495. yield getattr(sys, stream_name)
  496. finally:
  497. setattr(sys, stream_name, orig_stdout)
  498. def captured_stdout():
  499. """Capture the output of sys.stdout:
  500. with captured_stdout() as stdout:
  501. print('hello')
  502. self.assertEqual(stdout.getvalue(), 'hello\n')
  503. Taken from Lib/support/__init__.py in the CPython repo.
  504. """
  505. return captured_output('stdout')
  506. def captured_stderr():
  507. """
  508. See captured_stdout().
  509. """
  510. return captured_output('stderr')
  511. def get_installed_version(dist_name, working_set=None):
  512. """Get the installed version of dist_name avoiding pkg_resources cache"""
  513. # Create a requirement that we'll look for inside of setuptools.
  514. req = pkg_resources.Requirement.parse(dist_name)
  515. if working_set is None:
  516. # We want to avoid having this cached, so we need to construct a new
  517. # working set each time.
  518. working_set = pkg_resources.WorkingSet()
  519. # Get the installed distribution from our working set
  520. dist = working_set.find(req)
  521. # Check to see if we got an installed distribution or not, if we did
  522. # we want to return it's version.
  523. return dist.version if dist else None
  524. def consume(iterator):
  525. """Consume an iterable at C speed."""
  526. deque(iterator, maxlen=0)
  527. # Simulates an enum
  528. def enum(*sequential, **named):
  529. enums = dict(zip(sequential, range(len(sequential))), **named)
  530. reverse = {value: key for key, value in enums.items()}
  531. enums['reverse_mapping'] = reverse
  532. return type('Enum', (), enums)
  533. def build_netloc(host, port):
  534. # type: (str, Optional[int]) -> str
  535. """
  536. Build a netloc from a host-port pair
  537. """
  538. if port is None:
  539. return host
  540. if ':' in host:
  541. # Only wrap host with square brackets when it is IPv6
  542. host = '[{}]'.format(host)
  543. return '{}:{}'.format(host, port)
  544. def build_url_from_netloc(netloc, scheme='https'):
  545. # type: (str, str) -> str
  546. """
  547. Build a full URL from a netloc.
  548. """
  549. if netloc.count(':') >= 2 and '@' not in netloc and '[' not in netloc:
  550. # It must be a bare IPv6 address, so wrap it with brackets.
  551. netloc = '[{}]'.format(netloc)
  552. return '{}://{}'.format(scheme, netloc)
  553. def parse_netloc(netloc):
  554. # type: (str) -> Tuple[str, Optional[int]]
  555. """
  556. Return the host-port pair from a netloc.
  557. """
  558. url = build_url_from_netloc(netloc)
  559. parsed = urllib_parse.urlparse(url)
  560. return parsed.hostname, parsed.port
  561. def split_auth_from_netloc(netloc):
  562. """
  563. Parse out and remove the auth information from a netloc.
  564. Returns: (netloc, (username, password)).
  565. """
  566. if '@' not in netloc:
  567. return netloc, (None, None)
  568. # Split from the right because that's how urllib.parse.urlsplit()
  569. # behaves if more than one @ is present (which can be checked using
  570. # the password attribute of urlsplit()'s return value).
  571. auth, netloc = netloc.rsplit('@', 1)
  572. if ':' in auth:
  573. # Split from the left because that's how urllib.parse.urlsplit()
  574. # behaves if more than one : is present (which again can be checked
  575. # using the password attribute of the return value)
  576. user_pass = auth.split(':', 1)
  577. else:
  578. user_pass = auth, None
  579. user_pass = tuple(
  580. None if x is None else urllib_unquote(x) for x in user_pass
  581. )
  582. return netloc, user_pass
  583. def redact_netloc(netloc):
  584. # type: (str) -> str
  585. """
  586. Replace the sensitive data in a netloc with "****", if it exists.
  587. For example:
  588. - "user:pass@example.com" returns "user:****@example.com"
  589. - "accesstoken@example.com" returns "****@example.com"
  590. """
  591. netloc, (user, password) = split_auth_from_netloc(netloc)
  592. if user is None:
  593. return netloc
  594. if password is None:
  595. user = '****'
  596. password = ''
  597. else:
  598. user = urllib_parse.quote(user)
  599. password = ':****'
  600. return '{user}{password}@{netloc}'.format(user=user,
  601. password=password,
  602. netloc=netloc)
  603. def _transform_url(url, transform_netloc):
  604. """Transform and replace netloc in a url.
  605. transform_netloc is a function taking the netloc and returning a
  606. tuple. The first element of this tuple is the new netloc. The
  607. entire tuple is returned.
  608. Returns a tuple containing the transformed url as item 0 and the
  609. original tuple returned by transform_netloc as item 1.
  610. """
  611. purl = urllib_parse.urlsplit(url)
  612. netloc_tuple = transform_netloc(purl.netloc)
  613. # stripped url
  614. url_pieces = (
  615. purl.scheme, netloc_tuple[0], purl.path, purl.query, purl.fragment
  616. )
  617. surl = urllib_parse.urlunsplit(url_pieces)
  618. return surl, netloc_tuple
  619. def _get_netloc(netloc):
  620. return split_auth_from_netloc(netloc)
  621. def _redact_netloc(netloc):
  622. return (redact_netloc(netloc),)
  623. def split_auth_netloc_from_url(url):
  624. # type: (str) -> Tuple[str, str, Tuple[str, str]]
  625. """
  626. Parse a url into separate netloc, auth, and url with no auth.
  627. Returns: (url_without_auth, netloc, (username, password))
  628. """
  629. url_without_auth, (netloc, auth) = _transform_url(url, _get_netloc)
  630. return url_without_auth, netloc, auth
  631. def remove_auth_from_url(url):
  632. # type: (str) -> str
  633. """Return a copy of url with 'username:password@' removed."""
  634. # username/pass params are passed to subversion through flags
  635. # and are not recognized in the url.
  636. return _transform_url(url, _get_netloc)[0]
  637. def redact_auth_from_url(url):
  638. # type: (str) -> str
  639. """Replace the password in a given url with ****."""
  640. return _transform_url(url, _redact_netloc)[0]
  641. class HiddenText(object):
  642. def __init__(
  643. self,
  644. secret, # type: str
  645. redacted, # type: str
  646. ):
  647. # type: (...) -> None
  648. self.secret = secret
  649. self.redacted = redacted
  650. def __repr__(self):
  651. # type: (...) -> str
  652. return '<HiddenText {!r}>'.format(str(self))
  653. def __str__(self):
  654. # type: (...) -> str
  655. return self.redacted
  656. # This is useful for testing.
  657. def __eq__(self, other):
  658. # type: (Any) -> bool
  659. if type(self) != type(other):
  660. return False
  661. # The string being used for redaction doesn't also have to match,
  662. # just the raw, original string.
  663. return (self.secret == other.secret)
  664. # We need to provide an explicit __ne__ implementation for Python 2.
  665. # TODO: remove this when we drop PY2 support.
  666. def __ne__(self, other):
  667. # type: (Any) -> bool
  668. return not self == other
  669. def hide_value(value):
  670. # type: (str) -> HiddenText
  671. return HiddenText(value, redacted='****')
  672. def hide_url(url):
  673. # type: (str) -> HiddenText
  674. redacted = redact_auth_from_url(url)
  675. return HiddenText(url, redacted=redacted)
  676. def protect_pip_from_modification_on_windows(modifying_pip):
  677. # type: (bool) -> None
  678. """Protection of pip.exe from modification on Windows
  679. On Windows, any operation modifying pip should be run as:
  680. python -m pip ...
  681. """
  682. pip_names = [
  683. "pip.exe",
  684. "pip{}.exe".format(sys.version_info[0]),
  685. "pip{}.{}.exe".format(*sys.version_info[:2])
  686. ]
  687. # See https://github.com/pypa/pip/issues/1299 for more discussion
  688. should_show_use_python_msg = (
  689. modifying_pip and
  690. WINDOWS and
  691. os.path.basename(sys.argv[0]) in pip_names
  692. )
  693. if should_show_use_python_msg:
  694. new_command = [
  695. sys.executable, "-m", "pip"
  696. ] + sys.argv[1:]
  697. raise CommandError(
  698. 'To modify pip, please run the following command:\n{}'
  699. .format(" ".join(new_command))
  700. )
  701. def is_console_interactive():
  702. # type: () -> bool
  703. """Is this console interactive?
  704. """
  705. return sys.stdin is not None and sys.stdin.isatty()
  706. def hash_file(path, blocksize=1 << 20):
  707. # type: (Text, int) -> Tuple[Any, int]
  708. """Return (hash, length) for path using hashlib.sha256()
  709. """
  710. h = hashlib.sha256()
  711. length = 0
  712. with open(path, 'rb') as f:
  713. for block in read_chunks(f, size=blocksize):
  714. length += len(block)
  715. h.update(block)
  716. return h, length
  717. def is_wheel_installed():
  718. """
  719. Return whether the wheel package is installed.
  720. """
  721. try:
  722. import wheel # noqa: F401
  723. except ImportError:
  724. return False
  725. return True
  726. def pairwise(iterable):
  727. # type: (Iterable[Any]) -> Iterator[Tuple[Any, Any]]
  728. """
  729. Return paired elements.
  730. For example:
  731. s -> (s0, s1), (s2, s3), (s4, s5), ...
  732. """
  733. iterable = iter(iterable)
  734. return zip_longest(iterable, iterable)
  735. def partition(
  736. pred, # type: Callable[[T], bool]
  737. iterable, # type: Iterable[T]
  738. ):
  739. # type: (...) -> Tuple[Iterable[T], Iterable[T]]
  740. """
  741. Use a predicate to partition entries into false entries and true entries,
  742. like
  743. partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9
  744. """
  745. t1, t2 = tee(iterable)
  746. return filterfalse(pred, t1), filter(pred, t2)