git.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394
  1. # The following comment should be removed at some point in the future.
  2. # mypy: disallow-untyped-defs=False
  3. from __future__ import absolute_import
  4. import logging
  5. import os.path
  6. import re
  7. from pip._vendor.packaging.version import parse as parse_version
  8. from pip._vendor.six.moves.urllib import parse as urllib_parse
  9. from pip._vendor.six.moves.urllib import request as urllib_request
  10. from pip._internal.exceptions import BadCommand, InstallationError
  11. from pip._internal.utils.misc import display_path, hide_url
  12. from pip._internal.utils.subprocess import make_command
  13. from pip._internal.utils.temp_dir import TempDirectory
  14. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  15. from pip._internal.vcs.versioncontrol import (
  16. RemoteNotFoundError,
  17. VersionControl,
  18. find_path_to_setup_from_repo_root,
  19. vcs,
  20. )
  21. if MYPY_CHECK_RUNNING:
  22. from typing import Optional, Tuple
  23. from pip._internal.utils.misc import HiddenText
  24. from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions
  25. urlsplit = urllib_parse.urlsplit
  26. urlunsplit = urllib_parse.urlunsplit
  27. logger = logging.getLogger(__name__)
  28. HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$')
  29. def looks_like_hash(sha):
  30. return bool(HASH_REGEX.match(sha))
  31. class Git(VersionControl):
  32. name = 'git'
  33. dirname = '.git'
  34. repo_name = 'clone'
  35. schemes = (
  36. 'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file',
  37. )
  38. # Prevent the user's environment variables from interfering with pip:
  39. # https://github.com/pypa/pip/issues/1130
  40. unset_environ = ('GIT_DIR', 'GIT_WORK_TREE')
  41. default_arg_rev = 'HEAD'
  42. @staticmethod
  43. def get_base_rev_args(rev):
  44. return [rev]
  45. def is_immutable_rev_checkout(self, url, dest):
  46. # type: (str, str) -> bool
  47. _, rev_options = self.get_url_rev_options(hide_url(url))
  48. if not rev_options.rev:
  49. return False
  50. if not self.is_commit_id_equal(dest, rev_options.rev):
  51. # the current commit is different from rev,
  52. # which means rev was something else than a commit hash
  53. return False
  54. # return False in the rare case rev is both a commit hash
  55. # and a tag or a branch; we don't want to cache in that case
  56. # because that branch/tag could point to something else in the future
  57. is_tag_or_branch = bool(
  58. self.get_revision_sha(dest, rev_options.rev)[0]
  59. )
  60. return not is_tag_or_branch
  61. def get_git_version(self):
  62. VERSION_PFX = 'git version '
  63. version = self.run_command(['version'], show_stdout=False)
  64. if version.startswith(VERSION_PFX):
  65. version = version[len(VERSION_PFX):].split()[0]
  66. else:
  67. version = ''
  68. # get first 3 positions of the git version because
  69. # on windows it is x.y.z.windows.t, and this parses as
  70. # LegacyVersion which always smaller than a Version.
  71. version = '.'.join(version.split('.')[:3])
  72. return parse_version(version)
  73. @classmethod
  74. def get_current_branch(cls, location):
  75. """
  76. Return the current branch, or None if HEAD isn't at a branch
  77. (e.g. detached HEAD).
  78. """
  79. # git-symbolic-ref exits with empty stdout if "HEAD" is a detached
  80. # HEAD rather than a symbolic ref. In addition, the -q causes the
  81. # command to exit with status code 1 instead of 128 in this case
  82. # and to suppress the message to stderr.
  83. args = ['symbolic-ref', '-q', 'HEAD']
  84. output = cls.run_command(
  85. args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location,
  86. )
  87. ref = output.strip()
  88. if ref.startswith('refs/heads/'):
  89. return ref[len('refs/heads/'):]
  90. return None
  91. def export(self, location, url):
  92. # type: (str, HiddenText) -> None
  93. """Export the Git repository at the url to the destination location"""
  94. if not location.endswith('/'):
  95. location = location + '/'
  96. with TempDirectory(kind="export") as temp_dir:
  97. self.unpack(temp_dir.path, url=url)
  98. self.run_command(
  99. ['checkout-index', '-a', '-f', '--prefix', location],
  100. show_stdout=False, cwd=temp_dir.path
  101. )
  102. @classmethod
  103. def get_revision_sha(cls, dest, rev):
  104. """
  105. Return (sha_or_none, is_branch), where sha_or_none is a commit hash
  106. if the revision names a remote branch or tag, otherwise None.
  107. Args:
  108. dest: the repository directory.
  109. rev: the revision name.
  110. """
  111. # Pass rev to pre-filter the list.
  112. output = cls.run_command(['show-ref', rev], cwd=dest,
  113. show_stdout=False, on_returncode='ignore')
  114. refs = {}
  115. for line in output.strip().splitlines():
  116. try:
  117. sha, ref = line.split()
  118. except ValueError:
  119. # Include the offending line to simplify troubleshooting if
  120. # this error ever occurs.
  121. raise ValueError('unexpected show-ref line: {!r}'.format(line))
  122. refs[ref] = sha
  123. branch_ref = 'refs/remotes/origin/{}'.format(rev)
  124. tag_ref = 'refs/tags/{}'.format(rev)
  125. sha = refs.get(branch_ref)
  126. if sha is not None:
  127. return (sha, True)
  128. sha = refs.get(tag_ref)
  129. return (sha, False)
  130. @classmethod
  131. def resolve_revision(cls, dest, url, rev_options):
  132. # type: (str, HiddenText, RevOptions) -> RevOptions
  133. """
  134. Resolve a revision to a new RevOptions object with the SHA1 of the
  135. branch, tag, or ref if found.
  136. Args:
  137. rev_options: a RevOptions object.
  138. """
  139. rev = rev_options.arg_rev
  140. # The arg_rev property's implementation for Git ensures that the
  141. # rev return value is always non-None.
  142. assert rev is not None
  143. sha, is_branch = cls.get_revision_sha(dest, rev)
  144. if sha is not None:
  145. rev_options = rev_options.make_new(sha)
  146. rev_options.branch_name = rev if is_branch else None
  147. return rev_options
  148. # Do not show a warning for the common case of something that has
  149. # the form of a Git commit hash.
  150. if not looks_like_hash(rev):
  151. logger.warning(
  152. "Did not find branch or tag '%s', assuming revision or ref.",
  153. rev,
  154. )
  155. if not rev.startswith('refs/'):
  156. return rev_options
  157. # If it looks like a ref, we have to fetch it explicitly.
  158. cls.run_command(
  159. make_command('fetch', '-q', url, rev_options.to_args()),
  160. cwd=dest,
  161. )
  162. # Change the revision to the SHA of the ref we fetched
  163. sha = cls.get_revision(dest, rev='FETCH_HEAD')
  164. rev_options = rev_options.make_new(sha)
  165. return rev_options
  166. @classmethod
  167. def is_commit_id_equal(cls, dest, name):
  168. """
  169. Return whether the current commit hash equals the given name.
  170. Args:
  171. dest: the repository directory.
  172. name: a string name.
  173. """
  174. if not name:
  175. # Then avoid an unnecessary subprocess call.
  176. return False
  177. return cls.get_revision(dest) == name
  178. def fetch_new(self, dest, url, rev_options):
  179. # type: (str, HiddenText, RevOptions) -> None
  180. rev_display = rev_options.to_display()
  181. logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest))
  182. self.run_command(make_command('clone', '-q', url, dest))
  183. if rev_options.rev:
  184. # Then a specific revision was requested.
  185. rev_options = self.resolve_revision(dest, url, rev_options)
  186. branch_name = getattr(rev_options, 'branch_name', None)
  187. if branch_name is None:
  188. # Only do a checkout if the current commit id doesn't match
  189. # the requested revision.
  190. if not self.is_commit_id_equal(dest, rev_options.rev):
  191. cmd_args = make_command(
  192. 'checkout', '-q', rev_options.to_args(),
  193. )
  194. self.run_command(cmd_args, cwd=dest)
  195. elif self.get_current_branch(dest) != branch_name:
  196. # Then a specific branch was requested, and that branch
  197. # is not yet checked out.
  198. track_branch = 'origin/{}'.format(branch_name)
  199. cmd_args = [
  200. 'checkout', '-b', branch_name, '--track', track_branch,
  201. ]
  202. self.run_command(cmd_args, cwd=dest)
  203. #: repo may contain submodules
  204. self.update_submodules(dest)
  205. def switch(self, dest, url, rev_options):
  206. # type: (str, HiddenText, RevOptions) -> None
  207. self.run_command(
  208. make_command('config', 'remote.origin.url', url),
  209. cwd=dest,
  210. )
  211. cmd_args = make_command('checkout', '-q', rev_options.to_args())
  212. self.run_command(cmd_args, cwd=dest)
  213. self.update_submodules(dest)
  214. def update(self, dest, url, rev_options):
  215. # type: (str, HiddenText, RevOptions) -> None
  216. # First fetch changes from the default remote
  217. if self.get_git_version() >= parse_version('1.9.0'):
  218. # fetch tags in addition to everything else
  219. self.run_command(['fetch', '-q', '--tags'], cwd=dest)
  220. else:
  221. self.run_command(['fetch', '-q'], cwd=dest)
  222. # Then reset to wanted revision (maybe even origin/master)
  223. rev_options = self.resolve_revision(dest, url, rev_options)
  224. cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args())
  225. self.run_command(cmd_args, cwd=dest)
  226. #: update submodules
  227. self.update_submodules(dest)
  228. @classmethod
  229. def get_remote_url(cls, location):
  230. """
  231. Return URL of the first remote encountered.
  232. Raises RemoteNotFoundError if the repository does not have a remote
  233. url configured.
  234. """
  235. # We need to pass 1 for extra_ok_returncodes since the command
  236. # exits with return code 1 if there are no matching lines.
  237. stdout = cls.run_command(
  238. ['config', '--get-regexp', r'remote\..*\.url'],
  239. extra_ok_returncodes=(1, ), show_stdout=False, cwd=location,
  240. )
  241. remotes = stdout.splitlines()
  242. try:
  243. found_remote = remotes[0]
  244. except IndexError:
  245. raise RemoteNotFoundError
  246. for remote in remotes:
  247. if remote.startswith('remote.origin.url '):
  248. found_remote = remote
  249. break
  250. url = found_remote.split(' ')[1]
  251. return url.strip()
  252. @classmethod
  253. def get_revision(cls, location, rev=None):
  254. if rev is None:
  255. rev = 'HEAD'
  256. current_rev = cls.run_command(
  257. ['rev-parse', rev], show_stdout=False, cwd=location,
  258. )
  259. return current_rev.strip()
  260. @classmethod
  261. def get_subdirectory(cls, location):
  262. """
  263. Return the path to setup.py, relative to the repo root.
  264. Return None if setup.py is in the repo root.
  265. """
  266. # find the repo root
  267. git_dir = cls.run_command(
  268. ['rev-parse', '--git-dir'],
  269. show_stdout=False, cwd=location).strip()
  270. if not os.path.isabs(git_dir):
  271. git_dir = os.path.join(location, git_dir)
  272. repo_root = os.path.abspath(os.path.join(git_dir, '..'))
  273. return find_path_to_setup_from_repo_root(location, repo_root)
  274. @classmethod
  275. def get_url_rev_and_auth(cls, url):
  276. # type: (str) -> Tuple[str, Optional[str], AuthInfo]
  277. """
  278. Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'.
  279. That's required because although they use SSH they sometimes don't
  280. work with a ssh:// scheme (e.g. GitHub). But we need a scheme for
  281. parsing. Hence we remove it again afterwards and return it as a stub.
  282. """
  283. # Works around an apparent Git bug
  284. # (see https://article.gmane.org/gmane.comp.version-control.git/146500)
  285. scheme, netloc, path, query, fragment = urlsplit(url)
  286. if scheme.endswith('file'):
  287. initial_slashes = path[:-len(path.lstrip('/'))]
  288. newpath = (
  289. initial_slashes +
  290. urllib_request.url2pathname(path)
  291. .replace('\\', '/').lstrip('/')
  292. )
  293. url = urlunsplit((scheme, netloc, newpath, query, fragment))
  294. after_plus = scheme.find('+') + 1
  295. url = scheme[:after_plus] + urlunsplit(
  296. (scheme[after_plus:], netloc, newpath, query, fragment),
  297. )
  298. if '://' not in url:
  299. assert 'file:' not in url
  300. url = url.replace('git+', 'git+ssh://')
  301. url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url)
  302. url = url.replace('ssh://', '')
  303. else:
  304. url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url)
  305. return url, rev, user_pass
  306. @classmethod
  307. def update_submodules(cls, location):
  308. if not os.path.exists(os.path.join(location, '.gitmodules')):
  309. return
  310. cls.run_command(
  311. ['submodule', 'update', '--init', '--recursive', '-q'],
  312. cwd=location,
  313. )
  314. @classmethod
  315. def get_repository_root(cls, location):
  316. loc = super(Git, cls).get_repository_root(location)
  317. if loc:
  318. return loc
  319. try:
  320. r = cls.run_command(
  321. ['rev-parse', '--show-toplevel'],
  322. cwd=location,
  323. show_stdout=False,
  324. on_returncode='raise',
  325. log_failed_cmd=False,
  326. )
  327. except BadCommand:
  328. logger.debug("could not determine if %s is under git control "
  329. "because git is not available", location)
  330. return None
  331. except InstallationError:
  332. return None
  333. return os.path.normpath(r.rstrip('\r\n'))
  334. vcs.register(Git)