wheel_builder.py 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309
  1. """Orchestrator for building wheels from InstallRequirements.
  2. """
  3. # The following comment should be removed at some point in the future.
  4. # mypy: strict-optional=False
  5. import logging
  6. import os.path
  7. import re
  8. import shutil
  9. from pip._internal.models.link import Link
  10. from pip._internal.operations.build.wheel import build_wheel_pep517
  11. from pip._internal.operations.build.wheel_legacy import build_wheel_legacy
  12. from pip._internal.utils.logging import indent_log
  13. from pip._internal.utils.misc import ensure_dir, hash_file, is_wheel_installed
  14. from pip._internal.utils.setuptools_build import make_setuptools_clean_args
  15. from pip._internal.utils.subprocess import call_subprocess
  16. from pip._internal.utils.temp_dir import TempDirectory
  17. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  18. from pip._internal.utils.urls import path_to_url
  19. from pip._internal.vcs import vcs
  20. if MYPY_CHECK_RUNNING:
  21. from typing import (
  22. Any, Callable, Iterable, List, Optional, Pattern, Tuple,
  23. )
  24. from pip._internal.cache import WheelCache
  25. from pip._internal.req.req_install import InstallRequirement
  26. BinaryAllowedPredicate = Callable[[InstallRequirement], bool]
  27. BuildResult = Tuple[List[InstallRequirement], List[InstallRequirement]]
  28. logger = logging.getLogger(__name__)
  29. def _contains_egg_info(
  30. s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)):
  31. # type: (str, Pattern[str]) -> bool
  32. """Determine whether the string looks like an egg_info.
  33. :param s: The string to parse. E.g. foo-2.1
  34. """
  35. return bool(_egg_info_re.search(s))
  36. def _should_build(
  37. req, # type: InstallRequirement
  38. need_wheel, # type: bool
  39. check_binary_allowed, # type: BinaryAllowedPredicate
  40. ):
  41. # type: (...) -> bool
  42. """Return whether an InstallRequirement should be built into a wheel."""
  43. if req.constraint:
  44. # never build requirements that are merely constraints
  45. return False
  46. if req.is_wheel:
  47. if need_wheel:
  48. logger.info(
  49. 'Skipping %s, due to already being wheel.', req.name,
  50. )
  51. return False
  52. if need_wheel:
  53. # i.e. pip wheel, not pip install
  54. return True
  55. # From this point, this concerns the pip install command only
  56. # (need_wheel=False).
  57. if req.editable or not req.source_dir:
  58. return False
  59. if not check_binary_allowed(req):
  60. logger.info(
  61. "Skipping wheel build for %s, due to binaries "
  62. "being disabled for it.", req.name,
  63. )
  64. return False
  65. if not req.use_pep517 and not is_wheel_installed():
  66. # we don't build legacy requirements if wheel is not installed
  67. logger.info(
  68. "Using legacy setup.py install for %s, "
  69. "since package 'wheel' is not installed.", req.name,
  70. )
  71. return False
  72. return True
  73. def should_build_for_wheel_command(
  74. req, # type: InstallRequirement
  75. ):
  76. # type: (...) -> bool
  77. return _should_build(
  78. req, need_wheel=True, check_binary_allowed=_always_true
  79. )
  80. def should_build_for_install_command(
  81. req, # type: InstallRequirement
  82. check_binary_allowed, # type: BinaryAllowedPredicate
  83. ):
  84. # type: (...) -> bool
  85. return _should_build(
  86. req, need_wheel=False, check_binary_allowed=check_binary_allowed
  87. )
  88. def _should_cache(
  89. req, # type: InstallRequirement
  90. ):
  91. # type: (...) -> Optional[bool]
  92. """
  93. Return whether a built InstallRequirement can be stored in the persistent
  94. wheel cache, assuming the wheel cache is available, and _should_build()
  95. has determined a wheel needs to be built.
  96. """
  97. if not should_build_for_install_command(
  98. req, check_binary_allowed=_always_true
  99. ):
  100. # never cache if pip install would not have built
  101. # (editable mode, etc)
  102. return False
  103. if req.link and req.link.is_vcs:
  104. # VCS checkout. Do not cache
  105. # unless it points to an immutable commit hash.
  106. assert not req.editable
  107. assert req.source_dir
  108. vcs_backend = vcs.get_backend_for_scheme(req.link.scheme)
  109. assert vcs_backend
  110. if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir):
  111. return True
  112. return False
  113. base, ext = req.link.splitext()
  114. if _contains_egg_info(base):
  115. return True
  116. # Otherwise, do not cache.
  117. return False
  118. def _get_cache_dir(
  119. req, # type: InstallRequirement
  120. wheel_cache, # type: WheelCache
  121. ):
  122. # type: (...) -> str
  123. """Return the persistent or temporary cache directory where the built
  124. wheel need to be stored.
  125. """
  126. cache_available = bool(wheel_cache.cache_dir)
  127. if cache_available and _should_cache(req):
  128. cache_dir = wheel_cache.get_path_for_link(req.link)
  129. else:
  130. cache_dir = wheel_cache.get_ephem_path_for_link(req.link)
  131. return cache_dir
  132. def _always_true(_):
  133. # type: (Any) -> bool
  134. return True
  135. def _build_one(
  136. req, # type: InstallRequirement
  137. output_dir, # type: str
  138. build_options, # type: List[str]
  139. global_options, # type: List[str]
  140. ):
  141. # type: (...) -> Optional[str]
  142. """Build one wheel.
  143. :return: The filename of the built wheel, or None if the build failed.
  144. """
  145. try:
  146. ensure_dir(output_dir)
  147. except OSError as e:
  148. logger.warning(
  149. "Building wheel for %s failed: %s",
  150. req.name, e,
  151. )
  152. return None
  153. # Install build deps into temporary directory (PEP 518)
  154. with req.build_env:
  155. return _build_one_inside_env(
  156. req, output_dir, build_options, global_options
  157. )
  158. def _build_one_inside_env(
  159. req, # type: InstallRequirement
  160. output_dir, # type: str
  161. build_options, # type: List[str]
  162. global_options, # type: List[str]
  163. ):
  164. # type: (...) -> Optional[str]
  165. with TempDirectory(kind="wheel") as temp_dir:
  166. if req.use_pep517:
  167. wheel_path = build_wheel_pep517(
  168. name=req.name,
  169. backend=req.pep517_backend,
  170. metadata_directory=req.metadata_directory,
  171. build_options=build_options,
  172. tempd=temp_dir.path,
  173. )
  174. else:
  175. wheel_path = build_wheel_legacy(
  176. name=req.name,
  177. setup_py_path=req.setup_py_path,
  178. source_dir=req.unpacked_source_directory,
  179. global_options=global_options,
  180. build_options=build_options,
  181. tempd=temp_dir.path,
  182. )
  183. if wheel_path is not None:
  184. wheel_name = os.path.basename(wheel_path)
  185. dest_path = os.path.join(output_dir, wheel_name)
  186. try:
  187. wheel_hash, length = hash_file(wheel_path)
  188. shutil.move(wheel_path, dest_path)
  189. logger.info('Created wheel for %s: '
  190. 'filename=%s size=%d sha256=%s',
  191. req.name, wheel_name, length,
  192. wheel_hash.hexdigest())
  193. logger.info('Stored in directory: %s', output_dir)
  194. return dest_path
  195. except Exception as e:
  196. logger.warning(
  197. "Building wheel for %s failed: %s",
  198. req.name, e,
  199. )
  200. # Ignore return, we can't do anything else useful.
  201. if not req.use_pep517:
  202. _clean_one_legacy(req, global_options)
  203. return None
  204. def _clean_one_legacy(req, global_options):
  205. # type: (InstallRequirement, List[str]) -> bool
  206. clean_args = make_setuptools_clean_args(
  207. req.setup_py_path,
  208. global_options=global_options,
  209. )
  210. logger.info('Running setup.py clean for %s', req.name)
  211. try:
  212. call_subprocess(clean_args, cwd=req.source_dir)
  213. return True
  214. except Exception:
  215. logger.error('Failed cleaning build dir for %s', req.name)
  216. return False
  217. def build(
  218. requirements, # type: Iterable[InstallRequirement]
  219. wheel_cache, # type: WheelCache
  220. build_options, # type: List[str]
  221. global_options, # type: List[str]
  222. ):
  223. # type: (...) -> BuildResult
  224. """Build wheels.
  225. :return: The list of InstallRequirement that succeeded to build and
  226. the list of InstallRequirement that failed to build.
  227. """
  228. if not requirements:
  229. return [], []
  230. # Build the wheels.
  231. logger.info(
  232. 'Building wheels for collected packages: %s',
  233. ', '.join(req.name for req in requirements),
  234. )
  235. with indent_log():
  236. build_successes, build_failures = [], []
  237. for req in requirements:
  238. cache_dir = _get_cache_dir(req, wheel_cache)
  239. wheel_file = _build_one(
  240. req, cache_dir, build_options, global_options
  241. )
  242. if wheel_file:
  243. # Update the link for this.
  244. req.link = Link(path_to_url(wheel_file))
  245. req.local_file_path = req.link.file_path
  246. assert req.link.is_wheel
  247. build_successes.append(req)
  248. else:
  249. build_failures.append(req)
  250. # notify success/failure
  251. if build_successes:
  252. logger.info(
  253. 'Successfully built %s',
  254. ' '.join([req.name for req in build_successes]),
  255. )
  256. if build_failures:
  257. logger.info(
  258. 'Failed to build %s',
  259. ' '.join([req.name for req in build_failures]),
  260. )
  261. # Return a list of requirements that failed to build
  262. return build_successes, build_failures