wheel.py 40 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2013-2017 Vinay Sajip.
  4. # Licensed to the Python Software Foundation under a contributor agreement.
  5. # See LICENSE.txt and CONTRIBUTORS.txt.
  6. #
  7. from __future__ import unicode_literals
  8. import base64
  9. import codecs
  10. import datetime
  11. import distutils.util
  12. from email import message_from_file
  13. import hashlib
  14. import imp
  15. import json
  16. import logging
  17. import os
  18. import posixpath
  19. import re
  20. import shutil
  21. import sys
  22. import tempfile
  23. import zipfile
  24. from . import __version__, DistlibException
  25. from .compat import sysconfig, ZipFile, fsdecode, text_type, filter
  26. from .database import InstalledDistribution
  27. from .metadata import Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME
  28. from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache,
  29. cached_property, get_cache_base, read_exports, tempdir)
  30. from .version import NormalizedVersion, UnsupportedVersionError
  31. logger = logging.getLogger(__name__)
  32. cache = None # created when needed
  33. if hasattr(sys, 'pypy_version_info'): # pragma: no cover
  34. IMP_PREFIX = 'pp'
  35. elif sys.platform.startswith('java'): # pragma: no cover
  36. IMP_PREFIX = 'jy'
  37. elif sys.platform == 'cli': # pragma: no cover
  38. IMP_PREFIX = 'ip'
  39. else:
  40. IMP_PREFIX = 'cp'
  41. VER_SUFFIX = sysconfig.get_config_var('py_version_nodot')
  42. if not VER_SUFFIX: # pragma: no cover
  43. VER_SUFFIX = '%s%s' % sys.version_info[:2]
  44. PYVER = 'py' + VER_SUFFIX
  45. IMPVER = IMP_PREFIX + VER_SUFFIX
  46. ARCH = distutils.util.get_platform().replace('-', '_').replace('.', '_')
  47. ABI = sysconfig.get_config_var('SOABI')
  48. if ABI and ABI.startswith('cpython-'):
  49. ABI = ABI.replace('cpython-', 'cp')
  50. else:
  51. def _derive_abi():
  52. parts = ['cp', VER_SUFFIX]
  53. if sysconfig.get_config_var('Py_DEBUG'):
  54. parts.append('d')
  55. if sysconfig.get_config_var('WITH_PYMALLOC'):
  56. parts.append('m')
  57. if sysconfig.get_config_var('Py_UNICODE_SIZE') == 4:
  58. parts.append('u')
  59. return ''.join(parts)
  60. ABI = _derive_abi()
  61. del _derive_abi
  62. FILENAME_RE = re.compile(r'''
  63. (?P<nm>[^-]+)
  64. -(?P<vn>\d+[^-]*)
  65. (-(?P<bn>\d+[^-]*))?
  66. -(?P<py>\w+\d+(\.\w+\d+)*)
  67. -(?P<bi>\w+)
  68. -(?P<ar>\w+(\.\w+)*)
  69. \.whl$
  70. ''', re.IGNORECASE | re.VERBOSE)
  71. NAME_VERSION_RE = re.compile(r'''
  72. (?P<nm>[^-]+)
  73. -(?P<vn>\d+[^-]*)
  74. (-(?P<bn>\d+[^-]*))?$
  75. ''', re.IGNORECASE | re.VERBOSE)
  76. SHEBANG_RE = re.compile(br'\s*#![^\r\n]*')
  77. SHEBANG_DETAIL_RE = re.compile(br'^(\s*#!("[^"]+"|\S+))\s+(.*)$')
  78. SHEBANG_PYTHON = b'#!python'
  79. SHEBANG_PYTHONW = b'#!pythonw'
  80. if os.sep == '/':
  81. to_posix = lambda o: o
  82. else:
  83. to_posix = lambda o: o.replace(os.sep, '/')
  84. class Mounter(object):
  85. def __init__(self):
  86. self.impure_wheels = {}
  87. self.libs = {}
  88. def add(self, pathname, extensions):
  89. self.impure_wheels[pathname] = extensions
  90. self.libs.update(extensions)
  91. def remove(self, pathname):
  92. extensions = self.impure_wheels.pop(pathname)
  93. for k, v in extensions:
  94. if k in self.libs:
  95. del self.libs[k]
  96. def find_module(self, fullname, path=None):
  97. if fullname in self.libs:
  98. result = self
  99. else:
  100. result = None
  101. return result
  102. def load_module(self, fullname):
  103. if fullname in sys.modules:
  104. result = sys.modules[fullname]
  105. else:
  106. if fullname not in self.libs:
  107. raise ImportError('unable to find extension for %s' % fullname)
  108. result = imp.load_dynamic(fullname, self.libs[fullname])
  109. result.__loader__ = self
  110. parts = fullname.rsplit('.', 1)
  111. if len(parts) > 1:
  112. result.__package__ = parts[0]
  113. return result
  114. _hook = Mounter()
  115. class Wheel(object):
  116. """
  117. Class to build and install from Wheel files (PEP 427).
  118. """
  119. wheel_version = (1, 1)
  120. hash_kind = 'sha256'
  121. def __init__(self, filename=None, sign=False, verify=False):
  122. """
  123. Initialise an instance using a (valid) filename.
  124. """
  125. self.sign = sign
  126. self.should_verify = verify
  127. self.buildver = ''
  128. self.pyver = [PYVER]
  129. self.abi = ['none']
  130. self.arch = ['any']
  131. self.dirname = os.getcwd()
  132. if filename is None:
  133. self.name = 'dummy'
  134. self.version = '0.1'
  135. self._filename = self.filename
  136. else:
  137. m = NAME_VERSION_RE.match(filename)
  138. if m:
  139. info = m.groupdict('')
  140. self.name = info['nm']
  141. # Reinstate the local version separator
  142. self.version = info['vn'].replace('_', '-')
  143. self.buildver = info['bn']
  144. self._filename = self.filename
  145. else:
  146. dirname, filename = os.path.split(filename)
  147. m = FILENAME_RE.match(filename)
  148. if not m:
  149. raise DistlibException('Invalid name or '
  150. 'filename: %r' % filename)
  151. if dirname:
  152. self.dirname = os.path.abspath(dirname)
  153. self._filename = filename
  154. info = m.groupdict('')
  155. self.name = info['nm']
  156. self.version = info['vn']
  157. self.buildver = info['bn']
  158. self.pyver = info['py'].split('.')
  159. self.abi = info['bi'].split('.')
  160. self.arch = info['ar'].split('.')
  161. @property
  162. def filename(self):
  163. """
  164. Build and return a filename from the various components.
  165. """
  166. if self.buildver:
  167. buildver = '-' + self.buildver
  168. else:
  169. buildver = ''
  170. pyver = '.'.join(self.pyver)
  171. abi = '.'.join(self.abi)
  172. arch = '.'.join(self.arch)
  173. # replace - with _ as a local version separator
  174. version = self.version.replace('-', '_')
  175. return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver,
  176. pyver, abi, arch)
  177. @property
  178. def exists(self):
  179. path = os.path.join(self.dirname, self.filename)
  180. return os.path.isfile(path)
  181. @property
  182. def tags(self):
  183. for pyver in self.pyver:
  184. for abi in self.abi:
  185. for arch in self.arch:
  186. yield pyver, abi, arch
  187. @cached_property
  188. def metadata(self):
  189. pathname = os.path.join(self.dirname, self.filename)
  190. name_ver = '%s-%s' % (self.name, self.version)
  191. info_dir = '%s.dist-info' % name_ver
  192. wrapper = codecs.getreader('utf-8')
  193. with ZipFile(pathname, 'r') as zf:
  194. wheel_metadata = self.get_wheel_metadata(zf)
  195. wv = wheel_metadata['Wheel-Version'].split('.', 1)
  196. file_version = tuple([int(i) for i in wv])
  197. if file_version < (1, 1):
  198. fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME, 'METADATA']
  199. else:
  200. fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME]
  201. result = None
  202. for fn in fns:
  203. try:
  204. metadata_filename = posixpath.join(info_dir, fn)
  205. with zf.open(metadata_filename) as bf:
  206. wf = wrapper(bf)
  207. result = Metadata(fileobj=wf)
  208. if result:
  209. break
  210. except KeyError:
  211. pass
  212. if not result:
  213. raise ValueError('Invalid wheel, because metadata is '
  214. 'missing: looked in %s' % ', '.join(fns))
  215. return result
  216. def get_wheel_metadata(self, zf):
  217. name_ver = '%s-%s' % (self.name, self.version)
  218. info_dir = '%s.dist-info' % name_ver
  219. metadata_filename = posixpath.join(info_dir, 'WHEEL')
  220. with zf.open(metadata_filename) as bf:
  221. wf = codecs.getreader('utf-8')(bf)
  222. message = message_from_file(wf)
  223. return dict(message)
  224. @cached_property
  225. def info(self):
  226. pathname = os.path.join(self.dirname, self.filename)
  227. with ZipFile(pathname, 'r') as zf:
  228. result = self.get_wheel_metadata(zf)
  229. return result
  230. def process_shebang(self, data):
  231. m = SHEBANG_RE.match(data)
  232. if m:
  233. end = m.end()
  234. shebang, data_after_shebang = data[:end], data[end:]
  235. # Preserve any arguments after the interpreter
  236. if b'pythonw' in shebang.lower():
  237. shebang_python = SHEBANG_PYTHONW
  238. else:
  239. shebang_python = SHEBANG_PYTHON
  240. m = SHEBANG_DETAIL_RE.match(shebang)
  241. if m:
  242. args = b' ' + m.groups()[-1]
  243. else:
  244. args = b''
  245. shebang = shebang_python + args
  246. data = shebang + data_after_shebang
  247. else:
  248. cr = data.find(b'\r')
  249. lf = data.find(b'\n')
  250. if cr < 0 or cr > lf:
  251. term = b'\n'
  252. else:
  253. if data[cr:cr + 2] == b'\r\n':
  254. term = b'\r\n'
  255. else:
  256. term = b'\r'
  257. data = SHEBANG_PYTHON + term + data
  258. return data
  259. def get_hash(self, data, hash_kind=None):
  260. if hash_kind is None:
  261. hash_kind = self.hash_kind
  262. try:
  263. hasher = getattr(hashlib, hash_kind)
  264. except AttributeError:
  265. raise DistlibException('Unsupported hash algorithm: %r' % hash_kind)
  266. result = hasher(data).digest()
  267. result = base64.urlsafe_b64encode(result).rstrip(b'=').decode('ascii')
  268. return hash_kind, result
  269. def write_record(self, records, record_path, base):
  270. records = list(records) # make a copy for sorting
  271. p = to_posix(os.path.relpath(record_path, base))
  272. records.append((p, '', ''))
  273. records.sort()
  274. with CSVWriter(record_path) as writer:
  275. for row in records:
  276. writer.writerow(row)
  277. def write_records(self, info, libdir, archive_paths):
  278. records = []
  279. distinfo, info_dir = info
  280. hasher = getattr(hashlib, self.hash_kind)
  281. for ap, p in archive_paths:
  282. with open(p, 'rb') as f:
  283. data = f.read()
  284. digest = '%s=%s' % self.get_hash(data)
  285. size = os.path.getsize(p)
  286. records.append((ap, digest, size))
  287. p = os.path.join(distinfo, 'RECORD')
  288. self.write_record(records, p, libdir)
  289. ap = to_posix(os.path.join(info_dir, 'RECORD'))
  290. archive_paths.append((ap, p))
  291. def build_zip(self, pathname, archive_paths):
  292. with ZipFile(pathname, 'w', zipfile.ZIP_DEFLATED) as zf:
  293. for ap, p in archive_paths:
  294. logger.debug('Wrote %s to %s in wheel', p, ap)
  295. zf.write(p, ap)
  296. def build(self, paths, tags=None, wheel_version=None):
  297. """
  298. Build a wheel from files in specified paths, and use any specified tags
  299. when determining the name of the wheel.
  300. """
  301. if tags is None:
  302. tags = {}
  303. libkey = list(filter(lambda o: o in paths, ('purelib', 'platlib')))[0]
  304. if libkey == 'platlib':
  305. is_pure = 'false'
  306. default_pyver = [IMPVER]
  307. default_abi = [ABI]
  308. default_arch = [ARCH]
  309. else:
  310. is_pure = 'true'
  311. default_pyver = [PYVER]
  312. default_abi = ['none']
  313. default_arch = ['any']
  314. self.pyver = tags.get('pyver', default_pyver)
  315. self.abi = tags.get('abi', default_abi)
  316. self.arch = tags.get('arch', default_arch)
  317. libdir = paths[libkey]
  318. name_ver = '%s-%s' % (self.name, self.version)
  319. data_dir = '%s.data' % name_ver
  320. info_dir = '%s.dist-info' % name_ver
  321. archive_paths = []
  322. # First, stuff which is not in site-packages
  323. for key in ('data', 'headers', 'scripts'):
  324. if key not in paths:
  325. continue
  326. path = paths[key]
  327. if os.path.isdir(path):
  328. for root, dirs, files in os.walk(path):
  329. for fn in files:
  330. p = fsdecode(os.path.join(root, fn))
  331. rp = os.path.relpath(p, path)
  332. ap = to_posix(os.path.join(data_dir, key, rp))
  333. archive_paths.append((ap, p))
  334. if key == 'scripts' and not p.endswith('.exe'):
  335. with open(p, 'rb') as f:
  336. data = f.read()
  337. data = self.process_shebang(data)
  338. with open(p, 'wb') as f:
  339. f.write(data)
  340. # Now, stuff which is in site-packages, other than the
  341. # distinfo stuff.
  342. path = libdir
  343. distinfo = None
  344. for root, dirs, files in os.walk(path):
  345. if root == path:
  346. # At the top level only, save distinfo for later
  347. # and skip it for now
  348. for i, dn in enumerate(dirs):
  349. dn = fsdecode(dn)
  350. if dn.endswith('.dist-info'):
  351. distinfo = os.path.join(root, dn)
  352. del dirs[i]
  353. break
  354. assert distinfo, '.dist-info directory expected, not found'
  355. for fn in files:
  356. # comment out next suite to leave .pyc files in
  357. if fsdecode(fn).endswith(('.pyc', '.pyo')):
  358. continue
  359. p = os.path.join(root, fn)
  360. rp = to_posix(os.path.relpath(p, path))
  361. archive_paths.append((rp, p))
  362. # Now distinfo. Assumed to be flat, i.e. os.listdir is enough.
  363. files = os.listdir(distinfo)
  364. for fn in files:
  365. if fn not in ('RECORD', 'INSTALLER', 'SHARED', 'WHEEL'):
  366. p = fsdecode(os.path.join(distinfo, fn))
  367. ap = to_posix(os.path.join(info_dir, fn))
  368. archive_paths.append((ap, p))
  369. wheel_metadata = [
  370. 'Wheel-Version: %d.%d' % (wheel_version or self.wheel_version),
  371. 'Generator: distlib %s' % __version__,
  372. 'Root-Is-Purelib: %s' % is_pure,
  373. ]
  374. for pyver, abi, arch in self.tags:
  375. wheel_metadata.append('Tag: %s-%s-%s' % (pyver, abi, arch))
  376. p = os.path.join(distinfo, 'WHEEL')
  377. with open(p, 'w') as f:
  378. f.write('\n'.join(wheel_metadata))
  379. ap = to_posix(os.path.join(info_dir, 'WHEEL'))
  380. archive_paths.append((ap, p))
  381. # Now, at last, RECORD.
  382. # Paths in here are archive paths - nothing else makes sense.
  383. self.write_records((distinfo, info_dir), libdir, archive_paths)
  384. # Now, ready to build the zip file
  385. pathname = os.path.join(self.dirname, self.filename)
  386. self.build_zip(pathname, archive_paths)
  387. return pathname
  388. def skip_entry(self, arcname):
  389. """
  390. Determine whether an archive entry should be skipped when verifying
  391. or installing.
  392. """
  393. # The signature file won't be in RECORD,
  394. # and we don't currently don't do anything with it
  395. # We also skip directories, as they won't be in RECORD
  396. # either. See:
  397. #
  398. # https://github.com/pypa/wheel/issues/294
  399. # https://github.com/pypa/wheel/issues/287
  400. # https://github.com/pypa/wheel/pull/289
  401. #
  402. return arcname.endswith(('/', '/RECORD.jws'))
  403. def install(self, paths, maker, **kwargs):
  404. """
  405. Install a wheel to the specified paths. If kwarg ``warner`` is
  406. specified, it should be a callable, which will be called with two
  407. tuples indicating the wheel version of this software and the wheel
  408. version in the file, if there is a discrepancy in the versions.
  409. This can be used to issue any warnings to raise any exceptions.
  410. If kwarg ``lib_only`` is True, only the purelib/platlib files are
  411. installed, and the headers, scripts, data and dist-info metadata are
  412. not written. If kwarg ``bytecode_hashed_invalidation`` is True, written
  413. bytecode will try to use file-hash based invalidation (PEP-552) on
  414. supported interpreter versions (CPython 2.7+).
  415. The return value is a :class:`InstalledDistribution` instance unless
  416. ``options.lib_only`` is True, in which case the return value is ``None``.
  417. """
  418. dry_run = maker.dry_run
  419. warner = kwargs.get('warner')
  420. lib_only = kwargs.get('lib_only', False)
  421. bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False)
  422. pathname = os.path.join(self.dirname, self.filename)
  423. name_ver = '%s-%s' % (self.name, self.version)
  424. data_dir = '%s.data' % name_ver
  425. info_dir = '%s.dist-info' % name_ver
  426. metadata_name = posixpath.join(info_dir, METADATA_FILENAME)
  427. wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')
  428. record_name = posixpath.join(info_dir, 'RECORD')
  429. wrapper = codecs.getreader('utf-8')
  430. with ZipFile(pathname, 'r') as zf:
  431. with zf.open(wheel_metadata_name) as bwf:
  432. wf = wrapper(bwf)
  433. message = message_from_file(wf)
  434. wv = message['Wheel-Version'].split('.', 1)
  435. file_version = tuple([int(i) for i in wv])
  436. if (file_version != self.wheel_version) and warner:
  437. warner(self.wheel_version, file_version)
  438. if message['Root-Is-Purelib'] == 'true':
  439. libdir = paths['purelib']
  440. else:
  441. libdir = paths['platlib']
  442. records = {}
  443. with zf.open(record_name) as bf:
  444. with CSVReader(stream=bf) as reader:
  445. for row in reader:
  446. p = row[0]
  447. records[p] = row
  448. data_pfx = posixpath.join(data_dir, '')
  449. info_pfx = posixpath.join(info_dir, '')
  450. script_pfx = posixpath.join(data_dir, 'scripts', '')
  451. # make a new instance rather than a copy of maker's,
  452. # as we mutate it
  453. fileop = FileOperator(dry_run=dry_run)
  454. fileop.record = True # so we can rollback if needed
  455. bc = not sys.dont_write_bytecode # Double negatives. Lovely!
  456. outfiles = [] # for RECORD writing
  457. # for script copying/shebang processing
  458. workdir = tempfile.mkdtemp()
  459. # set target dir later
  460. # we default add_launchers to False, as the
  461. # Python Launcher should be used instead
  462. maker.source_dir = workdir
  463. maker.target_dir = None
  464. try:
  465. for zinfo in zf.infolist():
  466. arcname = zinfo.filename
  467. if isinstance(arcname, text_type):
  468. u_arcname = arcname
  469. else:
  470. u_arcname = arcname.decode('utf-8')
  471. if self.skip_entry(u_arcname):
  472. continue
  473. row = records[u_arcname]
  474. if row[2] and str(zinfo.file_size) != row[2]:
  475. raise DistlibException('size mismatch for '
  476. '%s' % u_arcname)
  477. if row[1]:
  478. kind, value = row[1].split('=', 1)
  479. with zf.open(arcname) as bf:
  480. data = bf.read()
  481. _, digest = self.get_hash(data, kind)
  482. if digest != value:
  483. raise DistlibException('digest mismatch for '
  484. '%s' % arcname)
  485. if lib_only and u_arcname.startswith((info_pfx, data_pfx)):
  486. logger.debug('lib_only: skipping %s', u_arcname)
  487. continue
  488. is_script = (u_arcname.startswith(script_pfx)
  489. and not u_arcname.endswith('.exe'))
  490. if u_arcname.startswith(data_pfx):
  491. _, where, rp = u_arcname.split('/', 2)
  492. outfile = os.path.join(paths[where], convert_path(rp))
  493. else:
  494. # meant for site-packages.
  495. if u_arcname in (wheel_metadata_name, record_name):
  496. continue
  497. outfile = os.path.join(libdir, convert_path(u_arcname))
  498. if not is_script:
  499. with zf.open(arcname) as bf:
  500. fileop.copy_stream(bf, outfile)
  501. outfiles.append(outfile)
  502. # Double check the digest of the written file
  503. if not dry_run and row[1]:
  504. with open(outfile, 'rb') as bf:
  505. data = bf.read()
  506. _, newdigest = self.get_hash(data, kind)
  507. if newdigest != digest:
  508. raise DistlibException('digest mismatch '
  509. 'on write for '
  510. '%s' % outfile)
  511. if bc and outfile.endswith('.py'):
  512. try:
  513. pyc = fileop.byte_compile(outfile,
  514. hashed_invalidation=bc_hashed_invalidation)
  515. outfiles.append(pyc)
  516. except Exception:
  517. # Don't give up if byte-compilation fails,
  518. # but log it and perhaps warn the user
  519. logger.warning('Byte-compilation failed',
  520. exc_info=True)
  521. else:
  522. fn = os.path.basename(convert_path(arcname))
  523. workname = os.path.join(workdir, fn)
  524. with zf.open(arcname) as bf:
  525. fileop.copy_stream(bf, workname)
  526. dn, fn = os.path.split(outfile)
  527. maker.target_dir = dn
  528. filenames = maker.make(fn)
  529. fileop.set_executable_mode(filenames)
  530. outfiles.extend(filenames)
  531. if lib_only:
  532. logger.debug('lib_only: returning None')
  533. dist = None
  534. else:
  535. # Generate scripts
  536. # Try to get pydist.json so we can see if there are
  537. # any commands to generate. If this fails (e.g. because
  538. # of a legacy wheel), log a warning but don't give up.
  539. commands = None
  540. file_version = self.info['Wheel-Version']
  541. if file_version == '1.0':
  542. # Use legacy info
  543. ep = posixpath.join(info_dir, 'entry_points.txt')
  544. try:
  545. with zf.open(ep) as bwf:
  546. epdata = read_exports(bwf)
  547. commands = {}
  548. for key in ('console', 'gui'):
  549. k = '%s_scripts' % key
  550. if k in epdata:
  551. commands['wrap_%s' % key] = d = {}
  552. for v in epdata[k].values():
  553. s = '%s:%s' % (v.prefix, v.suffix)
  554. if v.flags:
  555. s += ' %s' % v.flags
  556. d[v.name] = s
  557. except Exception:
  558. logger.warning('Unable to read legacy script '
  559. 'metadata, so cannot generate '
  560. 'scripts')
  561. else:
  562. try:
  563. with zf.open(metadata_name) as bwf:
  564. wf = wrapper(bwf)
  565. commands = json.load(wf).get('extensions')
  566. if commands:
  567. commands = commands.get('python.commands')
  568. except Exception:
  569. logger.warning('Unable to read JSON metadata, so '
  570. 'cannot generate scripts')
  571. if commands:
  572. console_scripts = commands.get('wrap_console', {})
  573. gui_scripts = commands.get('wrap_gui', {})
  574. if console_scripts or gui_scripts:
  575. script_dir = paths.get('scripts', '')
  576. if not os.path.isdir(script_dir):
  577. raise ValueError('Valid script path not '
  578. 'specified')
  579. maker.target_dir = script_dir
  580. for k, v in console_scripts.items():
  581. script = '%s = %s' % (k, v)
  582. filenames = maker.make(script)
  583. fileop.set_executable_mode(filenames)
  584. if gui_scripts:
  585. options = {'gui': True }
  586. for k, v in gui_scripts.items():
  587. script = '%s = %s' % (k, v)
  588. filenames = maker.make(script, options)
  589. fileop.set_executable_mode(filenames)
  590. p = os.path.join(libdir, info_dir)
  591. dist = InstalledDistribution(p)
  592. # Write SHARED
  593. paths = dict(paths) # don't change passed in dict
  594. del paths['purelib']
  595. del paths['platlib']
  596. paths['lib'] = libdir
  597. p = dist.write_shared_locations(paths, dry_run)
  598. if p:
  599. outfiles.append(p)
  600. # Write RECORD
  601. dist.write_installed_files(outfiles, paths['prefix'],
  602. dry_run)
  603. return dist
  604. except Exception: # pragma: no cover
  605. logger.exception('installation failed.')
  606. fileop.rollback()
  607. raise
  608. finally:
  609. shutil.rmtree(workdir)
  610. def _get_dylib_cache(self):
  611. global cache
  612. if cache is None:
  613. # Use native string to avoid issues on 2.x: see Python #20140.
  614. base = os.path.join(get_cache_base(), str('dylib-cache'),
  615. '%s.%s' % sys.version_info[:2])
  616. cache = Cache(base)
  617. return cache
  618. def _get_extensions(self):
  619. pathname = os.path.join(self.dirname, self.filename)
  620. name_ver = '%s-%s' % (self.name, self.version)
  621. info_dir = '%s.dist-info' % name_ver
  622. arcname = posixpath.join(info_dir, 'EXTENSIONS')
  623. wrapper = codecs.getreader('utf-8')
  624. result = []
  625. with ZipFile(pathname, 'r') as zf:
  626. try:
  627. with zf.open(arcname) as bf:
  628. wf = wrapper(bf)
  629. extensions = json.load(wf)
  630. cache = self._get_dylib_cache()
  631. prefix = cache.prefix_to_dir(pathname)
  632. cache_base = os.path.join(cache.base, prefix)
  633. if not os.path.isdir(cache_base):
  634. os.makedirs(cache_base)
  635. for name, relpath in extensions.items():
  636. dest = os.path.join(cache_base, convert_path(relpath))
  637. if not os.path.exists(dest):
  638. extract = True
  639. else:
  640. file_time = os.stat(dest).st_mtime
  641. file_time = datetime.datetime.fromtimestamp(file_time)
  642. info = zf.getinfo(relpath)
  643. wheel_time = datetime.datetime(*info.date_time)
  644. extract = wheel_time > file_time
  645. if extract:
  646. zf.extract(relpath, cache_base)
  647. result.append((name, dest))
  648. except KeyError:
  649. pass
  650. return result
  651. def is_compatible(self):
  652. """
  653. Determine if a wheel is compatible with the running system.
  654. """
  655. return is_compatible(self)
  656. def is_mountable(self):
  657. """
  658. Determine if a wheel is asserted as mountable by its metadata.
  659. """
  660. return True # for now - metadata details TBD
  661. def mount(self, append=False):
  662. pathname = os.path.abspath(os.path.join(self.dirname, self.filename))
  663. if not self.is_compatible():
  664. msg = 'Wheel %s not compatible with this Python.' % pathname
  665. raise DistlibException(msg)
  666. if not self.is_mountable():
  667. msg = 'Wheel %s is marked as not mountable.' % pathname
  668. raise DistlibException(msg)
  669. if pathname in sys.path:
  670. logger.debug('%s already in path', pathname)
  671. else:
  672. if append:
  673. sys.path.append(pathname)
  674. else:
  675. sys.path.insert(0, pathname)
  676. extensions = self._get_extensions()
  677. if extensions:
  678. if _hook not in sys.meta_path:
  679. sys.meta_path.append(_hook)
  680. _hook.add(pathname, extensions)
  681. def unmount(self):
  682. pathname = os.path.abspath(os.path.join(self.dirname, self.filename))
  683. if pathname not in sys.path:
  684. logger.debug('%s not in path', pathname)
  685. else:
  686. sys.path.remove(pathname)
  687. if pathname in _hook.impure_wheels:
  688. _hook.remove(pathname)
  689. if not _hook.impure_wheels:
  690. if _hook in sys.meta_path:
  691. sys.meta_path.remove(_hook)
  692. def verify(self):
  693. pathname = os.path.join(self.dirname, self.filename)
  694. name_ver = '%s-%s' % (self.name, self.version)
  695. data_dir = '%s.data' % name_ver
  696. info_dir = '%s.dist-info' % name_ver
  697. metadata_name = posixpath.join(info_dir, METADATA_FILENAME)
  698. wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')
  699. record_name = posixpath.join(info_dir, 'RECORD')
  700. wrapper = codecs.getreader('utf-8')
  701. with ZipFile(pathname, 'r') as zf:
  702. with zf.open(wheel_metadata_name) as bwf:
  703. wf = wrapper(bwf)
  704. message = message_from_file(wf)
  705. wv = message['Wheel-Version'].split('.', 1)
  706. file_version = tuple([int(i) for i in wv])
  707. # TODO version verification
  708. records = {}
  709. with zf.open(record_name) as bf:
  710. with CSVReader(stream=bf) as reader:
  711. for row in reader:
  712. p = row[0]
  713. records[p] = row
  714. for zinfo in zf.infolist():
  715. arcname = zinfo.filename
  716. if isinstance(arcname, text_type):
  717. u_arcname = arcname
  718. else:
  719. u_arcname = arcname.decode('utf-8')
  720. # See issue #115: some wheels have .. in their entries, but
  721. # in the filename ... e.g. __main__..py ! So the check is
  722. # updated to look for .. in the directory portions
  723. p = u_arcname.split('/')
  724. if '..' in p:
  725. raise DistlibException('invalid entry in '
  726. 'wheel: %r' % u_arcname)
  727. if self.skip_entry(u_arcname):
  728. continue
  729. row = records[u_arcname]
  730. if row[2] and str(zinfo.file_size) != row[2]:
  731. raise DistlibException('size mismatch for '
  732. '%s' % u_arcname)
  733. if row[1]:
  734. kind, value = row[1].split('=', 1)
  735. with zf.open(arcname) as bf:
  736. data = bf.read()
  737. _, digest = self.get_hash(data, kind)
  738. if digest != value:
  739. raise DistlibException('digest mismatch for '
  740. '%s' % arcname)
  741. def update(self, modifier, dest_dir=None, **kwargs):
  742. """
  743. Update the contents of a wheel in a generic way. The modifier should
  744. be a callable which expects a dictionary argument: its keys are
  745. archive-entry paths, and its values are absolute filesystem paths
  746. where the contents the corresponding archive entries can be found. The
  747. modifier is free to change the contents of the files pointed to, add
  748. new entries and remove entries, before returning. This method will
  749. extract the entire contents of the wheel to a temporary location, call
  750. the modifier, and then use the passed (and possibly updated)
  751. dictionary to write a new wheel. If ``dest_dir`` is specified, the new
  752. wheel is written there -- otherwise, the original wheel is overwritten.
  753. The modifier should return True if it updated the wheel, else False.
  754. This method returns the same value the modifier returns.
  755. """
  756. def get_version(path_map, info_dir):
  757. version = path = None
  758. key = '%s/%s' % (info_dir, METADATA_FILENAME)
  759. if key not in path_map:
  760. key = '%s/PKG-INFO' % info_dir
  761. if key in path_map:
  762. path = path_map[key]
  763. version = Metadata(path=path).version
  764. return version, path
  765. def update_version(version, path):
  766. updated = None
  767. try:
  768. v = NormalizedVersion(version)
  769. i = version.find('-')
  770. if i < 0:
  771. updated = '%s+1' % version
  772. else:
  773. parts = [int(s) for s in version[i + 1:].split('.')]
  774. parts[-1] += 1
  775. updated = '%s+%s' % (version[:i],
  776. '.'.join(str(i) for i in parts))
  777. except UnsupportedVersionError:
  778. logger.debug('Cannot update non-compliant (PEP-440) '
  779. 'version %r', version)
  780. if updated:
  781. md = Metadata(path=path)
  782. md.version = updated
  783. legacy = not path.endswith(METADATA_FILENAME)
  784. md.write(path=path, legacy=legacy)
  785. logger.debug('Version updated from %r to %r', version,
  786. updated)
  787. pathname = os.path.join(self.dirname, self.filename)
  788. name_ver = '%s-%s' % (self.name, self.version)
  789. info_dir = '%s.dist-info' % name_ver
  790. record_name = posixpath.join(info_dir, 'RECORD')
  791. with tempdir() as workdir:
  792. with ZipFile(pathname, 'r') as zf:
  793. path_map = {}
  794. for zinfo in zf.infolist():
  795. arcname = zinfo.filename
  796. if isinstance(arcname, text_type):
  797. u_arcname = arcname
  798. else:
  799. u_arcname = arcname.decode('utf-8')
  800. if u_arcname == record_name:
  801. continue
  802. if '..' in u_arcname:
  803. raise DistlibException('invalid entry in '
  804. 'wheel: %r' % u_arcname)
  805. zf.extract(zinfo, workdir)
  806. path = os.path.join(workdir, convert_path(u_arcname))
  807. path_map[u_arcname] = path
  808. # Remember the version.
  809. original_version, _ = get_version(path_map, info_dir)
  810. # Files extracted. Call the modifier.
  811. modified = modifier(path_map, **kwargs)
  812. if modified:
  813. # Something changed - need to build a new wheel.
  814. current_version, path = get_version(path_map, info_dir)
  815. if current_version and (current_version == original_version):
  816. # Add or update local version to signify changes.
  817. update_version(current_version, path)
  818. # Decide where the new wheel goes.
  819. if dest_dir is None:
  820. fd, newpath = tempfile.mkstemp(suffix='.whl',
  821. prefix='wheel-update-',
  822. dir=workdir)
  823. os.close(fd)
  824. else:
  825. if not os.path.isdir(dest_dir):
  826. raise DistlibException('Not a directory: %r' % dest_dir)
  827. newpath = os.path.join(dest_dir, self.filename)
  828. archive_paths = list(path_map.items())
  829. distinfo = os.path.join(workdir, info_dir)
  830. info = distinfo, info_dir
  831. self.write_records(info, workdir, archive_paths)
  832. self.build_zip(newpath, archive_paths)
  833. if dest_dir is None:
  834. shutil.copyfile(newpath, pathname)
  835. return modified
  836. def compatible_tags():
  837. """
  838. Return (pyver, abi, arch) tuples compatible with this Python.
  839. """
  840. versions = [VER_SUFFIX]
  841. major = VER_SUFFIX[0]
  842. for minor in range(sys.version_info[1] - 1, - 1, -1):
  843. versions.append(''.join([major, str(minor)]))
  844. abis = []
  845. for suffix, _, _ in imp.get_suffixes():
  846. if suffix.startswith('.abi'):
  847. abis.append(suffix.split('.', 2)[1])
  848. abis.sort()
  849. if ABI != 'none':
  850. abis.insert(0, ABI)
  851. abis.append('none')
  852. result = []
  853. arches = [ARCH]
  854. if sys.platform == 'darwin':
  855. m = re.match(r'(\w+)_(\d+)_(\d+)_(\w+)$', ARCH)
  856. if m:
  857. name, major, minor, arch = m.groups()
  858. minor = int(minor)
  859. matches = [arch]
  860. if arch in ('i386', 'ppc'):
  861. matches.append('fat')
  862. if arch in ('i386', 'ppc', 'x86_64'):
  863. matches.append('fat3')
  864. if arch in ('ppc64', 'x86_64'):
  865. matches.append('fat64')
  866. if arch in ('i386', 'x86_64'):
  867. matches.append('intel')
  868. if arch in ('i386', 'x86_64', 'intel', 'ppc', 'ppc64'):
  869. matches.append('universal')
  870. while minor >= 0:
  871. for match in matches:
  872. s = '%s_%s_%s_%s' % (name, major, minor, match)
  873. if s != ARCH: # already there
  874. arches.append(s)
  875. minor -= 1
  876. # Most specific - our Python version, ABI and arch
  877. for abi in abis:
  878. for arch in arches:
  879. result.append((''.join((IMP_PREFIX, versions[0])), abi, arch))
  880. # where no ABI / arch dependency, but IMP_PREFIX dependency
  881. for i, version in enumerate(versions):
  882. result.append((''.join((IMP_PREFIX, version)), 'none', 'any'))
  883. if i == 0:
  884. result.append((''.join((IMP_PREFIX, version[0])), 'none', 'any'))
  885. # no IMP_PREFIX, ABI or arch dependency
  886. for i, version in enumerate(versions):
  887. result.append((''.join(('py', version)), 'none', 'any'))
  888. if i == 0:
  889. result.append((''.join(('py', version[0])), 'none', 'any'))
  890. return set(result)
  891. COMPATIBLE_TAGS = compatible_tags()
  892. del compatible_tags
  893. def is_compatible(wheel, tags=None):
  894. if not isinstance(wheel, Wheel):
  895. wheel = Wheel(wheel) # assume it's a filename
  896. result = False
  897. if tags is None:
  898. tags = COMPATIBLE_TAGS
  899. for ver, abi, arch in tags:
  900. if ver in wheel.pyver and abi in wheel.abi and arch in wheel.arch:
  901. result = True
  902. break
  903. return result