util.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561
  1. """distutils.util
  2. Miscellaneous utility functions -- anything that doesn't fit into
  3. one of the other *util.py modules.
  4. """
  5. import os
  6. import re
  7. import importlib.util
  8. import string
  9. import sys
  10. from distutils.errors import DistutilsPlatformError
  11. from distutils.dep_util import newer
  12. from distutils.spawn import spawn
  13. from distutils import log
  14. from distutils.errors import DistutilsByteCompileError
  15. from .py35compat import _optim_args_from_interpreter_flags
  16. def get_host_platform():
  17. """Return a string that identifies the current platform. This is used mainly to
  18. distinguish platform-specific build directories and platform-specific built
  19. distributions. Typically includes the OS name and version and the
  20. architecture (as supplied by 'os.uname()'), although the exact information
  21. included depends on the OS; eg. on Linux, the kernel version isn't
  22. particularly important.
  23. Examples of returned values:
  24. linux-i586
  25. linux-alpha (?)
  26. solaris-2.6-sun4u
  27. Windows will return one of:
  28. win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
  29. win32 (all others - specifically, sys.platform is returned)
  30. For other non-POSIX platforms, currently just returns 'sys.platform'.
  31. """
  32. if os.name == 'nt':
  33. if 'amd64' in sys.version.lower():
  34. return 'win-amd64'
  35. if '(arm)' in sys.version.lower():
  36. return 'win-arm32'
  37. if '(arm64)' in sys.version.lower():
  38. return 'win-arm64'
  39. return sys.platform
  40. # Set for cross builds explicitly
  41. if "_PYTHON_HOST_PLATFORM" in os.environ:
  42. return os.environ["_PYTHON_HOST_PLATFORM"]
  43. if os.name != "posix" or not hasattr(os, 'uname'):
  44. # XXX what about the architecture? NT is Intel or Alpha,
  45. # Mac OS is M68k or PPC, etc.
  46. return sys.platform
  47. # Try to distinguish various flavours of Unix
  48. (osname, host, release, version, machine) = os.uname()
  49. # Convert the OS name to lowercase, remove '/' characters, and translate
  50. # spaces (for "Power Macintosh")
  51. osname = osname.lower().replace('/', '')
  52. machine = machine.replace(' ', '_')
  53. machine = machine.replace('/', '-')
  54. if osname[:5] == "linux":
  55. # At least on Linux/Intel, 'machine' is the processor --
  56. # i386, etc.
  57. # XXX what about Alpha, SPARC, etc?
  58. return "%s-%s" % (osname, machine)
  59. elif osname[:5] == "sunos":
  60. if release[0] >= "5": # SunOS 5 == Solaris 2
  61. osname = "solaris"
  62. release = "%d.%s" % (int(release[0]) - 3, release[2:])
  63. # We can't use "platform.architecture()[0]" because a
  64. # bootstrap problem. We use a dict to get an error
  65. # if some suspicious happens.
  66. bitness = {2147483647:"32bit", 9223372036854775807:"64bit"}
  67. machine += ".%s" % bitness[sys.maxsize]
  68. # fall through to standard osname-release-machine representation
  69. elif osname[:3] == "aix":
  70. from .py38compat import aix_platform
  71. return aix_platform(osname, version, release)
  72. elif osname[:6] == "cygwin":
  73. osname = "cygwin"
  74. rel_re = re.compile (r'[\d.]+', re.ASCII)
  75. m = rel_re.match(release)
  76. if m:
  77. release = m.group()
  78. elif osname[:6] == "darwin":
  79. import _osx_support, distutils.sysconfig
  80. osname, release, machine = _osx_support.get_platform_osx(
  81. distutils.sysconfig.get_config_vars(),
  82. osname, release, machine)
  83. return "%s-%s-%s" % (osname, release, machine)
  84. def get_platform():
  85. if os.name == 'nt':
  86. TARGET_TO_PLAT = {
  87. 'x86' : 'win32',
  88. 'x64' : 'win-amd64',
  89. 'arm' : 'win-arm32',
  90. }
  91. return TARGET_TO_PLAT.get(os.environ.get('VSCMD_ARG_TGT_ARCH')) or get_host_platform()
  92. else:
  93. return get_host_platform()
  94. def convert_path (pathname):
  95. """Return 'pathname' as a name that will work on the native filesystem,
  96. i.e. split it on '/' and put it back together again using the current
  97. directory separator. Needed because filenames in the setup script are
  98. always supplied in Unix style, and have to be converted to the local
  99. convention before we can actually use them in the filesystem. Raises
  100. ValueError on non-Unix-ish systems if 'pathname' either starts or
  101. ends with a slash.
  102. """
  103. if os.sep == '/':
  104. return pathname
  105. if not pathname:
  106. return pathname
  107. if pathname[0] == '/':
  108. raise ValueError("path '%s' cannot be absolute" % pathname)
  109. if pathname[-1] == '/':
  110. raise ValueError("path '%s' cannot end with '/'" % pathname)
  111. paths = pathname.split('/')
  112. while '.' in paths:
  113. paths.remove('.')
  114. if not paths:
  115. return os.curdir
  116. return os.path.join(*paths)
  117. # convert_path ()
  118. def change_root (new_root, pathname):
  119. """Return 'pathname' with 'new_root' prepended. If 'pathname' is
  120. relative, this is equivalent to "os.path.join(new_root,pathname)".
  121. Otherwise, it requires making 'pathname' relative and then joining the
  122. two, which is tricky on DOS/Windows and Mac OS.
  123. """
  124. if os.name == 'posix':
  125. if not os.path.isabs(pathname):
  126. return os.path.join(new_root, pathname)
  127. else:
  128. return os.path.join(new_root, pathname[1:])
  129. elif os.name == 'nt':
  130. (drive, path) = os.path.splitdrive(pathname)
  131. if path[0] == '\\':
  132. path = path[1:]
  133. return os.path.join(new_root, path)
  134. else:
  135. raise DistutilsPlatformError("nothing known about platform '%s'" % os.name)
  136. _environ_checked = 0
  137. def check_environ ():
  138. """Ensure that 'os.environ' has all the environment variables we
  139. guarantee that users can use in config files, command-line options,
  140. etc. Currently this includes:
  141. HOME - user's home directory (Unix only)
  142. PLAT - description of the current platform, including hardware
  143. and OS (see 'get_platform()')
  144. """
  145. global _environ_checked
  146. if _environ_checked:
  147. return
  148. if os.name == 'posix' and 'HOME' not in os.environ:
  149. try:
  150. import pwd
  151. os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]
  152. except (ImportError, KeyError):
  153. # bpo-10496: if the current user identifier doesn't exist in the
  154. # password database, do nothing
  155. pass
  156. if 'PLAT' not in os.environ:
  157. os.environ['PLAT'] = get_platform()
  158. _environ_checked = 1
  159. def subst_vars (s, local_vars):
  160. """Perform shell/Perl-style variable substitution on 'string'. Every
  161. occurrence of '$' followed by a name is considered a variable, and
  162. variable is substituted by the value found in the 'local_vars'
  163. dictionary, or in 'os.environ' if it's not in 'local_vars'.
  164. 'os.environ' is first checked/augmented to guarantee that it contains
  165. certain values: see 'check_environ()'. Raise ValueError for any
  166. variables not found in either 'local_vars' or 'os.environ'.
  167. """
  168. check_environ()
  169. def _subst (match, local_vars=local_vars):
  170. var_name = match.group(1)
  171. if var_name in local_vars:
  172. return str(local_vars[var_name])
  173. else:
  174. return os.environ[var_name]
  175. try:
  176. return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)
  177. except KeyError as var:
  178. raise ValueError("invalid variable '$%s'" % var)
  179. # subst_vars ()
  180. def grok_environment_error (exc, prefix="error: "):
  181. # Function kept for backward compatibility.
  182. # Used to try clever things with EnvironmentErrors,
  183. # but nowadays str(exception) produces good messages.
  184. return prefix + str(exc)
  185. # Needed by 'split_quoted()'
  186. _wordchars_re = _squote_re = _dquote_re = None
  187. def _init_regex():
  188. global _wordchars_re, _squote_re, _dquote_re
  189. _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)
  190. _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")
  191. _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')
  192. def split_quoted (s):
  193. """Split a string up according to Unix shell-like rules for quotes and
  194. backslashes. In short: words are delimited by spaces, as long as those
  195. spaces are not escaped by a backslash, or inside a quoted string.
  196. Single and double quotes are equivalent, and the quote characters can
  197. be backslash-escaped. The backslash is stripped from any two-character
  198. escape sequence, leaving only the escaped character. The quote
  199. characters are stripped from any quoted string. Returns a list of
  200. words.
  201. """
  202. # This is a nice algorithm for splitting up a single string, since it
  203. # doesn't require character-by-character examination. It was a little
  204. # bit of a brain-bender to get it working right, though...
  205. if _wordchars_re is None: _init_regex()
  206. s = s.strip()
  207. words = []
  208. pos = 0
  209. while s:
  210. m = _wordchars_re.match(s, pos)
  211. end = m.end()
  212. if end == len(s):
  213. words.append(s[:end])
  214. break
  215. if s[end] in string.whitespace: # unescaped, unquoted whitespace: now
  216. words.append(s[:end]) # we definitely have a word delimiter
  217. s = s[end:].lstrip()
  218. pos = 0
  219. elif s[end] == '\\': # preserve whatever is being escaped;
  220. # will become part of the current word
  221. s = s[:end] + s[end+1:]
  222. pos = end+1
  223. else:
  224. if s[end] == "'": # slurp singly-quoted string
  225. m = _squote_re.match(s, end)
  226. elif s[end] == '"': # slurp doubly-quoted string
  227. m = _dquote_re.match(s, end)
  228. else:
  229. raise RuntimeError("this can't happen (bad char '%c')" % s[end])
  230. if m is None:
  231. raise ValueError("bad string (mismatched %s quotes?)" % s[end])
  232. (beg, end) = m.span()
  233. s = s[:beg] + s[beg+1:end-1] + s[end:]
  234. pos = m.end() - 2
  235. if pos >= len(s):
  236. words.append(s)
  237. break
  238. return words
  239. # split_quoted ()
  240. def execute (func, args, msg=None, verbose=0, dry_run=0):
  241. """Perform some action that affects the outside world (eg. by
  242. writing to the filesystem). Such actions are special because they
  243. are disabled by the 'dry_run' flag. This method takes care of all
  244. that bureaucracy for you; all you have to do is supply the
  245. function to call and an argument tuple for it (to embody the
  246. "external action" being performed), and an optional message to
  247. print.
  248. """
  249. if msg is None:
  250. msg = "%s%r" % (func.__name__, args)
  251. if msg[-2:] == ',)': # correct for singleton tuple
  252. msg = msg[0:-2] + ')'
  253. log.info(msg)
  254. if not dry_run:
  255. func(*args)
  256. def strtobool (val):
  257. """Convert a string representation of truth to true (1) or false (0).
  258. True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
  259. are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if
  260. 'val' is anything else.
  261. """
  262. val = val.lower()
  263. if val in ('y', 'yes', 't', 'true', 'on', '1'):
  264. return 1
  265. elif val in ('n', 'no', 'f', 'false', 'off', '0'):
  266. return 0
  267. else:
  268. raise ValueError("invalid truth value %r" % (val,))
  269. def byte_compile (py_files,
  270. optimize=0, force=0,
  271. prefix=None, base_dir=None,
  272. verbose=1, dry_run=0,
  273. direct=None):
  274. """Byte-compile a collection of Python source files to .pyc
  275. files in a __pycache__ subdirectory. 'py_files' is a list
  276. of files to compile; any files that don't end in ".py" are silently
  277. skipped. 'optimize' must be one of the following:
  278. 0 - don't optimize
  279. 1 - normal optimization (like "python -O")
  280. 2 - extra optimization (like "python -OO")
  281. If 'force' is true, all files are recompiled regardless of
  282. timestamps.
  283. The source filename encoded in each bytecode file defaults to the
  284. filenames listed in 'py_files'; you can modify these with 'prefix' and
  285. 'basedir'. 'prefix' is a string that will be stripped off of each
  286. source filename, and 'base_dir' is a directory name that will be
  287. prepended (after 'prefix' is stripped). You can supply either or both
  288. (or neither) of 'prefix' and 'base_dir', as you wish.
  289. If 'dry_run' is true, doesn't actually do anything that would
  290. affect the filesystem.
  291. Byte-compilation is either done directly in this interpreter process
  292. with the standard py_compile module, or indirectly by writing a
  293. temporary script and executing it. Normally, you should let
  294. 'byte_compile()' figure out to use direct compilation or not (see
  295. the source for details). The 'direct' flag is used by the script
  296. generated in indirect mode; unless you know what you're doing, leave
  297. it set to None.
  298. """
  299. # Late import to fix a bootstrap issue: _posixsubprocess is built by
  300. # setup.py, but setup.py uses distutils.
  301. import subprocess
  302. # nothing is done if sys.dont_write_bytecode is True
  303. if sys.dont_write_bytecode:
  304. raise DistutilsByteCompileError('byte-compiling is disabled.')
  305. # First, if the caller didn't force us into direct or indirect mode,
  306. # figure out which mode we should be in. We take a conservative
  307. # approach: choose direct mode *only* if the current interpreter is
  308. # in debug mode and optimize is 0. If we're not in debug mode (-O
  309. # or -OO), we don't know which level of optimization this
  310. # interpreter is running with, so we can't do direct
  311. # byte-compilation and be certain that it's the right thing. Thus,
  312. # always compile indirectly if the current interpreter is in either
  313. # optimize mode, or if either optimization level was requested by
  314. # the caller.
  315. if direct is None:
  316. direct = (__debug__ and optimize == 0)
  317. # "Indirect" byte-compilation: write a temporary script and then
  318. # run it with the appropriate flags.
  319. if not direct:
  320. try:
  321. from tempfile import mkstemp
  322. (script_fd, script_name) = mkstemp(".py")
  323. except ImportError:
  324. from tempfile import mktemp
  325. (script_fd, script_name) = None, mktemp(".py")
  326. log.info("writing byte-compilation script '%s'", script_name)
  327. if not dry_run:
  328. if script_fd is not None:
  329. script = os.fdopen(script_fd, "w")
  330. else:
  331. script = open(script_name, "w")
  332. with script:
  333. script.write("""\
  334. from distutils.util import byte_compile
  335. files = [
  336. """)
  337. # XXX would be nice to write absolute filenames, just for
  338. # safety's sake (script should be more robust in the face of
  339. # chdir'ing before running it). But this requires abspath'ing
  340. # 'prefix' as well, and that breaks the hack in build_lib's
  341. # 'byte_compile()' method that carefully tacks on a trailing
  342. # slash (os.sep really) to make sure the prefix here is "just
  343. # right". This whole prefix business is rather delicate -- the
  344. # problem is that it's really a directory, but I'm treating it
  345. # as a dumb string, so trailing slashes and so forth matter.
  346. #py_files = map(os.path.abspath, py_files)
  347. #if prefix:
  348. # prefix = os.path.abspath(prefix)
  349. script.write(",\n".join(map(repr, py_files)) + "]\n")
  350. script.write("""
  351. byte_compile(files, optimize=%r, force=%r,
  352. prefix=%r, base_dir=%r,
  353. verbose=%r, dry_run=0,
  354. direct=1)
  355. """ % (optimize, force, prefix, base_dir, verbose))
  356. cmd = [sys.executable]
  357. cmd.extend(_optim_args_from_interpreter_flags())
  358. cmd.append(script_name)
  359. spawn(cmd, dry_run=dry_run)
  360. execute(os.remove, (script_name,), "removing %s" % script_name,
  361. dry_run=dry_run)
  362. # "Direct" byte-compilation: use the py_compile module to compile
  363. # right here, right now. Note that the script generated in indirect
  364. # mode simply calls 'byte_compile()' in direct mode, a weird sort of
  365. # cross-process recursion. Hey, it works!
  366. else:
  367. from py_compile import compile
  368. for file in py_files:
  369. if file[-3:] != ".py":
  370. # This lets us be lazy and not filter filenames in
  371. # the "install_lib" command.
  372. continue
  373. # Terminology from the py_compile module:
  374. # cfile - byte-compiled file
  375. # dfile - purported source filename (same as 'file' by default)
  376. if optimize >= 0:
  377. opt = '' if optimize == 0 else optimize
  378. cfile = importlib.util.cache_from_source(
  379. file, optimization=opt)
  380. else:
  381. cfile = importlib.util.cache_from_source(file)
  382. dfile = file
  383. if prefix:
  384. if file[:len(prefix)] != prefix:
  385. raise ValueError("invalid prefix: filename %r doesn't start with %r"
  386. % (file, prefix))
  387. dfile = dfile[len(prefix):]
  388. if base_dir:
  389. dfile = os.path.join(base_dir, dfile)
  390. cfile_base = os.path.basename(cfile)
  391. if direct:
  392. if force or newer(file, cfile):
  393. log.info("byte-compiling %s to %s", file, cfile_base)
  394. if not dry_run:
  395. compile(file, cfile, dfile)
  396. else:
  397. log.debug("skipping byte-compilation of %s to %s",
  398. file, cfile_base)
  399. # byte_compile ()
  400. def rfc822_escape (header):
  401. """Return a version of the string escaped for inclusion in an
  402. RFC-822 header, by ensuring there are 8 spaces space after each newline.
  403. """
  404. lines = header.split('\n')
  405. sep = '\n' + 8 * ' '
  406. return sep.join(lines)
  407. # 2to3 support
  408. def run_2to3(files, fixer_names=None, options=None, explicit=None):
  409. """Invoke 2to3 on a list of Python files.
  410. The files should all come from the build area, as the
  411. modification is done in-place. To reduce the build time,
  412. only files modified since the last invocation of this
  413. function should be passed in the files argument."""
  414. if not files:
  415. return
  416. # Make this class local, to delay import of 2to3
  417. from lib2to3.refactor import RefactoringTool, get_fixers_from_package
  418. class DistutilsRefactoringTool(RefactoringTool):
  419. def log_error(self, msg, *args, **kw):
  420. log.error(msg, *args)
  421. def log_message(self, msg, *args):
  422. log.info(msg, *args)
  423. def log_debug(self, msg, *args):
  424. log.debug(msg, *args)
  425. if fixer_names is None:
  426. fixer_names = get_fixers_from_package('lib2to3.fixes')
  427. r = DistutilsRefactoringTool(fixer_names, options=options)
  428. r.refactor(files, write=True)
  429. def copydir_run_2to3(src, dest, template=None, fixer_names=None,
  430. options=None, explicit=None):
  431. """Recursively copy a directory, only copying new and changed files,
  432. running run_2to3 over all newly copied Python modules afterward.
  433. If you give a template string, it's parsed like a MANIFEST.in.
  434. """
  435. from distutils.dir_util import mkpath
  436. from distutils.file_util import copy_file
  437. from distutils.filelist import FileList
  438. filelist = FileList()
  439. curdir = os.getcwd()
  440. os.chdir(src)
  441. try:
  442. filelist.findall()
  443. finally:
  444. os.chdir(curdir)
  445. filelist.files[:] = filelist.allfiles
  446. if template:
  447. for line in template.splitlines():
  448. line = line.strip()
  449. if not line: continue
  450. filelist.process_template_line(line)
  451. copied = []
  452. for filename in filelist.files:
  453. outname = os.path.join(dest, filename)
  454. mkpath(os.path.dirname(outname))
  455. res = copy_file(os.path.join(src, filename), outname, update=1)
  456. if res[1]: copied.append(outname)
  457. run_2to3([fn for fn in copied if fn.lower().endswith('.py')],
  458. fixer_names=fixer_names, options=options, explicit=explicit)
  459. return copied
  460. class Mixin2to3:
  461. '''Mixin class for commands that run 2to3.
  462. To configure 2to3, setup scripts may either change
  463. the class variables, or inherit from individual commands
  464. to override how 2to3 is invoked.'''
  465. # provide list of fixers to run;
  466. # defaults to all from lib2to3.fixers
  467. fixer_names = None
  468. # options dictionary
  469. options = None
  470. # list of fixers to invoke even though they are marked as explicit
  471. explicit = None
  472. def run_2to3(self, files):
  473. return run_2to3(files, self.fixer_names, self.options, self.explicit)