util.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559
  1. """distutils.util
  2. Miscellaneous utility functions -- anything that doesn't fit into
  3. one of the other *util.py modules.
  4. """
  5. import os
  6. import re
  7. import importlib.util
  8. import string
  9. import sys
  10. from distutils.errors import DistutilsPlatformError
  11. from distutils.dep_util import newer
  12. from distutils.spawn import spawn
  13. from distutils import log
  14. from distutils.errors import DistutilsByteCompileError
  15. def get_host_platform():
  16. """Return a string that identifies the current platform. This is used mainly to
  17. distinguish platform-specific build directories and platform-specific built
  18. distributions. Typically includes the OS name and version and the
  19. architecture (as supplied by 'os.uname()'), although the exact information
  20. included depends on the OS; eg. on Linux, the kernel version isn't
  21. particularly important.
  22. Examples of returned values:
  23. linux-i586
  24. linux-alpha (?)
  25. solaris-2.6-sun4u
  26. Windows will return one of:
  27. win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
  28. win32 (all others - specifically, sys.platform is returned)
  29. For other non-POSIX platforms, currently just returns 'sys.platform'.
  30. """
  31. if os.name == 'nt':
  32. if 'amd64' in sys.version.lower():
  33. return 'win-amd64'
  34. if '(arm)' in sys.version.lower():
  35. return 'win-arm32'
  36. if '(arm64)' in sys.version.lower():
  37. return 'win-arm64'
  38. return sys.platform
  39. # Set for cross builds explicitly
  40. if "_PYTHON_HOST_PLATFORM" in os.environ:
  41. return os.environ["_PYTHON_HOST_PLATFORM"]
  42. if os.name != "posix" or not hasattr(os, 'uname'):
  43. # XXX what about the architecture? NT is Intel or Alpha,
  44. # Mac OS is M68k or PPC, etc.
  45. return sys.platform
  46. # Try to distinguish various flavours of Unix
  47. (osname, host, release, version, machine) = os.uname()
  48. # Convert the OS name to lowercase, remove '/' characters, and translate
  49. # spaces (for "Power Macintosh")
  50. osname = osname.lower().replace('/', '')
  51. machine = machine.replace(' ', '_')
  52. machine = machine.replace('/', '-')
  53. if osname[:5] == "linux":
  54. # At least on Linux/Intel, 'machine' is the processor --
  55. # i386, etc.
  56. # XXX what about Alpha, SPARC, etc?
  57. return "%s-%s" % (osname, machine)
  58. elif osname[:5] == "sunos":
  59. if release[0] >= "5": # SunOS 5 == Solaris 2
  60. osname = "solaris"
  61. release = "%d.%s" % (int(release[0]) - 3, release[2:])
  62. # We can't use "platform.architecture()[0]" because a
  63. # bootstrap problem. We use a dict to get an error
  64. # if some suspicious happens.
  65. bitness = {2147483647:"32bit", 9223372036854775807:"64bit"}
  66. machine += ".%s" % bitness[sys.maxsize]
  67. # fall through to standard osname-release-machine representation
  68. elif osname[:3] == "aix":
  69. from _aix_support import aix_platform
  70. return aix_platform()
  71. elif osname[:6] == "cygwin":
  72. osname = "cygwin"
  73. rel_re = re.compile (r'[\d.]+', re.ASCII)
  74. m = rel_re.match(release)
  75. if m:
  76. release = m.group()
  77. elif osname[:6] == "darwin":
  78. import _osx_support, distutils.sysconfig
  79. osname, release, machine = _osx_support.get_platform_osx(
  80. distutils.sysconfig.get_config_vars(),
  81. osname, release, machine)
  82. return "%s-%s-%s" % (osname, release, machine)
  83. def get_platform():
  84. if os.name == 'nt':
  85. TARGET_TO_PLAT = {
  86. 'x86' : 'win32',
  87. 'x64' : 'win-amd64',
  88. 'arm' : 'win-arm32',
  89. }
  90. return TARGET_TO_PLAT.get(os.environ.get('VSCMD_ARG_TGT_ARCH')) or get_host_platform()
  91. else:
  92. return get_host_platform()
  93. def convert_path (pathname):
  94. """Return 'pathname' as a name that will work on the native filesystem,
  95. i.e. split it on '/' and put it back together again using the current
  96. directory separator. Needed because filenames in the setup script are
  97. always supplied in Unix style, and have to be converted to the local
  98. convention before we can actually use them in the filesystem. Raises
  99. ValueError on non-Unix-ish systems if 'pathname' either starts or
  100. ends with a slash.
  101. """
  102. if os.sep == '/':
  103. return pathname
  104. if not pathname:
  105. return pathname
  106. if pathname[0] == '/':
  107. raise ValueError("path '%s' cannot be absolute" % pathname)
  108. if pathname[-1] == '/':
  109. raise ValueError("path '%s' cannot end with '/'" % pathname)
  110. paths = pathname.split('/')
  111. while '.' in paths:
  112. paths.remove('.')
  113. if not paths:
  114. return os.curdir
  115. return os.path.join(*paths)
  116. # convert_path ()
  117. def change_root (new_root, pathname):
  118. """Return 'pathname' with 'new_root' prepended. If 'pathname' is
  119. relative, this is equivalent to "os.path.join(new_root,pathname)".
  120. Otherwise, it requires making 'pathname' relative and then joining the
  121. two, which is tricky on DOS/Windows and Mac OS.
  122. """
  123. if os.name == 'posix':
  124. if not os.path.isabs(pathname):
  125. return os.path.join(new_root, pathname)
  126. else:
  127. return os.path.join(new_root, pathname[1:])
  128. elif os.name == 'nt':
  129. (drive, path) = os.path.splitdrive(pathname)
  130. if path[0] == '\\':
  131. path = path[1:]
  132. return os.path.join(new_root, path)
  133. else:
  134. raise DistutilsPlatformError("nothing known about platform '%s'" % os.name)
  135. _environ_checked = 0
  136. def check_environ ():
  137. """Ensure that 'os.environ' has all the environment variables we
  138. guarantee that users can use in config files, command-line options,
  139. etc. Currently this includes:
  140. HOME - user's home directory (Unix only)
  141. PLAT - description of the current platform, including hardware
  142. and OS (see 'get_platform()')
  143. """
  144. global _environ_checked
  145. if _environ_checked:
  146. return
  147. if os.name == 'posix' and 'HOME' not in os.environ:
  148. try:
  149. import pwd
  150. os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]
  151. except (ImportError, KeyError):
  152. # bpo-10496: if the current user identifier doesn't exist in the
  153. # password database, do nothing
  154. pass
  155. if 'PLAT' not in os.environ:
  156. os.environ['PLAT'] = get_platform()
  157. _environ_checked = 1
  158. def subst_vars (s, local_vars):
  159. """Perform shell/Perl-style variable substitution on 'string'. Every
  160. occurrence of '$' followed by a name is considered a variable, and
  161. variable is substituted by the value found in the 'local_vars'
  162. dictionary, or in 'os.environ' if it's not in 'local_vars'.
  163. 'os.environ' is first checked/augmented to guarantee that it contains
  164. certain values: see 'check_environ()'. Raise ValueError for any
  165. variables not found in either 'local_vars' or 'os.environ'.
  166. """
  167. check_environ()
  168. def _subst (match, local_vars=local_vars):
  169. var_name = match.group(1)
  170. if var_name in local_vars:
  171. return str(local_vars[var_name])
  172. else:
  173. return os.environ[var_name]
  174. try:
  175. return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)
  176. except KeyError as var:
  177. raise ValueError("invalid variable '$%s'" % var)
  178. # subst_vars ()
  179. def grok_environment_error (exc, prefix="error: "):
  180. # Function kept for backward compatibility.
  181. # Used to try clever things with EnvironmentErrors,
  182. # but nowadays str(exception) produces good messages.
  183. return prefix + str(exc)
  184. # Needed by 'split_quoted()'
  185. _wordchars_re = _squote_re = _dquote_re = None
  186. def _init_regex():
  187. global _wordchars_re, _squote_re, _dquote_re
  188. _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)
  189. _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")
  190. _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')
  191. def split_quoted (s):
  192. """Split a string up according to Unix shell-like rules for quotes and
  193. backslashes. In short: words are delimited by spaces, as long as those
  194. spaces are not escaped by a backslash, or inside a quoted string.
  195. Single and double quotes are equivalent, and the quote characters can
  196. be backslash-escaped. The backslash is stripped from any two-character
  197. escape sequence, leaving only the escaped character. The quote
  198. characters are stripped from any quoted string. Returns a list of
  199. words.
  200. """
  201. # This is a nice algorithm for splitting up a single string, since it
  202. # doesn't require character-by-character examination. It was a little
  203. # bit of a brain-bender to get it working right, though...
  204. if _wordchars_re is None: _init_regex()
  205. s = s.strip()
  206. words = []
  207. pos = 0
  208. while s:
  209. m = _wordchars_re.match(s, pos)
  210. end = m.end()
  211. if end == len(s):
  212. words.append(s[:end])
  213. break
  214. if s[end] in string.whitespace: # unescaped, unquoted whitespace: now
  215. words.append(s[:end]) # we definitely have a word delimiter
  216. s = s[end:].lstrip()
  217. pos = 0
  218. elif s[end] == '\\': # preserve whatever is being escaped;
  219. # will become part of the current word
  220. s = s[:end] + s[end+1:]
  221. pos = end+1
  222. else:
  223. if s[end] == "'": # slurp singly-quoted string
  224. m = _squote_re.match(s, end)
  225. elif s[end] == '"': # slurp doubly-quoted string
  226. m = _dquote_re.match(s, end)
  227. else:
  228. raise RuntimeError("this can't happen (bad char '%c')" % s[end])
  229. if m is None:
  230. raise ValueError("bad string (mismatched %s quotes?)" % s[end])
  231. (beg, end) = m.span()
  232. s = s[:beg] + s[beg+1:end-1] + s[end:]
  233. pos = m.end() - 2
  234. if pos >= len(s):
  235. words.append(s)
  236. break
  237. return words
  238. # split_quoted ()
  239. def execute (func, args, msg=None, verbose=0, dry_run=0):
  240. """Perform some action that affects the outside world (eg. by
  241. writing to the filesystem). Such actions are special because they
  242. are disabled by the 'dry_run' flag. This method takes care of all
  243. that bureaucracy for you; all you have to do is supply the
  244. function to call and an argument tuple for it (to embody the
  245. "external action" being performed), and an optional message to
  246. print.
  247. """
  248. if msg is None:
  249. msg = "%s%r" % (func.__name__, args)
  250. if msg[-2:] == ',)': # correct for singleton tuple
  251. msg = msg[0:-2] + ')'
  252. log.info(msg)
  253. if not dry_run:
  254. func(*args)
  255. def strtobool (val):
  256. """Convert a string representation of truth to true (1) or false (0).
  257. True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
  258. are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if
  259. 'val' is anything else.
  260. """
  261. val = val.lower()
  262. if val in ('y', 'yes', 't', 'true', 'on', '1'):
  263. return 1
  264. elif val in ('n', 'no', 'f', 'false', 'off', '0'):
  265. return 0
  266. else:
  267. raise ValueError("invalid truth value %r" % (val,))
  268. def byte_compile (py_files,
  269. optimize=0, force=0,
  270. prefix=None, base_dir=None,
  271. verbose=1, dry_run=0,
  272. direct=None):
  273. """Byte-compile a collection of Python source files to .pyc
  274. files in a __pycache__ subdirectory. 'py_files' is a list
  275. of files to compile; any files that don't end in ".py" are silently
  276. skipped. 'optimize' must be one of the following:
  277. 0 - don't optimize
  278. 1 - normal optimization (like "python -O")
  279. 2 - extra optimization (like "python -OO")
  280. If 'force' is true, all files are recompiled regardless of
  281. timestamps.
  282. The source filename encoded in each bytecode file defaults to the
  283. filenames listed in 'py_files'; you can modify these with 'prefix' and
  284. 'basedir'. 'prefix' is a string that will be stripped off of each
  285. source filename, and 'base_dir' is a directory name that will be
  286. prepended (after 'prefix' is stripped). You can supply either or both
  287. (or neither) of 'prefix' and 'base_dir', as you wish.
  288. If 'dry_run' is true, doesn't actually do anything that would
  289. affect the filesystem.
  290. Byte-compilation is either done directly in this interpreter process
  291. with the standard py_compile module, or indirectly by writing a
  292. temporary script and executing it. Normally, you should let
  293. 'byte_compile()' figure out to use direct compilation or not (see
  294. the source for details). The 'direct' flag is used by the script
  295. generated in indirect mode; unless you know what you're doing, leave
  296. it set to None.
  297. """
  298. # Late import to fix a bootstrap issue: _posixsubprocess is built by
  299. # setup.py, but setup.py uses distutils.
  300. import subprocess
  301. # nothing is done if sys.dont_write_bytecode is True
  302. if sys.dont_write_bytecode:
  303. raise DistutilsByteCompileError('byte-compiling is disabled.')
  304. # First, if the caller didn't force us into direct or indirect mode,
  305. # figure out which mode we should be in. We take a conservative
  306. # approach: choose direct mode *only* if the current interpreter is
  307. # in debug mode and optimize is 0. If we're not in debug mode (-O
  308. # or -OO), we don't know which level of optimization this
  309. # interpreter is running with, so we can't do direct
  310. # byte-compilation and be certain that it's the right thing. Thus,
  311. # always compile indirectly if the current interpreter is in either
  312. # optimize mode, or if either optimization level was requested by
  313. # the caller.
  314. if direct is None:
  315. direct = (__debug__ and optimize == 0)
  316. # "Indirect" byte-compilation: write a temporary script and then
  317. # run it with the appropriate flags.
  318. if not direct:
  319. try:
  320. from tempfile import mkstemp
  321. (script_fd, script_name) = mkstemp(".py")
  322. except ImportError:
  323. from tempfile import mktemp
  324. (script_fd, script_name) = None, mktemp(".py")
  325. log.info("writing byte-compilation script '%s'", script_name)
  326. if not dry_run:
  327. if script_fd is not None:
  328. script = os.fdopen(script_fd, "w")
  329. else:
  330. script = open(script_name, "w")
  331. with script:
  332. script.write("""\
  333. from distutils.util import byte_compile
  334. files = [
  335. """)
  336. # XXX would be nice to write absolute filenames, just for
  337. # safety's sake (script should be more robust in the face of
  338. # chdir'ing before running it). But this requires abspath'ing
  339. # 'prefix' as well, and that breaks the hack in build_lib's
  340. # 'byte_compile()' method that carefully tacks on a trailing
  341. # slash (os.sep really) to make sure the prefix here is "just
  342. # right". This whole prefix business is rather delicate -- the
  343. # problem is that it's really a directory, but I'm treating it
  344. # as a dumb string, so trailing slashes and so forth matter.
  345. #py_files = map(os.path.abspath, py_files)
  346. #if prefix:
  347. # prefix = os.path.abspath(prefix)
  348. script.write(",\n".join(map(repr, py_files)) + "]\n")
  349. script.write("""
  350. byte_compile(files, optimize=%r, force=%r,
  351. prefix=%r, base_dir=%r,
  352. verbose=%r, dry_run=0,
  353. direct=1)
  354. """ % (optimize, force, prefix, base_dir, verbose))
  355. cmd = [sys.executable]
  356. cmd.extend(subprocess._optim_args_from_interpreter_flags())
  357. cmd.append(script_name)
  358. spawn(cmd, dry_run=dry_run)
  359. execute(os.remove, (script_name,), "removing %s" % script_name,
  360. dry_run=dry_run)
  361. # "Direct" byte-compilation: use the py_compile module to compile
  362. # right here, right now. Note that the script generated in indirect
  363. # mode simply calls 'byte_compile()' in direct mode, a weird sort of
  364. # cross-process recursion. Hey, it works!
  365. else:
  366. from py_compile import compile
  367. for file in py_files:
  368. if file[-3:] != ".py":
  369. # This lets us be lazy and not filter filenames in
  370. # the "install_lib" command.
  371. continue
  372. # Terminology from the py_compile module:
  373. # cfile - byte-compiled file
  374. # dfile - purported source filename (same as 'file' by default)
  375. if optimize >= 0:
  376. opt = '' if optimize == 0 else optimize
  377. cfile = importlib.util.cache_from_source(
  378. file, optimization=opt)
  379. else:
  380. cfile = importlib.util.cache_from_source(file)
  381. dfile = file
  382. if prefix:
  383. if file[:len(prefix)] != prefix:
  384. raise ValueError("invalid prefix: filename %r doesn't start with %r"
  385. % (file, prefix))
  386. dfile = dfile[len(prefix):]
  387. if base_dir:
  388. dfile = os.path.join(base_dir, dfile)
  389. cfile_base = os.path.basename(cfile)
  390. if direct:
  391. if force or newer(file, cfile):
  392. log.info("byte-compiling %s to %s", file, cfile_base)
  393. if not dry_run:
  394. compile(file, cfile, dfile)
  395. else:
  396. log.debug("skipping byte-compilation of %s to %s",
  397. file, cfile_base)
  398. # byte_compile ()
  399. def rfc822_escape (header):
  400. """Return a version of the string escaped for inclusion in an
  401. RFC-822 header, by ensuring there are 8 spaces space after each newline.
  402. """
  403. lines = header.split('\n')
  404. sep = '\n' + 8 * ' '
  405. return sep.join(lines)
  406. # 2to3 support
  407. def run_2to3(files, fixer_names=None, options=None, explicit=None):
  408. """Invoke 2to3 on a list of Python files.
  409. The files should all come from the build area, as the
  410. modification is done in-place. To reduce the build time,
  411. only files modified since the last invocation of this
  412. function should be passed in the files argument."""
  413. if not files:
  414. return
  415. # Make this class local, to delay import of 2to3
  416. from lib2to3.refactor import RefactoringTool, get_fixers_from_package
  417. class DistutilsRefactoringTool(RefactoringTool):
  418. def log_error(self, msg, *args, **kw):
  419. log.error(msg, *args)
  420. def log_message(self, msg, *args):
  421. log.info(msg, *args)
  422. def log_debug(self, msg, *args):
  423. log.debug(msg, *args)
  424. if fixer_names is None:
  425. fixer_names = get_fixers_from_package('lib2to3.fixes')
  426. r = DistutilsRefactoringTool(fixer_names, options=options)
  427. r.refactor(files, write=True)
  428. def copydir_run_2to3(src, dest, template=None, fixer_names=None,
  429. options=None, explicit=None):
  430. """Recursively copy a directory, only copying new and changed files,
  431. running run_2to3 over all newly copied Python modules afterward.
  432. If you give a template string, it's parsed like a MANIFEST.in.
  433. """
  434. from distutils.dir_util import mkpath
  435. from distutils.file_util import copy_file
  436. from distutils.filelist import FileList
  437. filelist = FileList()
  438. curdir = os.getcwd()
  439. os.chdir(src)
  440. try:
  441. filelist.findall()
  442. finally:
  443. os.chdir(curdir)
  444. filelist.files[:] = filelist.allfiles
  445. if template:
  446. for line in template.splitlines():
  447. line = line.strip()
  448. if not line: continue
  449. filelist.process_template_line(line)
  450. copied = []
  451. for filename in filelist.files:
  452. outname = os.path.join(dest, filename)
  453. mkpath(os.path.dirname(outname))
  454. res = copy_file(os.path.join(src, filename), outname, update=1)
  455. if res[1]: copied.append(outname)
  456. run_2to3([fn for fn in copied if fn.lower().endswith('.py')],
  457. fixer_names=fixer_names, options=options, explicit=explicit)
  458. return copied
  459. class Mixin2to3:
  460. '''Mixin class for commands that run 2to3.
  461. To configure 2to3, setup scripts may either change
  462. the class variables, or inherit from individual commands
  463. to override how 2to3 is invoked.'''
  464. # provide list of fixers to run;
  465. # defaults to all from lib2to3.fixers
  466. fixer_names = None
  467. # options dictionary
  468. options = None
  469. # list of fixers to invoke even though they are marked as explicit
  470. explicit = None
  471. def run_2to3(self, files):
  472. return run_2to3(files, self.fixer_names, self.options, self.explicit)