123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559 |
- """distutils.util
- Miscellaneous utility functions -- anything that doesn't fit into
- one of the other *util.py modules.
- """
- import os
- import re
- import importlib.util
- import string
- import sys
- from distutils.errors import DistutilsPlatformError
- from distutils.dep_util import newer
- from distutils.spawn import spawn
- from distutils import log
- from distutils.errors import DistutilsByteCompileError
- def get_host_platform():
- """Return a string that identifies the current platform. This is used mainly to
- distinguish platform-specific build directories and platform-specific built
- distributions. Typically includes the OS name and version and the
- architecture (as supplied by 'os.uname()'), although the exact information
- included depends on the OS; eg. on Linux, the kernel version isn't
- particularly important.
- Examples of returned values:
- linux-i586
- linux-alpha (?)
- solaris-2.6-sun4u
- Windows will return one of:
- win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
- win32 (all others - specifically, sys.platform is returned)
- For other non-POSIX platforms, currently just returns 'sys.platform'.
- """
- if os.name == 'nt':
- if 'amd64' in sys.version.lower():
- return 'win-amd64'
- if '(arm)' in sys.version.lower():
- return 'win-arm32'
- if '(arm64)' in sys.version.lower():
- return 'win-arm64'
- return sys.platform
- # Set for cross builds explicitly
- if "_PYTHON_HOST_PLATFORM" in os.environ:
- return os.environ["_PYTHON_HOST_PLATFORM"]
- if os.name != "posix" or not hasattr(os, 'uname'):
- # XXX what about the architecture? NT is Intel or Alpha,
- # Mac OS is M68k or PPC, etc.
- return sys.platform
- # Try to distinguish various flavours of Unix
- (osname, host, release, version, machine) = os.uname()
- # Convert the OS name to lowercase, remove '/' characters, and translate
- # spaces (for "Power Macintosh")
- osname = osname.lower().replace('/', '')
- machine = machine.replace(' ', '_')
- machine = machine.replace('/', '-')
- if osname[:5] == "linux":
- # At least on Linux/Intel, 'machine' is the processor --
- # i386, etc.
- # XXX what about Alpha, SPARC, etc?
- return "%s-%s" % (osname, machine)
- elif osname[:5] == "sunos":
- if release[0] >= "5": # SunOS 5 == Solaris 2
- osname = "solaris"
- release = "%d.%s" % (int(release[0]) - 3, release[2:])
- # We can't use "platform.architecture()[0]" because a
- # bootstrap problem. We use a dict to get an error
- # if some suspicious happens.
- bitness = {2147483647:"32bit", 9223372036854775807:"64bit"}
- machine += ".%s" % bitness[sys.maxsize]
- # fall through to standard osname-release-machine representation
- elif osname[:3] == "aix":
- from _aix_support import aix_platform
- return aix_platform()
- elif osname[:6] == "cygwin":
- osname = "cygwin"
- rel_re = re.compile (r'[\d.]+', re.ASCII)
- m = rel_re.match(release)
- if m:
- release = m.group()
- elif osname[:6] == "darwin":
- import _osx_support, distutils.sysconfig
- osname, release, machine = _osx_support.get_platform_osx(
- distutils.sysconfig.get_config_vars(),
- osname, release, machine)
- return "%s-%s-%s" % (osname, release, machine)
- def get_platform():
- if os.name == 'nt':
- TARGET_TO_PLAT = {
- 'x86' : 'win32',
- 'x64' : 'win-amd64',
- 'arm' : 'win-arm32',
- }
- return TARGET_TO_PLAT.get(os.environ.get('VSCMD_ARG_TGT_ARCH')) or get_host_platform()
- else:
- return get_host_platform()
- def convert_path (pathname):
- """Return 'pathname' as a name that will work on the native filesystem,
- i.e. split it on '/' and put it back together again using the current
- directory separator. Needed because filenames in the setup script are
- always supplied in Unix style, and have to be converted to the local
- convention before we can actually use them in the filesystem. Raises
- ValueError on non-Unix-ish systems if 'pathname' either starts or
- ends with a slash.
- """
- if os.sep == '/':
- return pathname
- if not pathname:
- return pathname
- if pathname[0] == '/':
- raise ValueError("path '%s' cannot be absolute" % pathname)
- if pathname[-1] == '/':
- raise ValueError("path '%s' cannot end with '/'" % pathname)
- paths = pathname.split('/')
- while '.' in paths:
- paths.remove('.')
- if not paths:
- return os.curdir
- return os.path.join(*paths)
- # convert_path ()
- def change_root (new_root, pathname):
- """Return 'pathname' with 'new_root' prepended. If 'pathname' is
- relative, this is equivalent to "os.path.join(new_root,pathname)".
- Otherwise, it requires making 'pathname' relative and then joining the
- two, which is tricky on DOS/Windows and Mac OS.
- """
- if os.name == 'posix':
- if not os.path.isabs(pathname):
- return os.path.join(new_root, pathname)
- else:
- return os.path.join(new_root, pathname[1:])
- elif os.name == 'nt':
- (drive, path) = os.path.splitdrive(pathname)
- if path[0] == '\\':
- path = path[1:]
- return os.path.join(new_root, path)
- else:
- raise DistutilsPlatformError("nothing known about platform '%s'" % os.name)
- _environ_checked = 0
- def check_environ ():
- """Ensure that 'os.environ' has all the environment variables we
- guarantee that users can use in config files, command-line options,
- etc. Currently this includes:
- HOME - user's home directory (Unix only)
- PLAT - description of the current platform, including hardware
- and OS (see 'get_platform()')
- """
- global _environ_checked
- if _environ_checked:
- return
- if os.name == 'posix' and 'HOME' not in os.environ:
- try:
- import pwd
- os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]
- except (ImportError, KeyError):
- # bpo-10496: if the current user identifier doesn't exist in the
- # password database, do nothing
- pass
- if 'PLAT' not in os.environ:
- os.environ['PLAT'] = get_platform()
- _environ_checked = 1
- def subst_vars (s, local_vars):
- """Perform shell/Perl-style variable substitution on 'string'. Every
- occurrence of '$' followed by a name is considered a variable, and
- variable is substituted by the value found in the 'local_vars'
- dictionary, or in 'os.environ' if it's not in 'local_vars'.
- 'os.environ' is first checked/augmented to guarantee that it contains
- certain values: see 'check_environ()'. Raise ValueError for any
- variables not found in either 'local_vars' or 'os.environ'.
- """
- check_environ()
- def _subst (match, local_vars=local_vars):
- var_name = match.group(1)
- if var_name in local_vars:
- return str(local_vars[var_name])
- else:
- return os.environ[var_name]
- try:
- return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)
- except KeyError as var:
- raise ValueError("invalid variable '$%s'" % var)
- # subst_vars ()
- def grok_environment_error (exc, prefix="error: "):
- # Function kept for backward compatibility.
- # Used to try clever things with EnvironmentErrors,
- # but nowadays str(exception) produces good messages.
- return prefix + str(exc)
- # Needed by 'split_quoted()'
- _wordchars_re = _squote_re = _dquote_re = None
- def _init_regex():
- global _wordchars_re, _squote_re, _dquote_re
- _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)
- _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")
- _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')
- def split_quoted (s):
- """Split a string up according to Unix shell-like rules for quotes and
- backslashes. In short: words are delimited by spaces, as long as those
- spaces are not escaped by a backslash, or inside a quoted string.
- Single and double quotes are equivalent, and the quote characters can
- be backslash-escaped. The backslash is stripped from any two-character
- escape sequence, leaving only the escaped character. The quote
- characters are stripped from any quoted string. Returns a list of
- words.
- """
- # This is a nice algorithm for splitting up a single string, since it
- # doesn't require character-by-character examination. It was a little
- # bit of a brain-bender to get it working right, though...
- if _wordchars_re is None: _init_regex()
- s = s.strip()
- words = []
- pos = 0
- while s:
- m = _wordchars_re.match(s, pos)
- end = m.end()
- if end == len(s):
- words.append(s[:end])
- break
- if s[end] in string.whitespace: # unescaped, unquoted whitespace: now
- words.append(s[:end]) # we definitely have a word delimiter
- s = s[end:].lstrip()
- pos = 0
- elif s[end] == '\\': # preserve whatever is being escaped;
- # will become part of the current word
- s = s[:end] + s[end+1:]
- pos = end+1
- else:
- if s[end] == "'": # slurp singly-quoted string
- m = _squote_re.match(s, end)
- elif s[end] == '"': # slurp doubly-quoted string
- m = _dquote_re.match(s, end)
- else:
- raise RuntimeError("this can't happen (bad char '%c')" % s[end])
- if m is None:
- raise ValueError("bad string (mismatched %s quotes?)" % s[end])
- (beg, end) = m.span()
- s = s[:beg] + s[beg+1:end-1] + s[end:]
- pos = m.end() - 2
- if pos >= len(s):
- words.append(s)
- break
- return words
- # split_quoted ()
- def execute (func, args, msg=None, verbose=0, dry_run=0):
- """Perform some action that affects the outside world (eg. by
- writing to the filesystem). Such actions are special because they
- are disabled by the 'dry_run' flag. This method takes care of all
- that bureaucracy for you; all you have to do is supply the
- function to call and an argument tuple for it (to embody the
- "external action" being performed), and an optional message to
- print.
- """
- if msg is None:
- msg = "%s%r" % (func.__name__, args)
- if msg[-2:] == ',)': # correct for singleton tuple
- msg = msg[0:-2] + ')'
- log.info(msg)
- if not dry_run:
- func(*args)
- def strtobool (val):
- """Convert a string representation of truth to true (1) or false (0).
- True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
- are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if
- 'val' is anything else.
- """
- val = val.lower()
- if val in ('y', 'yes', 't', 'true', 'on', '1'):
- return 1
- elif val in ('n', 'no', 'f', 'false', 'off', '0'):
- return 0
- else:
- raise ValueError("invalid truth value %r" % (val,))
- def byte_compile (py_files,
- optimize=0, force=0,
- prefix=None, base_dir=None,
- verbose=1, dry_run=0,
- direct=None):
- """Byte-compile a collection of Python source files to .pyc
- files in a __pycache__ subdirectory. 'py_files' is a list
- of files to compile; any files that don't end in ".py" are silently
- skipped. 'optimize' must be one of the following:
- 0 - don't optimize
- 1 - normal optimization (like "python -O")
- 2 - extra optimization (like "python -OO")
- If 'force' is true, all files are recompiled regardless of
- timestamps.
- The source filename encoded in each bytecode file defaults to the
- filenames listed in 'py_files'; you can modify these with 'prefix' and
- 'basedir'. 'prefix' is a string that will be stripped off of each
- source filename, and 'base_dir' is a directory name that will be
- prepended (after 'prefix' is stripped). You can supply either or both
- (or neither) of 'prefix' and 'base_dir', as you wish.
- If 'dry_run' is true, doesn't actually do anything that would
- affect the filesystem.
- Byte-compilation is either done directly in this interpreter process
- with the standard py_compile module, or indirectly by writing a
- temporary script and executing it. Normally, you should let
- 'byte_compile()' figure out to use direct compilation or not (see
- the source for details). The 'direct' flag is used by the script
- generated in indirect mode; unless you know what you're doing, leave
- it set to None.
- """
- # Late import to fix a bootstrap issue: _posixsubprocess is built by
- # setup.py, but setup.py uses distutils.
- import subprocess
- # nothing is done if sys.dont_write_bytecode is True
- if sys.dont_write_bytecode:
- raise DistutilsByteCompileError('byte-compiling is disabled.')
- # First, if the caller didn't force us into direct or indirect mode,
- # figure out which mode we should be in. We take a conservative
- # approach: choose direct mode *only* if the current interpreter is
- # in debug mode and optimize is 0. If we're not in debug mode (-O
- # or -OO), we don't know which level of optimization this
- # interpreter is running with, so we can't do direct
- # byte-compilation and be certain that it's the right thing. Thus,
- # always compile indirectly if the current interpreter is in either
- # optimize mode, or if either optimization level was requested by
- # the caller.
- if direct is None:
- direct = (__debug__ and optimize == 0)
- # "Indirect" byte-compilation: write a temporary script and then
- # run it with the appropriate flags.
- if not direct:
- try:
- from tempfile import mkstemp
- (script_fd, script_name) = mkstemp(".py")
- except ImportError:
- from tempfile import mktemp
- (script_fd, script_name) = None, mktemp(".py")
- log.info("writing byte-compilation script '%s'", script_name)
- if not dry_run:
- if script_fd is not None:
- script = os.fdopen(script_fd, "w")
- else:
- script = open(script_name, "w")
- with script:
- script.write("""\
- from distutils.util import byte_compile
- files = [
- """)
- # XXX would be nice to write absolute filenames, just for
- # safety's sake (script should be more robust in the face of
- # chdir'ing before running it). But this requires abspath'ing
- # 'prefix' as well, and that breaks the hack in build_lib's
- # 'byte_compile()' method that carefully tacks on a trailing
- # slash (os.sep really) to make sure the prefix here is "just
- # right". This whole prefix business is rather delicate -- the
- # problem is that it's really a directory, but I'm treating it
- # as a dumb string, so trailing slashes and so forth matter.
- #py_files = map(os.path.abspath, py_files)
- #if prefix:
- # prefix = os.path.abspath(prefix)
- script.write(",\n".join(map(repr, py_files)) + "]\n")
- script.write("""
- byte_compile(files, optimize=%r, force=%r,
- prefix=%r, base_dir=%r,
- verbose=%r, dry_run=0,
- direct=1)
- """ % (optimize, force, prefix, base_dir, verbose))
- cmd = [sys.executable]
- cmd.extend(subprocess._optim_args_from_interpreter_flags())
- cmd.append(script_name)
- spawn(cmd, dry_run=dry_run)
- execute(os.remove, (script_name,), "removing %s" % script_name,
- dry_run=dry_run)
- # "Direct" byte-compilation: use the py_compile module to compile
- # right here, right now. Note that the script generated in indirect
- # mode simply calls 'byte_compile()' in direct mode, a weird sort of
- # cross-process recursion. Hey, it works!
- else:
- from py_compile import compile
- for file in py_files:
- if file[-3:] != ".py":
- # This lets us be lazy and not filter filenames in
- # the "install_lib" command.
- continue
- # Terminology from the py_compile module:
- # cfile - byte-compiled file
- # dfile - purported source filename (same as 'file' by default)
- if optimize >= 0:
- opt = '' if optimize == 0 else optimize
- cfile = importlib.util.cache_from_source(
- file, optimization=opt)
- else:
- cfile = importlib.util.cache_from_source(file)
- dfile = file
- if prefix:
- if file[:len(prefix)] != prefix:
- raise ValueError("invalid prefix: filename %r doesn't start with %r"
- % (file, prefix))
- dfile = dfile[len(prefix):]
- if base_dir:
- dfile = os.path.join(base_dir, dfile)
- cfile_base = os.path.basename(cfile)
- if direct:
- if force or newer(file, cfile):
- log.info("byte-compiling %s to %s", file, cfile_base)
- if not dry_run:
- compile(file, cfile, dfile)
- else:
- log.debug("skipping byte-compilation of %s to %s",
- file, cfile_base)
- # byte_compile ()
- def rfc822_escape (header):
- """Return a version of the string escaped for inclusion in an
- RFC-822 header, by ensuring there are 8 spaces space after each newline.
- """
- lines = header.split('\n')
- sep = '\n' + 8 * ' '
- return sep.join(lines)
- # 2to3 support
- def run_2to3(files, fixer_names=None, options=None, explicit=None):
- """Invoke 2to3 on a list of Python files.
- The files should all come from the build area, as the
- modification is done in-place. To reduce the build time,
- only files modified since the last invocation of this
- function should be passed in the files argument."""
- if not files:
- return
- # Make this class local, to delay import of 2to3
- from lib2to3.refactor import RefactoringTool, get_fixers_from_package
- class DistutilsRefactoringTool(RefactoringTool):
- def log_error(self, msg, *args, **kw):
- log.error(msg, *args)
- def log_message(self, msg, *args):
- log.info(msg, *args)
- def log_debug(self, msg, *args):
- log.debug(msg, *args)
- if fixer_names is None:
- fixer_names = get_fixers_from_package('lib2to3.fixes')
- r = DistutilsRefactoringTool(fixer_names, options=options)
- r.refactor(files, write=True)
- def copydir_run_2to3(src, dest, template=None, fixer_names=None,
- options=None, explicit=None):
- """Recursively copy a directory, only copying new and changed files,
- running run_2to3 over all newly copied Python modules afterward.
- If you give a template string, it's parsed like a MANIFEST.in.
- """
- from distutils.dir_util import mkpath
- from distutils.file_util import copy_file
- from distutils.filelist import FileList
- filelist = FileList()
- curdir = os.getcwd()
- os.chdir(src)
- try:
- filelist.findall()
- finally:
- os.chdir(curdir)
- filelist.files[:] = filelist.allfiles
- if template:
- for line in template.splitlines():
- line = line.strip()
- if not line: continue
- filelist.process_template_line(line)
- copied = []
- for filename in filelist.files:
- outname = os.path.join(dest, filename)
- mkpath(os.path.dirname(outname))
- res = copy_file(os.path.join(src, filename), outname, update=1)
- if res[1]: copied.append(outname)
- run_2to3([fn for fn in copied if fn.lower().endswith('.py')],
- fixer_names=fixer_names, options=options, explicit=explicit)
- return copied
- class Mixin2to3:
- '''Mixin class for commands that run 2to3.
- To configure 2to3, setup scripts may either change
- the class variables, or inherit from individual commands
- to override how 2to3 is invoked.'''
- # provide list of fixers to run;
- # defaults to all from lib2to3.fixers
- fixer_names = None
- # options dictionary
- options = None
- # list of fixers to invoke even though they are marked as explicit
- explicit = None
- def run_2to3(self, files):
- return run_2to3(files, self.fixer_names, self.options, self.explicit)
|