sandbox.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492
  1. import os
  2. import sys
  3. import tempfile
  4. import operator
  5. import functools
  6. import itertools
  7. import re
  8. import contextlib
  9. import pickle
  10. import textwrap
  11. from setuptools.extern import six
  12. from setuptools.extern.six.moves import builtins, map
  13. import pkg_resources.py31compat
  14. from distutils.errors import DistutilsError
  15. from pkg_resources import working_set
  16. if sys.platform.startswith('java'):
  17. import org.python.modules.posix.PosixModule as _os
  18. else:
  19. _os = sys.modules[os.name]
  20. try:
  21. _file = file
  22. except NameError:
  23. _file = None
  24. _open = open
  25. __all__ = [
  26. "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
  27. ]
  28. def _execfile(filename, globals, locals=None):
  29. """
  30. Python 3 implementation of execfile.
  31. """
  32. mode = 'rb'
  33. with open(filename, mode) as stream:
  34. script = stream.read()
  35. if locals is None:
  36. locals = globals
  37. code = compile(script, filename, 'exec')
  38. exec(code, globals, locals)
  39. @contextlib.contextmanager
  40. def save_argv(repl=None):
  41. saved = sys.argv[:]
  42. if repl is not None:
  43. sys.argv[:] = repl
  44. try:
  45. yield saved
  46. finally:
  47. sys.argv[:] = saved
  48. @contextlib.contextmanager
  49. def save_path():
  50. saved = sys.path[:]
  51. try:
  52. yield saved
  53. finally:
  54. sys.path[:] = saved
  55. @contextlib.contextmanager
  56. def override_temp(replacement):
  57. """
  58. Monkey-patch tempfile.tempdir with replacement, ensuring it exists
  59. """
  60. pkg_resources.py31compat.makedirs(replacement, exist_ok=True)
  61. saved = tempfile.tempdir
  62. tempfile.tempdir = replacement
  63. try:
  64. yield
  65. finally:
  66. tempfile.tempdir = saved
  67. @contextlib.contextmanager
  68. def pushd(target):
  69. saved = os.getcwd()
  70. os.chdir(target)
  71. try:
  72. yield saved
  73. finally:
  74. os.chdir(saved)
  75. class UnpickleableException(Exception):
  76. """
  77. An exception representing another Exception that could not be pickled.
  78. """
  79. @staticmethod
  80. def dump(type, exc):
  81. """
  82. Always return a dumped (pickled) type and exc. If exc can't be pickled,
  83. wrap it in UnpickleableException first.
  84. """
  85. try:
  86. return pickle.dumps(type), pickle.dumps(exc)
  87. except Exception:
  88. # get UnpickleableException inside the sandbox
  89. from setuptools.sandbox import UnpickleableException as cls
  90. return cls.dump(cls, cls(repr(exc)))
  91. class ExceptionSaver:
  92. """
  93. A Context Manager that will save an exception, serialized, and restore it
  94. later.
  95. """
  96. def __enter__(self):
  97. return self
  98. def __exit__(self, type, exc, tb):
  99. if not exc:
  100. return
  101. # dump the exception
  102. self._saved = UnpickleableException.dump(type, exc)
  103. self._tb = tb
  104. # suppress the exception
  105. return True
  106. def resume(self):
  107. "restore and re-raise any exception"
  108. if '_saved' not in vars(self):
  109. return
  110. type, exc = map(pickle.loads, self._saved)
  111. six.reraise(type, exc, self._tb)
  112. @contextlib.contextmanager
  113. def save_modules():
  114. """
  115. Context in which imported modules are saved.
  116. Translates exceptions internal to the context into the equivalent exception
  117. outside the context.
  118. """
  119. saved = sys.modules.copy()
  120. with ExceptionSaver() as saved_exc:
  121. yield saved
  122. sys.modules.update(saved)
  123. # remove any modules imported since
  124. del_modules = (
  125. mod_name for mod_name in sys.modules
  126. if mod_name not in saved
  127. # exclude any encodings modules. See #285
  128. and not mod_name.startswith('encodings.')
  129. )
  130. _clear_modules(del_modules)
  131. saved_exc.resume()
  132. def _clear_modules(module_names):
  133. for mod_name in list(module_names):
  134. del sys.modules[mod_name]
  135. @contextlib.contextmanager
  136. def save_pkg_resources_state():
  137. saved = pkg_resources.__getstate__()
  138. try:
  139. yield saved
  140. finally:
  141. pkg_resources.__setstate__(saved)
  142. @contextlib.contextmanager
  143. def setup_context(setup_dir):
  144. temp_dir = os.path.join(setup_dir, 'temp')
  145. with save_pkg_resources_state():
  146. with save_modules():
  147. hide_setuptools()
  148. with save_path():
  149. with save_argv():
  150. with override_temp(temp_dir):
  151. with pushd(setup_dir):
  152. # ensure setuptools commands are available
  153. __import__('setuptools')
  154. yield
  155. def _needs_hiding(mod_name):
  156. """
  157. >>> _needs_hiding('setuptools')
  158. True
  159. >>> _needs_hiding('pkg_resources')
  160. True
  161. >>> _needs_hiding('setuptools_plugin')
  162. False
  163. >>> _needs_hiding('setuptools.__init__')
  164. True
  165. >>> _needs_hiding('distutils')
  166. True
  167. >>> _needs_hiding('os')
  168. False
  169. >>> _needs_hiding('Cython')
  170. True
  171. """
  172. pattern = re.compile(r'(setuptools|pkg_resources|distutils|Cython)(\.|$)')
  173. return bool(pattern.match(mod_name))
  174. def hide_setuptools():
  175. """
  176. Remove references to setuptools' modules from sys.modules to allow the
  177. invocation to import the most appropriate setuptools. This technique is
  178. necessary to avoid issues such as #315 where setuptools upgrading itself
  179. would fail to find a function declared in the metadata.
  180. """
  181. modules = filter(_needs_hiding, sys.modules)
  182. _clear_modules(modules)
  183. def run_setup(setup_script, args):
  184. """Run a distutils setup script, sandboxed in its directory"""
  185. setup_dir = os.path.abspath(os.path.dirname(setup_script))
  186. with setup_context(setup_dir):
  187. try:
  188. sys.argv[:] = [setup_script] + list(args)
  189. sys.path.insert(0, setup_dir)
  190. # reset to include setup dir, w/clean callback list
  191. working_set.__init__()
  192. working_set.callbacks.append(lambda dist: dist.activate())
  193. # __file__ should be a byte string on Python 2 (#712)
  194. dunder_file = (
  195. setup_script
  196. if isinstance(setup_script, str) else
  197. setup_script.encode(sys.getfilesystemencoding())
  198. )
  199. with DirectorySandbox(setup_dir):
  200. ns = dict(__file__=dunder_file, __name__='__main__')
  201. _execfile(setup_script, ns)
  202. except SystemExit as v:
  203. if v.args and v.args[0]:
  204. raise
  205. # Normal exit, just return
  206. class AbstractSandbox:
  207. """Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""
  208. _active = False
  209. def __init__(self):
  210. self._attrs = [
  211. name for name in dir(_os)
  212. if not name.startswith('_') and hasattr(self, name)
  213. ]
  214. def _copy(self, source):
  215. for name in self._attrs:
  216. setattr(os, name, getattr(source, name))
  217. def __enter__(self):
  218. self._copy(self)
  219. if _file:
  220. builtins.file = self._file
  221. builtins.open = self._open
  222. self._active = True
  223. def __exit__(self, exc_type, exc_value, traceback):
  224. self._active = False
  225. if _file:
  226. builtins.file = _file
  227. builtins.open = _open
  228. self._copy(_os)
  229. def run(self, func):
  230. """Run 'func' under os sandboxing"""
  231. with self:
  232. return func()
  233. def _mk_dual_path_wrapper(name):
  234. original = getattr(_os, name)
  235. def wrap(self, src, dst, *args, **kw):
  236. if self._active:
  237. src, dst = self._remap_pair(name, src, dst, *args, **kw)
  238. return original(src, dst, *args, **kw)
  239. return wrap
  240. for name in ["rename", "link", "symlink"]:
  241. if hasattr(_os, name):
  242. locals()[name] = _mk_dual_path_wrapper(name)
  243. def _mk_single_path_wrapper(name, original=None):
  244. original = original or getattr(_os, name)
  245. def wrap(self, path, *args, **kw):
  246. if self._active:
  247. path = self._remap_input(name, path, *args, **kw)
  248. return original(path, *args, **kw)
  249. return wrap
  250. if _file:
  251. _file = _mk_single_path_wrapper('file', _file)
  252. _open = _mk_single_path_wrapper('open', _open)
  253. for name in [
  254. "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
  255. "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
  256. "startfile", "mkfifo", "mknod", "pathconf", "access"
  257. ]:
  258. if hasattr(_os, name):
  259. locals()[name] = _mk_single_path_wrapper(name)
  260. def _mk_single_with_return(name):
  261. original = getattr(_os, name)
  262. def wrap(self, path, *args, **kw):
  263. if self._active:
  264. path = self._remap_input(name, path, *args, **kw)
  265. return self._remap_output(name, original(path, *args, **kw))
  266. return original(path, *args, **kw)
  267. return wrap
  268. for name in ['readlink', 'tempnam']:
  269. if hasattr(_os, name):
  270. locals()[name] = _mk_single_with_return(name)
  271. def _mk_query(name):
  272. original = getattr(_os, name)
  273. def wrap(self, *args, **kw):
  274. retval = original(*args, **kw)
  275. if self._active:
  276. return self._remap_output(name, retval)
  277. return retval
  278. return wrap
  279. for name in ['getcwd', 'tmpnam']:
  280. if hasattr(_os, name):
  281. locals()[name] = _mk_query(name)
  282. def _validate_path(self, path):
  283. """Called to remap or validate any path, whether input or output"""
  284. return path
  285. def _remap_input(self, operation, path, *args, **kw):
  286. """Called for path inputs"""
  287. return self._validate_path(path)
  288. def _remap_output(self, operation, path):
  289. """Called for path outputs"""
  290. return self._validate_path(path)
  291. def _remap_pair(self, operation, src, dst, *args, **kw):
  292. """Called for path pairs like rename, link, and symlink operations"""
  293. return (
  294. self._remap_input(operation + '-from', src, *args, **kw),
  295. self._remap_input(operation + '-to', dst, *args, **kw)
  296. )
  297. if hasattr(os, 'devnull'):
  298. _EXCEPTIONS = [os.devnull]
  299. else:
  300. _EXCEPTIONS = []
  301. class DirectorySandbox(AbstractSandbox):
  302. """Restrict operations to a single subdirectory - pseudo-chroot"""
  303. write_ops = dict.fromkeys([
  304. "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
  305. "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
  306. ])
  307. _exception_patterns = [
  308. # Allow lib2to3 to attempt to save a pickled grammar object (#121)
  309. r'.*lib2to3.*\.pickle$',
  310. ]
  311. "exempt writing to paths that match the pattern"
  312. def __init__(self, sandbox, exceptions=_EXCEPTIONS):
  313. self._sandbox = os.path.normcase(os.path.realpath(sandbox))
  314. self._prefix = os.path.join(self._sandbox, '')
  315. self._exceptions = [
  316. os.path.normcase(os.path.realpath(path))
  317. for path in exceptions
  318. ]
  319. AbstractSandbox.__init__(self)
  320. def _violation(self, operation, *args, **kw):
  321. from setuptools.sandbox import SandboxViolation
  322. raise SandboxViolation(operation, args, kw)
  323. if _file:
  324. def _file(self, path, mode='r', *args, **kw):
  325. if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
  326. self._violation("file", path, mode, *args, **kw)
  327. return _file(path, mode, *args, **kw)
  328. def _open(self, path, mode='r', *args, **kw):
  329. if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
  330. self._violation("open", path, mode, *args, **kw)
  331. return _open(path, mode, *args, **kw)
  332. def tmpnam(self):
  333. self._violation("tmpnam")
  334. def _ok(self, path):
  335. active = self._active
  336. try:
  337. self._active = False
  338. realpath = os.path.normcase(os.path.realpath(path))
  339. return (
  340. self._exempted(realpath)
  341. or realpath == self._sandbox
  342. or realpath.startswith(self._prefix)
  343. )
  344. finally:
  345. self._active = active
  346. def _exempted(self, filepath):
  347. start_matches = (
  348. filepath.startswith(exception)
  349. for exception in self._exceptions
  350. )
  351. pattern_matches = (
  352. re.match(pattern, filepath)
  353. for pattern in self._exception_patterns
  354. )
  355. candidates = itertools.chain(start_matches, pattern_matches)
  356. return any(candidates)
  357. def _remap_input(self, operation, path, *args, **kw):
  358. """Called for path inputs"""
  359. if operation in self.write_ops and not self._ok(path):
  360. self._violation(operation, os.path.realpath(path), *args, **kw)
  361. return path
  362. def _remap_pair(self, operation, src, dst, *args, **kw):
  363. """Called for path pairs like rename, link, and symlink operations"""
  364. if not self._ok(src) or not self._ok(dst):
  365. self._violation(operation, src, dst, *args, **kw)
  366. return (src, dst)
  367. def open(self, file, flags, mode=0o777, *args, **kw):
  368. """Called for low-level os.open()"""
  369. if flags & WRITE_FLAGS and not self._ok(file):
  370. self._violation("os.open", file, flags, mode, *args, **kw)
  371. return _os.open(file, flags, mode, *args, **kw)
  372. WRITE_FLAGS = functools.reduce(
  373. operator.or_, [
  374. getattr(_os, a, 0) for a in
  375. "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split()]
  376. )
  377. class SandboxViolation(DistutilsError):
  378. """A setup script attempted to modify the filesystem outside the sandbox"""
  379. tmpl = textwrap.dedent("""
  380. SandboxViolation: {cmd}{args!r} {kwargs}
  381. The package setup script has attempted to modify files on your system
  382. that are not within the EasyInstall build area, and has been aborted.
  383. This package cannot be safely installed by EasyInstall, and may not
  384. support alternate installation locations even if you run its setup
  385. script by hand. Please inform the package's author and the EasyInstall
  386. maintainers to find out if a fix or workaround is available.
  387. """).lstrip()
  388. def __str__(self):
  389. cmd, args, kwargs = self.args
  390. return self.tmpl.format(**locals())