build_py.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276
  1. from glob import glob
  2. from distutils.util import convert_path
  3. import distutils.command.build_py as orig
  4. import os
  5. import fnmatch
  6. import textwrap
  7. import io
  8. import distutils.errors
  9. import itertools
  10. import stat
  11. from setuptools.extern import six
  12. from setuptools.extern.six.moves import map, filter, filterfalse
  13. try:
  14. from setuptools.lib2to3_ex import Mixin2to3
  15. except ImportError:
  16. class Mixin2to3:
  17. def run_2to3(self, files, doctests=True):
  18. "do nothing"
  19. def make_writable(target):
  20. os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)
  21. class build_py(orig.build_py, Mixin2to3):
  22. """Enhanced 'build_py' command that includes data files with packages
  23. The data files are specified via a 'package_data' argument to 'setup()'.
  24. See 'setuptools.dist.Distribution' for more details.
  25. Also, this version of the 'build_py' command allows you to specify both
  26. 'py_modules' and 'packages' in the same setup operation.
  27. """
  28. def finalize_options(self):
  29. orig.build_py.finalize_options(self)
  30. self.package_data = self.distribution.package_data
  31. self.exclude_package_data = (self.distribution.exclude_package_data or
  32. {})
  33. if 'data_files' in self.__dict__:
  34. del self.__dict__['data_files']
  35. self.__updated_files = []
  36. self.__doctests_2to3 = []
  37. def run(self):
  38. """Build modules, packages, and copy data files to build directory"""
  39. if not self.py_modules and not self.packages:
  40. return
  41. if self.py_modules:
  42. self.build_modules()
  43. if self.packages:
  44. self.build_packages()
  45. self.build_package_data()
  46. self.run_2to3(self.__updated_files, False)
  47. self.run_2to3(self.__updated_files, True)
  48. self.run_2to3(self.__doctests_2to3, True)
  49. # Only compile actual .py files, using our base class' idea of what our
  50. # output files are.
  51. self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))
  52. def __getattr__(self, attr):
  53. "lazily compute data files"
  54. if attr == 'data_files':
  55. self.data_files = self._get_data_files()
  56. return self.data_files
  57. return orig.build_py.__getattr__(self, attr)
  58. def build_module(self, module, module_file, package):
  59. if six.PY2 and isinstance(package, six.string_types):
  60. # avoid errors on Python 2 when unicode is passed (#190)
  61. package = package.split('.')
  62. outfile, copied = orig.build_py.build_module(self, module, module_file,
  63. package)
  64. if copied:
  65. self.__updated_files.append(outfile)
  66. return outfile, copied
  67. def _get_data_files(self):
  68. """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
  69. self.analyze_manifest()
  70. return list(map(self._get_pkg_data_files, self.packages or ()))
  71. def _get_pkg_data_files(self, package):
  72. # Locate package source directory
  73. src_dir = self.get_package_dir(package)
  74. # Compute package build directory
  75. build_dir = os.path.join(*([self.build_lib] + package.split('.')))
  76. # Strip directory from globbed filenames
  77. filenames = [
  78. os.path.relpath(file, src_dir)
  79. for file in self.find_data_files(package, src_dir)
  80. ]
  81. return package, src_dir, build_dir, filenames
  82. def find_data_files(self, package, src_dir):
  83. """Return filenames for package's data files in 'src_dir'"""
  84. patterns = self._get_platform_patterns(
  85. self.package_data,
  86. package,
  87. src_dir,
  88. )
  89. globs_expanded = map(glob, patterns)
  90. # flatten the expanded globs into an iterable of matches
  91. globs_matches = itertools.chain.from_iterable(globs_expanded)
  92. glob_files = filter(os.path.isfile, globs_matches)
  93. files = itertools.chain(
  94. self.manifest_files.get(package, []),
  95. glob_files,
  96. )
  97. return self.exclude_data_files(package, src_dir, files)
  98. def build_package_data(self):
  99. """Copy data files into build directory"""
  100. for package, src_dir, build_dir, filenames in self.data_files:
  101. for filename in filenames:
  102. target = os.path.join(build_dir, filename)
  103. self.mkpath(os.path.dirname(target))
  104. srcfile = os.path.join(src_dir, filename)
  105. outf, copied = self.copy_file(srcfile, target)
  106. make_writable(target)
  107. srcfile = os.path.abspath(srcfile)
  108. if (copied and
  109. srcfile in self.distribution.convert_2to3_doctests):
  110. self.__doctests_2to3.append(outf)
  111. def analyze_manifest(self):
  112. self.manifest_files = mf = {}
  113. if not self.distribution.include_package_data:
  114. return
  115. src_dirs = {}
  116. for package in self.packages or ():
  117. # Locate package source directory
  118. src_dirs[assert_relative(self.get_package_dir(package))] = package
  119. self.run_command('egg_info')
  120. ei_cmd = self.get_finalized_command('egg_info')
  121. for path in ei_cmd.filelist.files:
  122. d, f = os.path.split(assert_relative(path))
  123. prev = None
  124. oldf = f
  125. while d and d != prev and d not in src_dirs:
  126. prev = d
  127. d, df = os.path.split(d)
  128. f = os.path.join(df, f)
  129. if d in src_dirs:
  130. if path.endswith('.py') and f == oldf:
  131. continue # it's a module, not data
  132. mf.setdefault(src_dirs[d], []).append(path)
  133. def get_data_files(self):
  134. pass # Lazily compute data files in _get_data_files() function.
  135. def check_package(self, package, package_dir):
  136. """Check namespace packages' __init__ for declare_namespace"""
  137. try:
  138. return self.packages_checked[package]
  139. except KeyError:
  140. pass
  141. init_py = orig.build_py.check_package(self, package, package_dir)
  142. self.packages_checked[package] = init_py
  143. if not init_py or not self.distribution.namespace_packages:
  144. return init_py
  145. for pkg in self.distribution.namespace_packages:
  146. if pkg == package or pkg.startswith(package + '.'):
  147. break
  148. else:
  149. return init_py
  150. with io.open(init_py, 'rb') as f:
  151. contents = f.read()
  152. if b'declare_namespace' not in contents:
  153. raise distutils.errors.DistutilsError(
  154. "Namespace package problem: %s is a namespace package, but "
  155. "its\n__init__.py does not call declare_namespace()! Please "
  156. 'fix it.\n(See the setuptools manual under '
  157. '"Namespace Packages" for details.)\n"' % (package,)
  158. )
  159. return init_py
  160. def initialize_options(self):
  161. self.packages_checked = {}
  162. orig.build_py.initialize_options(self)
  163. def get_package_dir(self, package):
  164. res = orig.build_py.get_package_dir(self, package)
  165. if self.distribution.src_root is not None:
  166. return os.path.join(self.distribution.src_root, res)
  167. return res
  168. def exclude_data_files(self, package, src_dir, files):
  169. """Filter filenames for package's data files in 'src_dir'"""
  170. files = list(files)
  171. patterns = self._get_platform_patterns(
  172. self.exclude_package_data,
  173. package,
  174. src_dir,
  175. )
  176. match_groups = (
  177. fnmatch.filter(files, pattern)
  178. for pattern in patterns
  179. )
  180. # flatten the groups of matches into an iterable of matches
  181. matches = itertools.chain.from_iterable(match_groups)
  182. bad = set(matches)
  183. keepers = (
  184. fn
  185. for fn in files
  186. if fn not in bad
  187. )
  188. # ditch dupes
  189. return list(_unique_everseen(keepers))
  190. @staticmethod
  191. def _get_platform_patterns(spec, package, src_dir):
  192. """
  193. yield platform-specific path patterns (suitable for glob
  194. or fn_match) from a glob-based spec (such as
  195. self.package_data or self.exclude_package_data)
  196. matching package in src_dir.
  197. """
  198. raw_patterns = itertools.chain(
  199. spec.get('', []),
  200. spec.get(package, []),
  201. )
  202. return (
  203. # Each pattern has to be converted to a platform-specific path
  204. os.path.join(src_dir, convert_path(pattern))
  205. for pattern in raw_patterns
  206. )
  207. # from Python docs
  208. def _unique_everseen(iterable, key=None):
  209. "List unique elements, preserving order. Remember all elements ever seen."
  210. # unique_everseen('AAAABBBCCDAABBB') --> A B C D
  211. # unique_everseen('ABBCcAD', str.lower) --> A B C D
  212. seen = set()
  213. seen_add = seen.add
  214. if key is None:
  215. for element in filterfalse(seen.__contains__, iterable):
  216. seen_add(element)
  217. yield element
  218. else:
  219. for element in iterable:
  220. k = key(element)
  221. if k not in seen:
  222. seen_add(k)
  223. yield element
  224. def assert_relative(path):
  225. if not os.path.isabs(path):
  226. return path
  227. from distutils.errors import DistutilsSetupError
  228. msg = textwrap.dedent("""
  229. Error: setup script specifies an absolute path:
  230. %s
  231. setup() arguments must *always* be /-separated paths relative to the
  232. setup.py directory, *never* absolute paths.
  233. """).lstrip() % path
  234. raise DistutilsSetupError(msg)