build_py.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. from glob import glob
  2. from distutils.util import convert_path
  3. import distutils.command.build_py as orig
  4. import os
  5. import fnmatch
  6. import textwrap
  7. import io
  8. import distutils.errors
  9. import itertools
  10. import stat
  11. try:
  12. from setuptools.lib2to3_ex import Mixin2to3
  13. except Exception:
  14. class Mixin2to3:
  15. def run_2to3(self, files, doctests=True):
  16. "do nothing"
  17. def make_writable(target):
  18. os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)
  19. class build_py(orig.build_py, Mixin2to3):
  20. """Enhanced 'build_py' command that includes data files with packages
  21. The data files are specified via a 'package_data' argument to 'setup()'.
  22. See 'setuptools.dist.Distribution' for more details.
  23. Also, this version of the 'build_py' command allows you to specify both
  24. 'py_modules' and 'packages' in the same setup operation.
  25. """
  26. def finalize_options(self):
  27. orig.build_py.finalize_options(self)
  28. self.package_data = self.distribution.package_data
  29. self.exclude_package_data = (self.distribution.exclude_package_data or
  30. {})
  31. if 'data_files' in self.__dict__:
  32. del self.__dict__['data_files']
  33. self.__updated_files = []
  34. self.__doctests_2to3 = []
  35. def run(self):
  36. """Build modules, packages, and copy data files to build directory"""
  37. if not self.py_modules and not self.packages:
  38. return
  39. if self.py_modules:
  40. self.build_modules()
  41. if self.packages:
  42. self.build_packages()
  43. self.build_package_data()
  44. self.run_2to3(self.__updated_files, False)
  45. self.run_2to3(self.__updated_files, True)
  46. self.run_2to3(self.__doctests_2to3, True)
  47. # Only compile actual .py files, using our base class' idea of what our
  48. # output files are.
  49. self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))
  50. def __getattr__(self, attr):
  51. "lazily compute data files"
  52. if attr == 'data_files':
  53. self.data_files = self._get_data_files()
  54. return self.data_files
  55. return orig.build_py.__getattr__(self, attr)
  56. def build_module(self, module, module_file, package):
  57. outfile, copied = orig.build_py.build_module(self, module, module_file,
  58. package)
  59. if copied:
  60. self.__updated_files.append(outfile)
  61. return outfile, copied
  62. def _get_data_files(self):
  63. """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
  64. self.analyze_manifest()
  65. return list(map(self._get_pkg_data_files, self.packages or ()))
  66. def _get_pkg_data_files(self, package):
  67. # Locate package source directory
  68. src_dir = self.get_package_dir(package)
  69. # Compute package build directory
  70. build_dir = os.path.join(*([self.build_lib] + package.split('.')))
  71. # Strip directory from globbed filenames
  72. filenames = [
  73. os.path.relpath(file, src_dir)
  74. for file in self.find_data_files(package, src_dir)
  75. ]
  76. return package, src_dir, build_dir, filenames
  77. def find_data_files(self, package, src_dir):
  78. """Return filenames for package's data files in 'src_dir'"""
  79. patterns = self._get_platform_patterns(
  80. self.package_data,
  81. package,
  82. src_dir,
  83. )
  84. globs_expanded = map(glob, patterns)
  85. # flatten the expanded globs into an iterable of matches
  86. globs_matches = itertools.chain.from_iterable(globs_expanded)
  87. glob_files = filter(os.path.isfile, globs_matches)
  88. files = itertools.chain(
  89. self.manifest_files.get(package, []),
  90. glob_files,
  91. )
  92. return self.exclude_data_files(package, src_dir, files)
  93. def build_package_data(self):
  94. """Copy data files into build directory"""
  95. for package, src_dir, build_dir, filenames in self.data_files:
  96. for filename in filenames:
  97. target = os.path.join(build_dir, filename)
  98. self.mkpath(os.path.dirname(target))
  99. srcfile = os.path.join(src_dir, filename)
  100. outf, copied = self.copy_file(srcfile, target)
  101. make_writable(target)
  102. srcfile = os.path.abspath(srcfile)
  103. if (copied and
  104. srcfile in self.distribution.convert_2to3_doctests):
  105. self.__doctests_2to3.append(outf)
  106. def analyze_manifest(self):
  107. self.manifest_files = mf = {}
  108. if not self.distribution.include_package_data:
  109. return
  110. src_dirs = {}
  111. for package in self.packages or ():
  112. # Locate package source directory
  113. src_dirs[assert_relative(self.get_package_dir(package))] = package
  114. self.run_command('egg_info')
  115. ei_cmd = self.get_finalized_command('egg_info')
  116. for path in ei_cmd.filelist.files:
  117. d, f = os.path.split(assert_relative(path))
  118. prev = None
  119. oldf = f
  120. while d and d != prev and d not in src_dirs:
  121. prev = d
  122. d, df = os.path.split(d)
  123. f = os.path.join(df, f)
  124. if d in src_dirs:
  125. if path.endswith('.py') and f == oldf:
  126. continue # it's a module, not data
  127. mf.setdefault(src_dirs[d], []).append(path)
  128. def get_data_files(self):
  129. pass # Lazily compute data files in _get_data_files() function.
  130. def check_package(self, package, package_dir):
  131. """Check namespace packages' __init__ for declare_namespace"""
  132. try:
  133. return self.packages_checked[package]
  134. except KeyError:
  135. pass
  136. init_py = orig.build_py.check_package(self, package, package_dir)
  137. self.packages_checked[package] = init_py
  138. if not init_py or not self.distribution.namespace_packages:
  139. return init_py
  140. for pkg in self.distribution.namespace_packages:
  141. if pkg == package or pkg.startswith(package + '.'):
  142. break
  143. else:
  144. return init_py
  145. with io.open(init_py, 'rb') as f:
  146. contents = f.read()
  147. if b'declare_namespace' not in contents:
  148. raise distutils.errors.DistutilsError(
  149. "Namespace package problem: %s is a namespace package, but "
  150. "its\n__init__.py does not call declare_namespace()! Please "
  151. 'fix it.\n(See the setuptools manual under '
  152. '"Namespace Packages" for details.)\n"' % (package,)
  153. )
  154. return init_py
  155. def initialize_options(self):
  156. self.packages_checked = {}
  157. orig.build_py.initialize_options(self)
  158. def get_package_dir(self, package):
  159. res = orig.build_py.get_package_dir(self, package)
  160. if self.distribution.src_root is not None:
  161. return os.path.join(self.distribution.src_root, res)
  162. return res
  163. def exclude_data_files(self, package, src_dir, files):
  164. """Filter filenames for package's data files in 'src_dir'"""
  165. files = list(files)
  166. patterns = self._get_platform_patterns(
  167. self.exclude_package_data,
  168. package,
  169. src_dir,
  170. )
  171. match_groups = (
  172. fnmatch.filter(files, pattern)
  173. for pattern in patterns
  174. )
  175. # flatten the groups of matches into an iterable of matches
  176. matches = itertools.chain.from_iterable(match_groups)
  177. bad = set(matches)
  178. keepers = (
  179. fn
  180. for fn in files
  181. if fn not in bad
  182. )
  183. # ditch dupes
  184. return list(_unique_everseen(keepers))
  185. @staticmethod
  186. def _get_platform_patterns(spec, package, src_dir):
  187. """
  188. yield platform-specific path patterns (suitable for glob
  189. or fn_match) from a glob-based spec (such as
  190. self.package_data or self.exclude_package_data)
  191. matching package in src_dir.
  192. """
  193. raw_patterns = itertools.chain(
  194. spec.get('', []),
  195. spec.get(package, []),
  196. )
  197. return (
  198. # Each pattern has to be converted to a platform-specific path
  199. os.path.join(src_dir, convert_path(pattern))
  200. for pattern in raw_patterns
  201. )
  202. # from Python docs
  203. def _unique_everseen(iterable, key=None):
  204. "List unique elements, preserving order. Remember all elements ever seen."
  205. # unique_everseen('AAAABBBCCDAABBB') --> A B C D
  206. # unique_everseen('ABBCcAD', str.lower) --> A B C D
  207. seen = set()
  208. seen_add = seen.add
  209. if key is None:
  210. for element in itertools.filterfalse(seen.__contains__, iterable):
  211. seen_add(element)
  212. yield element
  213. else:
  214. for element in iterable:
  215. k = key(element)
  216. if k not in seen:
  217. seen_add(k)
  218. yield element
  219. def assert_relative(path):
  220. if not os.path.isabs(path):
  221. return path
  222. from distutils.errors import DistutilsSetupError
  223. msg = textwrap.dedent("""
  224. Error: setup script specifies an absolute path:
  225. %s
  226. setup() arguments must *always* be /-separated paths relative to the
  227. setup.py directory, *never* absolute paths.
  228. """).lstrip() % path
  229. raise DistutilsSetupError(msg)