build_ext.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332
  1. import os
  2. import sys
  3. import itertools
  4. from distutils.command.build_ext import build_ext as _du_build_ext
  5. from distutils.file_util import copy_file
  6. from distutils.ccompiler import new_compiler
  7. from distutils.sysconfig import customize_compiler, get_config_var
  8. from distutils.errors import DistutilsError
  9. from distutils import log
  10. from setuptools.extension import Library
  11. from setuptools.extern import six
  12. if six.PY2:
  13. import imp
  14. EXTENSION_SUFFIXES = [
  15. s for s, _, tp in imp.get_suffixes() if tp == imp.C_EXTENSION]
  16. else:
  17. from importlib.machinery import EXTENSION_SUFFIXES
  18. try:
  19. # Attempt to use Cython for building extensions, if available
  20. from Cython.Distutils.build_ext import build_ext as _build_ext
  21. # Additionally, assert that the compiler module will load
  22. # also. Ref #1229.
  23. __import__('Cython.Compiler.Main')
  24. except ImportError:
  25. _build_ext = _du_build_ext
  26. # make sure _config_vars is initialized
  27. get_config_var("LDSHARED")
  28. from distutils.sysconfig import _config_vars as _CONFIG_VARS # noqa
  29. def _customize_compiler_for_shlib(compiler):
  30. if sys.platform == "darwin":
  31. # building .dylib requires additional compiler flags on OSX; here we
  32. # temporarily substitute the pyconfig.h variables so that distutils'
  33. # 'customize_compiler' uses them before we build the shared libraries.
  34. tmp = _CONFIG_VARS.copy()
  35. try:
  36. # XXX Help! I don't have any idea whether these are right...
  37. _CONFIG_VARS['LDSHARED'] = (
  38. "gcc -Wl,-x -dynamiclib -undefined dynamic_lookup")
  39. _CONFIG_VARS['CCSHARED'] = " -dynamiclib"
  40. _CONFIG_VARS['SO'] = ".dylib"
  41. customize_compiler(compiler)
  42. finally:
  43. _CONFIG_VARS.clear()
  44. _CONFIG_VARS.update(tmp)
  45. else:
  46. customize_compiler(compiler)
  47. have_rtld = False
  48. use_stubs = False
  49. libtype = 'shared'
  50. if sys.platform == "darwin":
  51. use_stubs = True
  52. elif os.name != 'nt':
  53. try:
  54. import dl
  55. use_stubs = have_rtld = hasattr(dl, 'RTLD_NOW')
  56. except ImportError:
  57. pass
  58. def if_dl(s):
  59. return s if have_rtld else ''
  60. def get_abi3_suffix():
  61. """Return the file extension for an abi3-compliant Extension()"""
  62. for suffix in EXTENSION_SUFFIXES:
  63. if '.abi3' in suffix: # Unix
  64. return suffix
  65. elif suffix == '.pyd': # Windows
  66. return suffix
  67. class build_ext(_build_ext):
  68. def run(self):
  69. """Build extensions in build directory, then copy if --inplace"""
  70. old_inplace, self.inplace = self.inplace, 0
  71. _build_ext.run(self)
  72. self.inplace = old_inplace
  73. if old_inplace:
  74. self.copy_extensions_to_source()
  75. def copy_extensions_to_source(self):
  76. build_py = self.get_finalized_command('build_py')
  77. for ext in self.extensions:
  78. fullname = self.get_ext_fullname(ext.name)
  79. filename = self.get_ext_filename(fullname)
  80. modpath = fullname.split('.')
  81. package = '.'.join(modpath[:-1])
  82. package_dir = build_py.get_package_dir(package)
  83. dest_filename = os.path.join(package_dir,
  84. os.path.basename(filename))
  85. src_filename = os.path.join(self.build_lib, filename)
  86. # Always copy, even if source is older than destination, to ensure
  87. # that the right extensions for the current Python/platform are
  88. # used.
  89. copy_file(
  90. src_filename, dest_filename, verbose=self.verbose,
  91. dry_run=self.dry_run
  92. )
  93. if ext._needs_stub:
  94. self.write_stub(package_dir or os.curdir, ext, True)
  95. def get_ext_filename(self, fullname):
  96. filename = _build_ext.get_ext_filename(self, fullname)
  97. if fullname in self.ext_map:
  98. ext = self.ext_map[fullname]
  99. use_abi3 = (
  100. not six.PY2
  101. and getattr(ext, 'py_limited_api')
  102. and get_abi3_suffix()
  103. )
  104. if use_abi3:
  105. so_ext = get_config_var('EXT_SUFFIX')
  106. filename = filename[:-len(so_ext)]
  107. filename = filename + get_abi3_suffix()
  108. if isinstance(ext, Library):
  109. fn, ext = os.path.splitext(filename)
  110. return self.shlib_compiler.library_filename(fn, libtype)
  111. elif use_stubs and ext._links_to_dynamic:
  112. d, fn = os.path.split(filename)
  113. return os.path.join(d, 'dl-' + fn)
  114. return filename
  115. def initialize_options(self):
  116. _build_ext.initialize_options(self)
  117. self.shlib_compiler = None
  118. self.shlibs = []
  119. self.ext_map = {}
  120. def finalize_options(self):
  121. _build_ext.finalize_options(self)
  122. self.extensions = self.extensions or []
  123. self.check_extensions_list(self.extensions)
  124. self.shlibs = [ext for ext in self.extensions
  125. if isinstance(ext, Library)]
  126. if self.shlibs:
  127. self.setup_shlib_compiler()
  128. for ext in self.extensions:
  129. ext._full_name = self.get_ext_fullname(ext.name)
  130. for ext in self.extensions:
  131. fullname = ext._full_name
  132. self.ext_map[fullname] = ext
  133. # distutils 3.1 will also ask for module names
  134. # XXX what to do with conflicts?
  135. self.ext_map[fullname.split('.')[-1]] = ext
  136. ltd = self.shlibs and self.links_to_dynamic(ext) or False
  137. ns = ltd and use_stubs and not isinstance(ext, Library)
  138. ext._links_to_dynamic = ltd
  139. ext._needs_stub = ns
  140. filename = ext._file_name = self.get_ext_filename(fullname)
  141. libdir = os.path.dirname(os.path.join(self.build_lib, filename))
  142. if ltd and libdir not in ext.library_dirs:
  143. ext.library_dirs.append(libdir)
  144. if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs:
  145. ext.runtime_library_dirs.append(os.curdir)
  146. def setup_shlib_compiler(self):
  147. compiler = self.shlib_compiler = new_compiler(
  148. compiler=self.compiler, dry_run=self.dry_run, force=self.force
  149. )
  150. _customize_compiler_for_shlib(compiler)
  151. if self.include_dirs is not None:
  152. compiler.set_include_dirs(self.include_dirs)
  153. if self.define is not None:
  154. # 'define' option is a list of (name,value) tuples
  155. for (name, value) in self.define:
  156. compiler.define_macro(name, value)
  157. if self.undef is not None:
  158. for macro in self.undef:
  159. compiler.undefine_macro(macro)
  160. if self.libraries is not None:
  161. compiler.set_libraries(self.libraries)
  162. if self.library_dirs is not None:
  163. compiler.set_library_dirs(self.library_dirs)
  164. if self.rpath is not None:
  165. compiler.set_runtime_library_dirs(self.rpath)
  166. if self.link_objects is not None:
  167. compiler.set_link_objects(self.link_objects)
  168. # hack so distutils' build_extension() builds a library instead
  169. compiler.link_shared_object = link_shared_object.__get__(compiler)
  170. def get_export_symbols(self, ext):
  171. if isinstance(ext, Library):
  172. return ext.export_symbols
  173. return _build_ext.get_export_symbols(self, ext)
  174. def build_extension(self, ext):
  175. ext._convert_pyx_sources_to_lang()
  176. _compiler = self.compiler
  177. try:
  178. if isinstance(ext, Library):
  179. self.compiler = self.shlib_compiler
  180. _build_ext.build_extension(self, ext)
  181. if ext._needs_stub:
  182. cmd = self.get_finalized_command('build_py').build_lib
  183. self.write_stub(cmd, ext)
  184. finally:
  185. self.compiler = _compiler
  186. def links_to_dynamic(self, ext):
  187. """Return true if 'ext' links to a dynamic lib in the same package"""
  188. # XXX this should check to ensure the lib is actually being built
  189. # XXX as dynamic, and not just using a locally-found version or a
  190. # XXX static-compiled version
  191. libnames = dict.fromkeys([lib._full_name for lib in self.shlibs])
  192. pkg = '.'.join(ext._full_name.split('.')[:-1] + [''])
  193. return any(pkg + libname in libnames for libname in ext.libraries)
  194. def get_outputs(self):
  195. return _build_ext.get_outputs(self) + self.__get_stubs_outputs()
  196. def __get_stubs_outputs(self):
  197. # assemble the base name for each extension that needs a stub
  198. ns_ext_bases = (
  199. os.path.join(self.build_lib, *ext._full_name.split('.'))
  200. for ext in self.extensions
  201. if ext._needs_stub
  202. )
  203. # pair each base with the extension
  204. pairs = itertools.product(ns_ext_bases, self.__get_output_extensions())
  205. return list(base + fnext for base, fnext in pairs)
  206. def __get_output_extensions(self):
  207. yield '.py'
  208. yield '.pyc'
  209. if self.get_finalized_command('build_py').optimize:
  210. yield '.pyo'
  211. def write_stub(self, output_dir, ext, compile=False):
  212. log.info("writing stub loader for %s to %s", ext._full_name,
  213. output_dir)
  214. stub_file = (os.path.join(output_dir, *ext._full_name.split('.')) +
  215. '.py')
  216. if compile and os.path.exists(stub_file):
  217. raise DistutilsError(stub_file + " already exists! Please delete.")
  218. if not self.dry_run:
  219. f = open(stub_file, 'w')
  220. f.write(
  221. '\n'.join([
  222. "def __bootstrap__():",
  223. " global __bootstrap__, __file__, __loader__",
  224. " import sys, os, pkg_resources" + if_dl(", dl"),
  225. " from importlib.machinery import ExtensionFileLoader",
  226. " __file__ = pkg_resources.resource_filename"
  227. "(__name__,%r)"
  228. % os.path.basename(ext._file_name),
  229. " del __bootstrap__",
  230. " if '__loader__' in globals():",
  231. " del __loader__",
  232. if_dl(" old_flags = sys.getdlopenflags()"),
  233. " old_dir = os.getcwd()",
  234. " try:",
  235. " os.chdir(os.path.dirname(__file__))",
  236. if_dl(" sys.setdlopenflags(dl.RTLD_NOW)"),
  237. " ExtensionFileLoader(__name__,",
  238. " __file__).exec_module()",
  239. " finally:",
  240. if_dl(" sys.setdlopenflags(old_flags)"),
  241. " os.chdir(old_dir)",
  242. "__bootstrap__()",
  243. "" # terminal \n
  244. ])
  245. )
  246. f.close()
  247. if compile:
  248. from distutils.util import byte_compile
  249. byte_compile([stub_file], optimize=0,
  250. force=True, dry_run=self.dry_run)
  251. optimize = self.get_finalized_command('install_lib').optimize
  252. if optimize > 0:
  253. byte_compile([stub_file], optimize=optimize,
  254. force=True, dry_run=self.dry_run)
  255. if os.path.exists(stub_file) and not self.dry_run:
  256. os.unlink(stub_file)
  257. if use_stubs or os.name == 'nt':
  258. # Build shared libraries
  259. #
  260. def link_shared_object(
  261. self, objects, output_libname, output_dir=None, libraries=None,
  262. library_dirs=None, runtime_library_dirs=None, export_symbols=None,
  263. debug=0, extra_preargs=None, extra_postargs=None, build_temp=None,
  264. target_lang=None):
  265. self.link(
  266. self.SHARED_LIBRARY, objects, output_libname,
  267. output_dir, libraries, library_dirs, runtime_library_dirs,
  268. export_symbols, debug, extra_preargs, extra_postargs,
  269. build_temp, target_lang
  270. )
  271. else:
  272. # Build static libraries everywhere else
  273. libtype = 'static'
  274. def link_shared_object(
  275. self, objects, output_libname, output_dir=None, libraries=None,
  276. library_dirs=None, runtime_library_dirs=None, export_symbols=None,
  277. debug=0, extra_preargs=None, extra_postargs=None, build_temp=None,
  278. target_lang=None):
  279. # XXX we need to either disallow these attrs on Library instances,
  280. # or warn/abort here if set, or something...
  281. # libraries=None, library_dirs=None, runtime_library_dirs=None,
  282. # export_symbols=None, extra_preargs=None, extra_postargs=None,
  283. # build_temp=None
  284. assert output_dir is None # distutils build_ext doesn't pass this
  285. output_dir, filename = os.path.split(output_libname)
  286. basename, ext = os.path.splitext(filename)
  287. if self.library_filename("x").startswith('lib'):
  288. # strip 'lib' prefix; this is kludgy if some platform uses
  289. # a different prefix
  290. basename = basename[3:]
  291. self.create_static_lib(
  292. objects, basename, output_dir, debug, target_lang
  293. )