setup_common.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402
  1. from __future__ import division, absolute_import, print_function
  2. # Code common to build tools
  3. import sys
  4. import warnings
  5. import copy
  6. import binascii
  7. from numpy.distutils.misc_util import mingw32
  8. #-------------------
  9. # Versioning support
  10. #-------------------
  11. # How to change C_API_VERSION ?
  12. # - increase C_API_VERSION value
  13. # - record the hash for the new C API with the script cversions.py
  14. # and add the hash to cversions.txt
  15. # The hash values are used to remind developers when the C API number was not
  16. # updated - generates a MismatchCAPIWarning warning which is turned into an
  17. # exception for released version.
  18. # Binary compatibility version number. This number is increased whenever the
  19. # C-API is changed such that binary compatibility is broken, i.e. whenever a
  20. # recompile of extension modules is needed.
  21. C_ABI_VERSION = 0x01000009
  22. # Minor API version. This number is increased whenever a change is made to the
  23. # C-API -- whether it breaks binary compatibility or not. Some changes, such
  24. # as adding a function pointer to the end of the function table, can be made
  25. # without breaking binary compatibility. In this case, only the C_API_VERSION
  26. # (*not* C_ABI_VERSION) would be increased. Whenever binary compatibility is
  27. # broken, both C_API_VERSION and C_ABI_VERSION should be increased.
  28. #
  29. # 0x00000008 - 1.7.x
  30. # 0x00000009 - 1.8.x
  31. # 0x00000009 - 1.9.x
  32. # 0x0000000a - 1.10.x
  33. # 0x0000000a - 1.11.x
  34. # 0x0000000a - 1.12.x
  35. # 0x0000000b - 1.13.x
  36. # 0x0000000c - 1.14.x
  37. # 0x0000000c - 1.15.x
  38. # 0x0000000d - 1.16.x
  39. C_API_VERSION = 0x0000000d
  40. class MismatchCAPIWarning(Warning):
  41. pass
  42. def is_released(config):
  43. """Return True if a released version of numpy is detected."""
  44. from distutils.version import LooseVersion
  45. v = config.get_version('../version.py')
  46. if v is None:
  47. raise ValueError("Could not get version")
  48. pv = LooseVersion(vstring=v).version
  49. if len(pv) > 3:
  50. return False
  51. return True
  52. def get_api_versions(apiversion, codegen_dir):
  53. """
  54. Return current C API checksum and the recorded checksum.
  55. Return current C API checksum and the recorded checksum for the given
  56. version of the C API version.
  57. """
  58. # Compute the hash of the current API as defined in the .txt files in
  59. # code_generators
  60. sys.path.insert(0, codegen_dir)
  61. try:
  62. m = __import__('genapi')
  63. numpy_api = __import__('numpy_api')
  64. curapi_hash = m.fullapi_hash(numpy_api.full_api)
  65. apis_hash = m.get_versions_hash()
  66. finally:
  67. del sys.path[0]
  68. return curapi_hash, apis_hash[apiversion]
  69. def check_api_version(apiversion, codegen_dir):
  70. """Emits a MismacthCAPIWarning if the C API version needs updating."""
  71. curapi_hash, api_hash = get_api_versions(apiversion, codegen_dir)
  72. # If different hash, it means that the api .txt files in
  73. # codegen_dir have been updated without the API version being
  74. # updated. Any modification in those .txt files should be reflected
  75. # in the api and eventually abi versions.
  76. # To compute the checksum of the current API, use
  77. # code_generators/cversions.py script
  78. if not curapi_hash == api_hash:
  79. msg = ("API mismatch detected, the C API version "
  80. "numbers have to be updated. Current C api version is %d, "
  81. "with checksum %s, but recorded checksum for C API version %d in "
  82. "codegen_dir/cversions.txt is %s. If functions were added in the "
  83. "C API, you have to update C_API_VERSION in %s."
  84. )
  85. warnings.warn(msg % (apiversion, curapi_hash, apiversion, api_hash,
  86. __file__),
  87. MismatchCAPIWarning, stacklevel=2)
  88. # Mandatory functions: if not found, fail the build
  89. MANDATORY_FUNCS = ["sin", "cos", "tan", "sinh", "cosh", "tanh", "fabs",
  90. "floor", "ceil", "sqrt", "log10", "log", "exp", "asin",
  91. "acos", "atan", "fmod", 'modf', 'frexp', 'ldexp']
  92. # Standard functions which may not be available and for which we have a
  93. # replacement implementation. Note that some of these are C99 functions.
  94. OPTIONAL_STDFUNCS = ["expm1", "log1p", "acosh", "asinh", "atanh",
  95. "rint", "trunc", "exp2", "log2", "hypot", "atan2", "pow",
  96. "copysign", "nextafter", "ftello", "fseeko",
  97. "strtoll", "strtoull", "cbrt", "strtold_l", "fallocate",
  98. "backtrace", "madvise"]
  99. OPTIONAL_HEADERS = [
  100. # sse headers only enabled automatically on amd64/x32 builds
  101. "xmmintrin.h", # SSE
  102. "emmintrin.h", # SSE2
  103. "features.h", # for glibc version linux
  104. "xlocale.h", # see GH#8367
  105. "dlfcn.h", # dladdr
  106. "sys/mman.h", #madvise
  107. ]
  108. # optional gcc compiler builtins and their call arguments and optional a
  109. # required header and definition name (HAVE_ prepended)
  110. # call arguments are required as the compiler will do strict signature checking
  111. OPTIONAL_INTRINSICS = [("__builtin_isnan", '5.'),
  112. ("__builtin_isinf", '5.'),
  113. ("__builtin_isfinite", '5.'),
  114. ("__builtin_bswap32", '5u'),
  115. ("__builtin_bswap64", '5u'),
  116. ("__builtin_expect", '5, 0'),
  117. ("__builtin_mul_overflow", '5, 5, (int*)5'),
  118. # broken on OSX 10.11, make sure its not optimized away
  119. ("volatile int r = __builtin_cpu_supports", '"sse"',
  120. "stdio.h", "__BUILTIN_CPU_SUPPORTS"),
  121. # MMX only needed for icc, but some clangs don't have it
  122. ("_m_from_int64", '0', "emmintrin.h"),
  123. ("_mm_load_ps", '(float*)0', "xmmintrin.h"), # SSE
  124. ("_mm_prefetch", '(float*)0, _MM_HINT_NTA',
  125. "xmmintrin.h"), # SSE
  126. ("_mm_load_pd", '(double*)0', "emmintrin.h"), # SSE2
  127. ("__builtin_prefetch", "(float*)0, 0, 3"),
  128. # check that the linker can handle avx
  129. ("__asm__ volatile", '"vpand %xmm1, %xmm2, %xmm3"',
  130. "stdio.h", "LINK_AVX"),
  131. ("__asm__ volatile", '"vpand %ymm1, %ymm2, %ymm3"',
  132. "stdio.h", "LINK_AVX2"),
  133. ("__asm__ volatile", '"xgetbv"', "stdio.h", "XGETBV"),
  134. ]
  135. # function attributes
  136. # tested via "int %s %s(void *);" % (attribute, name)
  137. # function name will be converted to HAVE_<upper-case-name> preprocessor macro
  138. OPTIONAL_FUNCTION_ATTRIBUTES = [('__attribute__((optimize("unroll-loops")))',
  139. 'attribute_optimize_unroll_loops'),
  140. ('__attribute__((optimize("O3")))',
  141. 'attribute_optimize_opt_3'),
  142. ('__attribute__((nonnull (1)))',
  143. 'attribute_nonnull'),
  144. ('__attribute__((target ("avx")))',
  145. 'attribute_target_avx'),
  146. ('__attribute__((target ("avx2")))',
  147. 'attribute_target_avx2'),
  148. ]
  149. # variable attributes tested via "int %s a" % attribute
  150. OPTIONAL_VARIABLE_ATTRIBUTES = ["__thread", "__declspec(thread)"]
  151. # Subset of OPTIONAL_STDFUNCS which may already have HAVE_* defined by Python.h
  152. OPTIONAL_STDFUNCS_MAYBE = [
  153. "expm1", "log1p", "acosh", "atanh", "asinh", "hypot", "copysign",
  154. "ftello", "fseeko"
  155. ]
  156. # C99 functions: float and long double versions
  157. C99_FUNCS = [
  158. "sin", "cos", "tan", "sinh", "cosh", "tanh", "fabs", "floor", "ceil",
  159. "rint", "trunc", "sqrt", "log10", "log", "log1p", "exp", "expm1",
  160. "asin", "acos", "atan", "asinh", "acosh", "atanh", "hypot", "atan2",
  161. "pow", "fmod", "modf", 'frexp', 'ldexp', "exp2", "log2", "copysign",
  162. "nextafter", "cbrt"
  163. ]
  164. C99_FUNCS_SINGLE = [f + 'f' for f in C99_FUNCS]
  165. C99_FUNCS_EXTENDED = [f + 'l' for f in C99_FUNCS]
  166. C99_COMPLEX_TYPES = [
  167. 'complex double', 'complex float', 'complex long double'
  168. ]
  169. C99_COMPLEX_FUNCS = [
  170. "cabs", "cacos", "cacosh", "carg", "casin", "casinh", "catan",
  171. "catanh", "ccos", "ccosh", "cexp", "cimag", "clog", "conj", "cpow",
  172. "cproj", "creal", "csin", "csinh", "csqrt", "ctan", "ctanh"
  173. ]
  174. def fname2def(name):
  175. return "HAVE_%s" % name.upper()
  176. def sym2def(symbol):
  177. define = symbol.replace(' ', '')
  178. return define.upper()
  179. def type2def(symbol):
  180. define = symbol.replace(' ', '_')
  181. return define.upper()
  182. # Code to detect long double representation taken from MPFR m4 macro
  183. def check_long_double_representation(cmd):
  184. cmd._check_compiler()
  185. body = LONG_DOUBLE_REPRESENTATION_SRC % {'type': 'long double'}
  186. # Disable whole program optimization (the default on vs2015, with python 3.5+)
  187. # which generates intermediary object files and prevents checking the
  188. # float representation.
  189. if sys.platform == "win32" and not mingw32():
  190. try:
  191. cmd.compiler.compile_options.remove("/GL")
  192. except (AttributeError, ValueError):
  193. pass
  194. # Disable multi-file interprocedural optimization in the Intel compiler on Linux
  195. # which generates intermediary object files and prevents checking the
  196. # float representation.
  197. elif (sys.platform != "win32"
  198. and cmd.compiler.compiler_type.startswith('intel')
  199. and '-ipo' in cmd.compiler.cc_exe):
  200. newcompiler = cmd.compiler.cc_exe.replace(' -ipo', '')
  201. cmd.compiler.set_executables(
  202. compiler=newcompiler,
  203. compiler_so=newcompiler,
  204. compiler_cxx=newcompiler,
  205. linker_exe=newcompiler,
  206. linker_so=newcompiler + ' -shared'
  207. )
  208. # We need to use _compile because we need the object filename
  209. src, obj = cmd._compile(body, None, None, 'c')
  210. try:
  211. ltype = long_double_representation(pyod(obj))
  212. return ltype
  213. except ValueError:
  214. # try linking to support CC="gcc -flto" or icc -ipo
  215. # struct needs to be volatile so it isn't optimized away
  216. # additionally "clang -flto" requires the foo struct to be used
  217. body = body.replace('struct', 'volatile struct')
  218. body += "int main(void) { return foo.before[0]; }\n"
  219. src, obj = cmd._compile(body, None, None, 'c')
  220. cmd.temp_files.append("_configtest")
  221. cmd.compiler.link_executable([obj], "_configtest")
  222. ltype = long_double_representation(pyod("_configtest"))
  223. return ltype
  224. finally:
  225. cmd._clean()
  226. LONG_DOUBLE_REPRESENTATION_SRC = r"""
  227. /* "before" is 16 bytes to ensure there's no padding between it and "x".
  228. * We're not expecting any "long double" bigger than 16 bytes or with
  229. * alignment requirements stricter than 16 bytes. */
  230. typedef %(type)s test_type;
  231. struct {
  232. char before[16];
  233. test_type x;
  234. char after[8];
  235. } foo = {
  236. { '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0',
  237. '\001', '\043', '\105', '\147', '\211', '\253', '\315', '\357' },
  238. -123456789.0,
  239. { '\376', '\334', '\272', '\230', '\166', '\124', '\062', '\020' }
  240. };
  241. """
  242. def pyod(filename):
  243. """Python implementation of the od UNIX utility (od -b, more exactly).
  244. Parameters
  245. ----------
  246. filename : str
  247. name of the file to get the dump from.
  248. Returns
  249. -------
  250. out : seq
  251. list of lines of od output
  252. Note
  253. ----
  254. We only implement enough to get the necessary information for long double
  255. representation, this is not intended as a compatible replacement for od.
  256. """
  257. def _pyod2():
  258. out = []
  259. fid = open(filename, 'rb')
  260. try:
  261. yo = [int(oct(int(binascii.b2a_hex(o), 16))) for o in fid.read()]
  262. for i in range(0, len(yo), 16):
  263. line = ['%07d' % int(oct(i))]
  264. line.extend(['%03d' % c for c in yo[i:i+16]])
  265. out.append(" ".join(line))
  266. return out
  267. finally:
  268. fid.close()
  269. def _pyod3():
  270. out = []
  271. fid = open(filename, 'rb')
  272. try:
  273. yo2 = [oct(o)[2:] for o in fid.read()]
  274. for i in range(0, len(yo2), 16):
  275. line = ['%07d' % int(oct(i)[2:])]
  276. line.extend(['%03d' % int(c) for c in yo2[i:i+16]])
  277. out.append(" ".join(line))
  278. return out
  279. finally:
  280. fid.close()
  281. if sys.version_info[0] < 3:
  282. return _pyod2()
  283. else:
  284. return _pyod3()
  285. _BEFORE_SEQ = ['000', '000', '000', '000', '000', '000', '000', '000',
  286. '001', '043', '105', '147', '211', '253', '315', '357']
  287. _AFTER_SEQ = ['376', '334', '272', '230', '166', '124', '062', '020']
  288. _IEEE_DOUBLE_BE = ['301', '235', '157', '064', '124', '000', '000', '000']
  289. _IEEE_DOUBLE_LE = _IEEE_DOUBLE_BE[::-1]
  290. _INTEL_EXTENDED_12B = ['000', '000', '000', '000', '240', '242', '171', '353',
  291. '031', '300', '000', '000']
  292. _INTEL_EXTENDED_16B = ['000', '000', '000', '000', '240', '242', '171', '353',
  293. '031', '300', '000', '000', '000', '000', '000', '000']
  294. _MOTOROLA_EXTENDED_12B = ['300', '031', '000', '000', '353', '171',
  295. '242', '240', '000', '000', '000', '000']
  296. _IEEE_QUAD_PREC_BE = ['300', '031', '326', '363', '105', '100', '000', '000',
  297. '000', '000', '000', '000', '000', '000', '000', '000']
  298. _IEEE_QUAD_PREC_LE = _IEEE_QUAD_PREC_BE[::-1]
  299. _IBM_DOUBLE_DOUBLE_BE = (['301', '235', '157', '064', '124', '000', '000', '000'] +
  300. ['000'] * 8)
  301. _IBM_DOUBLE_DOUBLE_LE = (['000', '000', '000', '124', '064', '157', '235', '301'] +
  302. ['000'] * 8)
  303. def long_double_representation(lines):
  304. """Given a binary dump as given by GNU od -b, look for long double
  305. representation."""
  306. # Read contains a list of 32 items, each item is a byte (in octal
  307. # representation, as a string). We 'slide' over the output until read is of
  308. # the form before_seq + content + after_sequence, where content is the long double
  309. # representation:
  310. # - content is 12 bytes: 80 bits Intel representation
  311. # - content is 16 bytes: 80 bits Intel representation (64 bits) or quad precision
  312. # - content is 8 bytes: same as double (not implemented yet)
  313. read = [''] * 32
  314. saw = None
  315. for line in lines:
  316. # we skip the first word, as od -b output an index at the beginning of
  317. # each line
  318. for w in line.split()[1:]:
  319. read.pop(0)
  320. read.append(w)
  321. # If the end of read is equal to the after_sequence, read contains
  322. # the long double
  323. if read[-8:] == _AFTER_SEQ:
  324. saw = copy.copy(read)
  325. # if the content was 12 bytes, we only have 32 - 8 - 12 = 12
  326. # "before" bytes. In other words the first 4 "before" bytes went
  327. # past the sliding window.
  328. if read[:12] == _BEFORE_SEQ[4:]:
  329. if read[12:-8] == _INTEL_EXTENDED_12B:
  330. return 'INTEL_EXTENDED_12_BYTES_LE'
  331. if read[12:-8] == _MOTOROLA_EXTENDED_12B:
  332. return 'MOTOROLA_EXTENDED_12_BYTES_BE'
  333. # if the content was 16 bytes, we are left with 32-8-16 = 16
  334. # "before" bytes, so 8 went past the sliding window.
  335. elif read[:8] == _BEFORE_SEQ[8:]:
  336. if read[8:-8] == _INTEL_EXTENDED_16B:
  337. return 'INTEL_EXTENDED_16_BYTES_LE'
  338. elif read[8:-8] == _IEEE_QUAD_PREC_BE:
  339. return 'IEEE_QUAD_BE'
  340. elif read[8:-8] == _IEEE_QUAD_PREC_LE:
  341. return 'IEEE_QUAD_LE'
  342. elif read[8:-8] == _IBM_DOUBLE_DOUBLE_LE:
  343. return 'IBM_DOUBLE_DOUBLE_LE'
  344. elif read[8:-8] == _IBM_DOUBLE_DOUBLE_BE:
  345. return 'IBM_DOUBLE_DOUBLE_BE'
  346. # if the content was 8 bytes, left with 32-8-8 = 16 bytes
  347. elif read[:16] == _BEFORE_SEQ:
  348. if read[16:-8] == _IEEE_DOUBLE_LE:
  349. return 'IEEE_DOUBLE_LE'
  350. elif read[16:-8] == _IEEE_DOUBLE_BE:
  351. return 'IEEE_DOUBLE_BE'
  352. if saw is not None:
  353. raise ValueError("Unrecognized format (%s)" % saw)
  354. else:
  355. # We never detected the after_sequence
  356. raise ValueError("Could not lock sequences (%s)" % saw)