MemoryView.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856
  1. from __future__ import absolute_import
  2. from .Errors import CompileError, error
  3. from . import ExprNodes
  4. from .ExprNodes import IntNode, NameNode, AttributeNode
  5. from . import Options
  6. from .Code import UtilityCode, TempitaUtilityCode
  7. from .UtilityCode import CythonUtilityCode
  8. from . import Buffer
  9. from . import PyrexTypes
  10. from . import ModuleNode
  11. START_ERR = "Start must not be given."
  12. STOP_ERR = "Axis specification only allowed in the 'step' slot."
  13. STEP_ERR = "Step must be omitted, 1, or a valid specifier."
  14. BOTH_CF_ERR = "Cannot specify an array that is both C and Fortran contiguous."
  15. INVALID_ERR = "Invalid axis specification."
  16. NOT_CIMPORTED_ERR = "Variable was not cimported from cython.view"
  17. EXPR_ERR = "no expressions allowed in axis spec, only names and literals."
  18. CF_ERR = "Invalid axis specification for a C/Fortran contiguous array."
  19. ERR_UNINITIALIZED = ("Cannot check if memoryview %s is initialized without the "
  20. "GIL, consider using initializedcheck(False)")
  21. def concat_flags(*flags):
  22. return "(%s)" % "|".join(flags)
  23. format_flag = "PyBUF_FORMAT"
  24. memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT)"
  25. memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT)"
  26. memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT)"
  27. memview_full_access = "PyBUF_FULL_RO"
  28. #memview_strided_access = "PyBUF_STRIDED_RO"
  29. memview_strided_access = "PyBUF_RECORDS_RO"
  30. MEMVIEW_DIRECT = '__Pyx_MEMVIEW_DIRECT'
  31. MEMVIEW_PTR = '__Pyx_MEMVIEW_PTR'
  32. MEMVIEW_FULL = '__Pyx_MEMVIEW_FULL'
  33. MEMVIEW_CONTIG = '__Pyx_MEMVIEW_CONTIG'
  34. MEMVIEW_STRIDED= '__Pyx_MEMVIEW_STRIDED'
  35. MEMVIEW_FOLLOW = '__Pyx_MEMVIEW_FOLLOW'
  36. _spec_to_const = {
  37. 'direct' : MEMVIEW_DIRECT,
  38. 'ptr' : MEMVIEW_PTR,
  39. 'full' : MEMVIEW_FULL,
  40. 'contig' : MEMVIEW_CONTIG,
  41. 'strided': MEMVIEW_STRIDED,
  42. 'follow' : MEMVIEW_FOLLOW,
  43. }
  44. _spec_to_abbrev = {
  45. 'direct' : 'd',
  46. 'ptr' : 'p',
  47. 'full' : 'f',
  48. 'contig' : 'c',
  49. 'strided' : 's',
  50. 'follow' : '_',
  51. }
  52. memslice_entry_init = "{ 0, 0, { 0 }, { 0 }, { 0 } }"
  53. memview_name = u'memoryview'
  54. memview_typeptr_cname = '__pyx_memoryview_type'
  55. memview_objstruct_cname = '__pyx_memoryview_obj'
  56. memviewslice_cname = u'__Pyx_memviewslice'
  57. def put_init_entry(mv_cname, code):
  58. code.putln("%s.data = NULL;" % mv_cname)
  59. code.putln("%s.memview = NULL;" % mv_cname)
  60. #def axes_to_str(axes):
  61. # return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
  62. def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code,
  63. have_gil=False, first_assignment=True):
  64. "We can avoid decreffing the lhs if we know it is the first assignment"
  65. assert rhs.type.is_memoryviewslice
  66. pretty_rhs = rhs.result_in_temp() or rhs.is_simple()
  67. if pretty_rhs:
  68. rhstmp = rhs.result()
  69. else:
  70. rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False)
  71. code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type)))
  72. # Allow uninitialized assignment
  73. #code.putln(code.put_error_if_unbound(lhs_pos, rhs.entry))
  74. put_assign_to_memviewslice(lhs_cname, rhs, rhstmp, lhs_type, code,
  75. have_gil=have_gil, first_assignment=first_assignment)
  76. if not pretty_rhs:
  77. code.funcstate.release_temp(rhstmp)
  78. def put_assign_to_memviewslice(lhs_cname, rhs, rhs_cname, memviewslicetype, code,
  79. have_gil=False, first_assignment=False):
  80. if not first_assignment:
  81. code.put_xdecref_memoryviewslice(lhs_cname, have_gil=have_gil)
  82. if not rhs.result_in_temp():
  83. rhs.make_owned_memoryviewslice(code)
  84. code.putln("%s = %s;" % (lhs_cname, rhs_cname))
  85. def get_buf_flags(specs):
  86. is_c_contig, is_f_contig = is_cf_contig(specs)
  87. if is_c_contig:
  88. return memview_c_contiguous
  89. elif is_f_contig:
  90. return memview_f_contiguous
  91. access, packing = zip(*specs)
  92. if 'full' in access or 'ptr' in access:
  93. return memview_full_access
  94. else:
  95. return memview_strided_access
  96. def insert_newaxes(memoryviewtype, n):
  97. axes = [('direct', 'strided')] * n
  98. axes.extend(memoryviewtype.axes)
  99. return PyrexTypes.MemoryViewSliceType(memoryviewtype.dtype, axes)
  100. def broadcast_types(src, dst):
  101. n = abs(src.ndim - dst.ndim)
  102. if src.ndim < dst.ndim:
  103. return insert_newaxes(src, n), dst
  104. else:
  105. return src, insert_newaxes(dst, n)
  106. def valid_memslice_dtype(dtype, i=0):
  107. """
  108. Return whether type dtype can be used as the base type of a
  109. memoryview slice.
  110. We support structs, numeric types and objects
  111. """
  112. if dtype.is_complex and dtype.real_type.is_int:
  113. return False
  114. if dtype is PyrexTypes.c_bint_type:
  115. return False
  116. if dtype.is_struct and dtype.kind == 'struct':
  117. for member in dtype.scope.var_entries:
  118. if not valid_memslice_dtype(member.type):
  119. return False
  120. return True
  121. return (
  122. dtype.is_error or
  123. # Pointers are not valid (yet)
  124. # (dtype.is_ptr and valid_memslice_dtype(dtype.base_type)) or
  125. (dtype.is_array and i < 8 and
  126. valid_memslice_dtype(dtype.base_type, i + 1)) or
  127. dtype.is_numeric or
  128. dtype.is_pyobject or
  129. dtype.is_fused or # accept this as it will be replaced by specializations later
  130. (dtype.is_typedef and valid_memslice_dtype(dtype.typedef_base_type))
  131. )
  132. class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
  133. """
  134. May be used during code generation time to be queried for
  135. shape/strides/suboffsets attributes, or to perform indexing or slicing.
  136. """
  137. def __init__(self, entry):
  138. self.entry = entry
  139. self.type = entry.type
  140. self.cname = entry.cname
  141. self.buf_ptr = "%s.data" % self.cname
  142. dtype = self.entry.type.dtype
  143. self.buf_ptr_type = PyrexTypes.CPtrType(dtype)
  144. self.init_attributes()
  145. def get_buf_suboffsetvars(self):
  146. return self._for_all_ndim("%s.suboffsets[%d]")
  147. def get_buf_stridevars(self):
  148. return self._for_all_ndim("%s.strides[%d]")
  149. def get_buf_shapevars(self):
  150. return self._for_all_ndim("%s.shape[%d]")
  151. def generate_buffer_lookup_code(self, code, index_cnames):
  152. axes = [(dim, index_cnames[dim], access, packing)
  153. for dim, (access, packing) in enumerate(self.type.axes)]
  154. return self._generate_buffer_lookup_code(code, axes)
  155. def _generate_buffer_lookup_code(self, code, axes, cast_result=True):
  156. """
  157. Generate a single expression that indexes the memory view slice
  158. in each dimension.
  159. """
  160. bufp = self.buf_ptr
  161. type_decl = self.type.dtype.empty_declaration_code()
  162. for dim, index, access, packing in axes:
  163. shape = "%s.shape[%d]" % (self.cname, dim)
  164. stride = "%s.strides[%d]" % (self.cname, dim)
  165. suboffset = "%s.suboffsets[%d]" % (self.cname, dim)
  166. flag = get_memoryview_flag(access, packing)
  167. if flag in ("generic", "generic_contiguous"):
  168. # Note: we cannot do cast tricks to avoid stride multiplication
  169. # for generic_contiguous, as we may have to do (dtype *)
  170. # or (dtype **) arithmetic, we won't know which unless
  171. # we check suboffsets
  172. code.globalstate.use_utility_code(memviewslice_index_helpers)
  173. bufp = ('__pyx_memviewslice_index_full(%s, %s, %s, %s)' %
  174. (bufp, index, stride, suboffset))
  175. elif flag == "indirect":
  176. bufp = "(%s + %s * %s)" % (bufp, index, stride)
  177. bufp = ("(*((char **) %s) + %s)" % (bufp, suboffset))
  178. elif flag == "indirect_contiguous":
  179. # Note: we do char ** arithmetic
  180. bufp = "(*((char **) %s + %s) + %s)" % (bufp, index, suboffset)
  181. elif flag == "strided":
  182. bufp = "(%s + %s * %s)" % (bufp, index, stride)
  183. else:
  184. assert flag == 'contiguous', flag
  185. bufp = '((char *) (((%s *) %s) + %s))' % (type_decl, bufp, index)
  186. bufp = '( /* dim=%d */ %s )' % (dim, bufp)
  187. if cast_result:
  188. return "((%s *) %s)" % (type_decl, bufp)
  189. return bufp
  190. def generate_buffer_slice_code(self, code, indices, dst, have_gil,
  191. have_slices, directives):
  192. """
  193. Slice a memoryviewslice.
  194. indices - list of index nodes. If not a SliceNode, or NoneNode,
  195. then it must be coercible to Py_ssize_t
  196. Simply call __pyx_memoryview_slice_memviewslice with the right
  197. arguments, unless the dimension is omitted or a bare ':', in which
  198. case we copy over the shape/strides/suboffsets attributes directly
  199. for that dimension.
  200. """
  201. src = self.cname
  202. code.putln("%(dst)s.data = %(src)s.data;" % locals())
  203. code.putln("%(dst)s.memview = %(src)s.memview;" % locals())
  204. code.put_incref_memoryviewslice(dst)
  205. all_dimensions_direct = all(access == 'direct' for access, packing in self.type.axes)
  206. suboffset_dim_temp = []
  207. def get_suboffset_dim():
  208. # create global temp variable at request
  209. if not suboffset_dim_temp:
  210. suboffset_dim = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
  211. code.putln("%s = -1;" % suboffset_dim)
  212. suboffset_dim_temp.append(suboffset_dim)
  213. return suboffset_dim_temp[0]
  214. dim = -1
  215. new_ndim = 0
  216. for index in indices:
  217. if index.is_none:
  218. # newaxis
  219. for attrib, value in [('shape', 1), ('strides', 0), ('suboffsets', -1)]:
  220. code.putln("%s.%s[%d] = %d;" % (dst, attrib, new_ndim, value))
  221. new_ndim += 1
  222. continue
  223. dim += 1
  224. access, packing = self.type.axes[dim]
  225. error_goto = code.error_goto(index.pos)
  226. if isinstance(index, ExprNodes.SliceNode):
  227. # slice, unspecified dimension, or part of ellipsis
  228. d = dict(locals())
  229. for s in "start stop step".split():
  230. idx = getattr(index, s)
  231. have_idx = d['have_' + s] = not idx.is_none
  232. d[s] = idx.result() if have_idx else "0"
  233. if not (d['have_start'] or d['have_stop'] or d['have_step']):
  234. # full slice (:), simply copy over the extent, stride
  235. # and suboffset. Also update suboffset_dim if needed
  236. d['access'] = access
  237. util_name = "SimpleSlice"
  238. else:
  239. util_name = "ToughSlice"
  240. new_ndim += 1
  241. else:
  242. # normal index
  243. idx = index.result()
  244. indirect = access != 'direct'
  245. if indirect:
  246. generic = access == 'full'
  247. if new_ndim != 0:
  248. return error(index.pos,
  249. "All preceding dimensions must be "
  250. "indexed and not sliced")
  251. d = dict(
  252. locals(),
  253. wraparound=int(directives['wraparound']),
  254. boundscheck=int(directives['boundscheck'])
  255. )
  256. util_name = "SliceIndex"
  257. _, impl = TempitaUtilityCode.load_as_string(util_name, "MemoryView_C.c", context=d)
  258. code.put(impl)
  259. if suboffset_dim_temp:
  260. code.funcstate.release_temp(suboffset_dim_temp[0])
  261. def empty_slice(pos):
  262. none = ExprNodes.NoneNode(pos)
  263. return ExprNodes.SliceNode(pos, start=none,
  264. stop=none, step=none)
  265. def unellipsify(indices, ndim):
  266. result = []
  267. seen_ellipsis = False
  268. have_slices = False
  269. newaxes = [newaxis for newaxis in indices if newaxis.is_none]
  270. n_indices = len(indices) - len(newaxes)
  271. for index in indices:
  272. if isinstance(index, ExprNodes.EllipsisNode):
  273. have_slices = True
  274. full_slice = empty_slice(index.pos)
  275. if seen_ellipsis:
  276. result.append(full_slice)
  277. else:
  278. nslices = ndim - n_indices + 1
  279. result.extend([full_slice] * nslices)
  280. seen_ellipsis = True
  281. else:
  282. have_slices = have_slices or index.is_slice or index.is_none
  283. result.append(index)
  284. result_length = len(result) - len(newaxes)
  285. if result_length < ndim:
  286. have_slices = True
  287. nslices = ndim - result_length
  288. result.extend([empty_slice(indices[-1].pos)] * nslices)
  289. return have_slices, result, newaxes
  290. def get_memoryview_flag(access, packing):
  291. if access == 'full' and packing in ('strided', 'follow'):
  292. return 'generic'
  293. elif access == 'full' and packing == 'contig':
  294. return 'generic_contiguous'
  295. elif access == 'ptr' and packing in ('strided', 'follow'):
  296. return 'indirect'
  297. elif access == 'ptr' and packing == 'contig':
  298. return 'indirect_contiguous'
  299. elif access == 'direct' and packing in ('strided', 'follow'):
  300. return 'strided'
  301. else:
  302. assert (access, packing) == ('direct', 'contig'), (access, packing)
  303. return 'contiguous'
  304. def get_is_contig_func_name(contig_type, ndim):
  305. assert contig_type in ('C', 'F')
  306. return "__pyx_memviewslice_is_contig_%s%d" % (contig_type, ndim)
  307. def get_is_contig_utility(contig_type, ndim):
  308. assert contig_type in ('C', 'F')
  309. C = dict(context, ndim=ndim, contig_type=contig_type)
  310. utility = load_memview_c_utility("MemviewSliceCheckContig", C, requires=[is_contig_utility])
  311. return utility
  312. def slice_iter(slice_type, slice_result, ndim, code):
  313. if slice_type.is_c_contig or slice_type.is_f_contig:
  314. return ContigSliceIter(slice_type, slice_result, ndim, code)
  315. else:
  316. return StridedSliceIter(slice_type, slice_result, ndim, code)
  317. class SliceIter(object):
  318. def __init__(self, slice_type, slice_result, ndim, code):
  319. self.slice_type = slice_type
  320. self.slice_result = slice_result
  321. self.code = code
  322. self.ndim = ndim
  323. class ContigSliceIter(SliceIter):
  324. def start_loops(self):
  325. code = self.code
  326. code.begin_block()
  327. type_decl = self.slice_type.dtype.empty_declaration_code()
  328. total_size = ' * '.join("%s.shape[%d]" % (self.slice_result, i)
  329. for i in range(self.ndim))
  330. code.putln("Py_ssize_t __pyx_temp_extent = %s;" % total_size)
  331. code.putln("Py_ssize_t __pyx_temp_idx;")
  332. code.putln("%s *__pyx_temp_pointer = (%s *) %s.data;" % (
  333. type_decl, type_decl, self.slice_result))
  334. code.putln("for (__pyx_temp_idx = 0; "
  335. "__pyx_temp_idx < __pyx_temp_extent; "
  336. "__pyx_temp_idx++) {")
  337. return "__pyx_temp_pointer"
  338. def end_loops(self):
  339. self.code.putln("__pyx_temp_pointer += 1;")
  340. self.code.putln("}")
  341. self.code.end_block()
  342. class StridedSliceIter(SliceIter):
  343. def start_loops(self):
  344. code = self.code
  345. code.begin_block()
  346. for i in range(self.ndim):
  347. t = i, self.slice_result, i
  348. code.putln("Py_ssize_t __pyx_temp_extent_%d = %s.shape[%d];" % t)
  349. code.putln("Py_ssize_t __pyx_temp_stride_%d = %s.strides[%d];" % t)
  350. code.putln("char *__pyx_temp_pointer_%d;" % i)
  351. code.putln("Py_ssize_t __pyx_temp_idx_%d;" % i)
  352. code.putln("__pyx_temp_pointer_0 = %s.data;" % self.slice_result)
  353. for i in range(self.ndim):
  354. if i > 0:
  355. code.putln("__pyx_temp_pointer_%d = __pyx_temp_pointer_%d;" % (i, i - 1))
  356. code.putln("for (__pyx_temp_idx_%d = 0; "
  357. "__pyx_temp_idx_%d < __pyx_temp_extent_%d; "
  358. "__pyx_temp_idx_%d++) {" % (i, i, i, i))
  359. return "__pyx_temp_pointer_%d" % (self.ndim - 1)
  360. def end_loops(self):
  361. code = self.code
  362. for i in range(self.ndim - 1, -1, -1):
  363. code.putln("__pyx_temp_pointer_%d += __pyx_temp_stride_%d;" % (i, i))
  364. code.putln("}")
  365. code.end_block()
  366. def copy_c_or_fortran_cname(memview):
  367. if memview.is_c_contig:
  368. c_or_f = 'c'
  369. else:
  370. c_or_f = 'f'
  371. return "__pyx_memoryview_copy_slice_%s_%s" % (
  372. memview.specialization_suffix(), c_or_f)
  373. def get_copy_new_utility(pos, from_memview, to_memview):
  374. if (from_memview.dtype != to_memview.dtype and
  375. not (from_memview.dtype.is_const and from_memview.dtype.const_base_type == to_memview.dtype)):
  376. error(pos, "dtypes must be the same!")
  377. return
  378. if len(from_memview.axes) != len(to_memview.axes):
  379. error(pos, "number of dimensions must be same")
  380. return
  381. if not (to_memview.is_c_contig or to_memview.is_f_contig):
  382. error(pos, "to_memview must be c or f contiguous.")
  383. return
  384. for (access, packing) in from_memview.axes:
  385. if access != 'direct':
  386. error(pos, "cannot handle 'full' or 'ptr' access at this time.")
  387. return
  388. if to_memview.is_c_contig:
  389. mode = 'c'
  390. contig_flag = memview_c_contiguous
  391. elif to_memview.is_f_contig:
  392. mode = 'fortran'
  393. contig_flag = memview_f_contiguous
  394. return load_memview_c_utility(
  395. "CopyContentsUtility",
  396. context=dict(
  397. context,
  398. mode=mode,
  399. dtype_decl=to_memview.dtype.empty_declaration_code(),
  400. contig_flag=contig_flag,
  401. ndim=to_memview.ndim,
  402. func_cname=copy_c_or_fortran_cname(to_memview),
  403. dtype_is_object=int(to_memview.dtype.is_pyobject)),
  404. requires=[copy_contents_new_utility])
  405. def get_axes_specs(env, axes):
  406. '''
  407. get_axes_specs(env, axes) -> list of (access, packing) specs for each axis.
  408. access is one of 'full', 'ptr' or 'direct'
  409. packing is one of 'contig', 'strided' or 'follow'
  410. '''
  411. cythonscope = env.global_scope().context.cython_scope
  412. cythonscope.load_cythonscope()
  413. viewscope = cythonscope.viewscope
  414. access_specs = tuple([viewscope.lookup(name)
  415. for name in ('full', 'direct', 'ptr')])
  416. packing_specs = tuple([viewscope.lookup(name)
  417. for name in ('contig', 'strided', 'follow')])
  418. is_f_contig, is_c_contig = False, False
  419. default_access, default_packing = 'direct', 'strided'
  420. cf_access, cf_packing = default_access, 'follow'
  421. axes_specs = []
  422. # analyse all axes.
  423. for idx, axis in enumerate(axes):
  424. if not axis.start.is_none:
  425. raise CompileError(axis.start.pos, START_ERR)
  426. if not axis.stop.is_none:
  427. raise CompileError(axis.stop.pos, STOP_ERR)
  428. if axis.step.is_none:
  429. axes_specs.append((default_access, default_packing))
  430. elif isinstance(axis.step, IntNode):
  431. # the packing for the ::1 axis is contiguous,
  432. # all others are cf_packing.
  433. if axis.step.compile_time_value(env) != 1:
  434. raise CompileError(axis.step.pos, STEP_ERR)
  435. axes_specs.append((cf_access, 'cfcontig'))
  436. elif isinstance(axis.step, (NameNode, AttributeNode)):
  437. entry = _get_resolved_spec(env, axis.step)
  438. if entry.name in view_constant_to_access_packing:
  439. axes_specs.append(view_constant_to_access_packing[entry.name])
  440. else:
  441. raise CompileError(axis.step.pos, INVALID_ERR)
  442. else:
  443. raise CompileError(axis.step.pos, INVALID_ERR)
  444. # First, find out if we have a ::1 somewhere
  445. contig_dim = 0
  446. is_contig = False
  447. for idx, (access, packing) in enumerate(axes_specs):
  448. if packing == 'cfcontig':
  449. if is_contig:
  450. raise CompileError(axis.step.pos, BOTH_CF_ERR)
  451. contig_dim = idx
  452. axes_specs[idx] = (access, 'contig')
  453. is_contig = True
  454. if is_contig:
  455. # We have a ::1 somewhere, see if we're C or Fortran contiguous
  456. if contig_dim == len(axes) - 1:
  457. is_c_contig = True
  458. else:
  459. is_f_contig = True
  460. if contig_dim and not axes_specs[contig_dim - 1][0] in ('full', 'ptr'):
  461. raise CompileError(axes[contig_dim].pos,
  462. "Fortran contiguous specifier must follow an indirect dimension")
  463. if is_c_contig:
  464. # Contiguous in the last dimension, find the last indirect dimension
  465. contig_dim = -1
  466. for idx, (access, packing) in enumerate(reversed(axes_specs)):
  467. if access in ('ptr', 'full'):
  468. contig_dim = len(axes) - idx - 1
  469. # Replace 'strided' with 'follow' for any dimension following the last
  470. # indirect dimension, the first dimension or the dimension following
  471. # the ::1.
  472. # int[::indirect, ::1, :, :]
  473. # ^ ^
  474. # int[::indirect, :, :, ::1]
  475. # ^ ^
  476. start = contig_dim + 1
  477. stop = len(axes) - is_c_contig
  478. for idx, (access, packing) in enumerate(axes_specs[start:stop]):
  479. idx = contig_dim + 1 + idx
  480. if access != 'direct':
  481. raise CompileError(axes[idx].pos,
  482. "Indirect dimension may not follow "
  483. "Fortran contiguous dimension")
  484. if packing == 'contig':
  485. raise CompileError(axes[idx].pos,
  486. "Dimension may not be contiguous")
  487. axes_specs[idx] = (access, cf_packing)
  488. if is_c_contig:
  489. # For C contiguity, we need to fix the 'contig' dimension
  490. # after the loop
  491. a, p = axes_specs[-1]
  492. axes_specs[-1] = a, 'contig'
  493. validate_axes_specs([axis.start.pos for axis in axes],
  494. axes_specs,
  495. is_c_contig,
  496. is_f_contig)
  497. return axes_specs
  498. def validate_axes(pos, axes):
  499. if len(axes) >= Options.buffer_max_dims:
  500. error(pos, "More dimensions than the maximum number"
  501. " of buffer dimensions were used.")
  502. return False
  503. return True
  504. def is_cf_contig(specs):
  505. is_c_contig = is_f_contig = False
  506. if len(specs) == 1 and specs == [('direct', 'contig')]:
  507. is_c_contig = True
  508. elif (specs[-1] == ('direct','contig') and
  509. all(axis == ('direct','follow') for axis in specs[:-1])):
  510. # c_contiguous: 'follow', 'follow', ..., 'follow', 'contig'
  511. is_c_contig = True
  512. elif (len(specs) > 1 and
  513. specs[0] == ('direct','contig') and
  514. all(axis == ('direct','follow') for axis in specs[1:])):
  515. # f_contiguous: 'contig', 'follow', 'follow', ..., 'follow'
  516. is_f_contig = True
  517. return is_c_contig, is_f_contig
  518. def get_mode(specs):
  519. is_c_contig, is_f_contig = is_cf_contig(specs)
  520. if is_c_contig:
  521. return 'c'
  522. elif is_f_contig:
  523. return 'fortran'
  524. for access, packing in specs:
  525. if access in ('ptr', 'full'):
  526. return 'full'
  527. return 'strided'
  528. view_constant_to_access_packing = {
  529. 'generic': ('full', 'strided'),
  530. 'strided': ('direct', 'strided'),
  531. 'indirect': ('ptr', 'strided'),
  532. 'generic_contiguous': ('full', 'contig'),
  533. 'contiguous': ('direct', 'contig'),
  534. 'indirect_contiguous': ('ptr', 'contig'),
  535. }
  536. def validate_axes_specs(positions, specs, is_c_contig, is_f_contig):
  537. packing_specs = ('contig', 'strided', 'follow')
  538. access_specs = ('direct', 'ptr', 'full')
  539. # is_c_contig, is_f_contig = is_cf_contig(specs)
  540. has_contig = has_follow = has_strided = has_generic_contig = False
  541. last_indirect_dimension = -1
  542. for idx, (access, packing) in enumerate(specs):
  543. if access == 'ptr':
  544. last_indirect_dimension = idx
  545. for idx, (pos, (access, packing)) in enumerate(zip(positions, specs)):
  546. if not (access in access_specs and
  547. packing in packing_specs):
  548. raise CompileError(pos, "Invalid axes specification.")
  549. if packing == 'strided':
  550. has_strided = True
  551. elif packing == 'contig':
  552. if has_contig:
  553. raise CompileError(pos, "Only one direct contiguous "
  554. "axis may be specified.")
  555. valid_contig_dims = last_indirect_dimension + 1, len(specs) - 1
  556. if idx not in valid_contig_dims and access != 'ptr':
  557. if last_indirect_dimension + 1 != len(specs) - 1:
  558. dims = "dimensions %d and %d" % valid_contig_dims
  559. else:
  560. dims = "dimension %d" % valid_contig_dims[0]
  561. raise CompileError(pos, "Only %s may be contiguous and direct" % dims)
  562. has_contig = access != 'ptr'
  563. elif packing == 'follow':
  564. if has_strided:
  565. raise CompileError(pos, "A memoryview cannot have both follow and strided axis specifiers.")
  566. if not (is_c_contig or is_f_contig):
  567. raise CompileError(pos, "Invalid use of the follow specifier.")
  568. if access in ('ptr', 'full'):
  569. has_strided = False
  570. def _get_resolved_spec(env, spec):
  571. # spec must be a NameNode or an AttributeNode
  572. if isinstance(spec, NameNode):
  573. return _resolve_NameNode(env, spec)
  574. elif isinstance(spec, AttributeNode):
  575. return _resolve_AttributeNode(env, spec)
  576. else:
  577. raise CompileError(spec.pos, INVALID_ERR)
  578. def _resolve_NameNode(env, node):
  579. try:
  580. resolved_name = env.lookup(node.name).name
  581. except AttributeError:
  582. raise CompileError(node.pos, INVALID_ERR)
  583. viewscope = env.global_scope().context.cython_scope.viewscope
  584. entry = viewscope.lookup(resolved_name)
  585. if entry is None:
  586. raise CompileError(node.pos, NOT_CIMPORTED_ERR)
  587. return entry
  588. def _resolve_AttributeNode(env, node):
  589. path = []
  590. while isinstance(node, AttributeNode):
  591. path.insert(0, node.attribute)
  592. node = node.obj
  593. if isinstance(node, NameNode):
  594. path.insert(0, node.name)
  595. else:
  596. raise CompileError(node.pos, EXPR_ERR)
  597. modnames = path[:-1]
  598. # must be at least 1 module name, o/w not an AttributeNode.
  599. assert modnames
  600. scope = env
  601. for modname in modnames:
  602. mod = scope.lookup(modname)
  603. if not mod or not mod.as_module:
  604. raise CompileError(
  605. node.pos, "undeclared name not builtin: %s" % modname)
  606. scope = mod.as_module
  607. entry = scope.lookup(path[-1])
  608. if not entry:
  609. raise CompileError(node.pos, "No such attribute '%s'" % path[-1])
  610. return entry
  611. #
  612. ### Utility loading
  613. #
  614. def load_memview_cy_utility(util_code_name, context=None, **kwargs):
  615. return CythonUtilityCode.load(util_code_name, "MemoryView.pyx",
  616. context=context, **kwargs)
  617. def load_memview_c_utility(util_code_name, context=None, **kwargs):
  618. if context is None:
  619. return UtilityCode.load(util_code_name, "MemoryView_C.c", **kwargs)
  620. else:
  621. return TempitaUtilityCode.load(util_code_name, "MemoryView_C.c",
  622. context=context, **kwargs)
  623. def use_cython_array_utility_code(env):
  624. cython_scope = env.global_scope().context.cython_scope
  625. cython_scope.load_cythonscope()
  626. cython_scope.viewscope.lookup('array_cwrapper').used = True
  627. context = {
  628. 'memview_struct_name': memview_objstruct_cname,
  629. 'max_dims': Options.buffer_max_dims,
  630. 'memviewslice_name': memviewslice_cname,
  631. 'memslice_init': memslice_entry_init,
  632. }
  633. memviewslice_declare_code = load_memview_c_utility(
  634. "MemviewSliceStruct",
  635. context=context,
  636. requires=[])
  637. atomic_utility = load_memview_c_utility("Atomics", context)
  638. memviewslice_init_code = load_memview_c_utility(
  639. "MemviewSliceInit",
  640. context=dict(context, BUF_MAX_NDIMS=Options.buffer_max_dims),
  641. requires=[memviewslice_declare_code,
  642. atomic_utility],
  643. )
  644. memviewslice_index_helpers = load_memview_c_utility("MemviewSliceIndex")
  645. typeinfo_to_format_code = load_memview_cy_utility(
  646. "BufferFormatFromTypeInfo", requires=[Buffer._typeinfo_to_format_code])
  647. is_contig_utility = load_memview_c_utility("MemviewSliceIsContig", context)
  648. overlapping_utility = load_memview_c_utility("OverlappingSlices", context)
  649. copy_contents_new_utility = load_memview_c_utility(
  650. "MemviewSliceCopyTemplate",
  651. context,
  652. requires=[], # require cython_array_utility_code
  653. )
  654. view_utility_code = load_memview_cy_utility(
  655. "View.MemoryView",
  656. context=context,
  657. requires=[Buffer.GetAndReleaseBufferUtilityCode(),
  658. Buffer.buffer_struct_declare_code,
  659. Buffer.buffer_formats_declare_code,
  660. memviewslice_init_code,
  661. is_contig_utility,
  662. overlapping_utility,
  663. copy_contents_new_utility,
  664. ModuleNode.capsule_utility_code],
  665. )
  666. view_utility_whitelist = ('array', 'memoryview', 'array_cwrapper',
  667. 'generic', 'strided', 'indirect', 'contiguous',
  668. 'indirect_contiguous')
  669. memviewslice_declare_code.requires.append(view_utility_code)
  670. copy_contents_new_utility.requires.append(view_utility_code)