From e4d678a2f5859d29a853d617e9e5bbd4b6241898 Mon Sep 17 00:00:00 2001 From: Jeff VanOss Date: Sun, 25 Mar 2018 01:29:20 -0400 Subject: BUG: Provide correct format in Py_buffer for scalars (#10564) Unifies scalar and ndarray pep3118 format string generation --- numpy/core/src/multiarray/buffer.c | 143 ++++++++++++++++++++++------ numpy/core/src/multiarray/buffer.h | 3 + numpy/core/src/multiarray/scalartypes.c.src | 17 +--- numpy/core/tests/test_scalarbuffer.py | 84 ++++++++++++++++ 4 files changed, 201 insertions(+), 46 deletions(-) create mode 100644 numpy/core/tests/test_scalarbuffer.py diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 4aa25a196..21dbdefd6 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -15,6 +15,7 @@ #include "common.h" #include "numpyos.h" #include "arrayobject.h" +#include "scalartypes.h" /************************************************************************* **************** Implement Buffer Protocol **************************** @@ -176,7 +177,7 @@ _is_natively_aligned_at(PyArray_Descr *descr, static int _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, - PyArrayObject* arr, Py_ssize_t *offset, + PyObject* obj, Py_ssize_t *offset, char *active_byteorder) { int k; @@ -223,7 +224,7 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, Py_DECREF(subarray_tuple); old_offset = *offset; - ret = _buffer_format_string(descr->subarray->base, str, arr, offset, + ret = _buffer_format_string(descr->subarray->base, str, obj, offset, active_byteorder); *offset = old_offset + (*offset - old_offset) * total_count; return ret; @@ -265,7 +266,7 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, } /* Insert child item */ - _buffer_format_string(child, str, arr, offset, + _buffer_format_string(child, str, obj, offset, active_byteorder); /* Insert field name */ @@ -302,6 +303,7 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, } else { int is_standard_size = 1; + int is_natively_aligned; int is_native_only_type = (descr->type_num == NPY_LONGDOUBLE || descr->type_num == NPY_CLONGDOUBLE); if (sizeof(npy_longlong) != 8) { @@ -312,8 +314,16 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, *offset += descr->elsize; - if (descr->byteorder == '=' && - _is_natively_aligned_at(descr, arr, *offset)) { + if (PyArray_IsScalar(obj, Generic)) { + /* scalars are always natively aligned */ + is_natively_aligned = 1; + } + else { + is_natively_aligned = _is_natively_aligned_at(descr, + (PyArrayObject*)obj, *offset); + } + + if (descr->byteorder == '=' && is_natively_aligned) { /* Prefer native types, to cater for Cython */ is_standard_size = 0; if (*active_byteorder != '@') { @@ -448,43 +458,61 @@ static PyObject *_buffer_info_cache = NULL; /* Fill in the info structure */ static _buffer_info_t* -_buffer_info_new(PyArrayObject *arr) +_buffer_info_new(PyObject *obj) { _buffer_info_t *info; _tmp_string_t fmt = {NULL, 0, 0}; int k; + PyArray_Descr *descr = NULL; + int err = 0; info = malloc(sizeof(_buffer_info_t)); if (info == NULL) { goto fail; } - /* Fill in format */ - if (_buffer_format_string(PyArray_DESCR(arr), &fmt, arr, NULL, NULL) != 0) { - free(fmt.s); - goto fail; - } - _append_char(&fmt, '\0'); - info->format = fmt.s; - - /* Fill in shape and strides */ - info->ndim = PyArray_NDIM(arr); - - if (info->ndim == 0) { + if (PyArray_IsScalar(obj, Generic)) { + descr = PyArray_DescrFromScalar(obj); + if (descr == NULL) { + goto fail; + } + info->ndim = 0; info->shape = NULL; info->strides = NULL; } else { - info->shape = malloc(sizeof(Py_ssize_t) * PyArray_NDIM(arr) * 2 + 1); - if (info->shape == NULL) { - goto fail; + PyArrayObject * arr = (PyArrayObject *)obj; + descr = PyArray_DESCR(arr); + /* Fill in shape and strides */ + info->ndim = PyArray_NDIM(arr); + + if (info->ndim == 0) { + info->shape = NULL; + info->strides = NULL; } - info->strides = info->shape + PyArray_NDIM(arr); - for (k = 0; k < PyArray_NDIM(arr); ++k) { - info->shape[k] = PyArray_DIMS(arr)[k]; - info->strides[k] = PyArray_STRIDES(arr)[k]; + else { + info->shape = malloc(sizeof(Py_ssize_t) * PyArray_NDIM(arr) * 2 + 1); + if (info->shape == NULL) { + goto fail; + } + info->strides = info->shape + PyArray_NDIM(arr); + for (k = 0; k < PyArray_NDIM(arr); ++k) { + info->shape[k] = PyArray_DIMS(arr)[k]; + info->strides[k] = PyArray_STRIDES(arr)[k]; + } } + Py_INCREF(descr); + } + + /* Fill in format */ + err = _buffer_format_string(descr, &fmt, obj, NULL, NULL); + Py_DECREF(descr); + if (err != 0) { + free(fmt.s); + goto fail; } + _append_char(&fmt, '\0'); + info->format = fmt.s; return info; @@ -530,7 +558,7 @@ _buffer_info_free(_buffer_info_t *info) /* Get buffer info from the global dictionary */ static _buffer_info_t* -_buffer_get_info(PyObject *arr) +_buffer_get_info(PyObject *obj) { PyObject *key = NULL, *item_list = NULL, *item = NULL; _buffer_info_t *info = NULL, *old_info = NULL; @@ -543,13 +571,13 @@ _buffer_get_info(PyObject *arr) } /* Compute information */ - info = _buffer_info_new((PyArrayObject*)arr); + info = _buffer_info_new(obj); if (info == NULL) { return NULL; } /* Check if it is identical with an old one; reuse old one, if yes */ - key = PyLong_FromVoidPtr((void*)arr); + key = PyLong_FromVoidPtr((void*)obj); if (key == NULL) { goto fail; } @@ -627,9 +655,8 @@ _buffer_clear_info(PyObject *arr) } /* - * Retrieving buffers + * Retrieving buffers for ndarray */ - static int array_getbuffer(PyObject *obj, Py_buffer *view, int flags) { @@ -751,6 +778,62 @@ fail: return -1; } +/* + * Retrieving buffers for scalars + */ +int +gentype_getbuffer(PyObject *self, Py_buffer *view, int flags) +{ + _buffer_info_t *info = NULL; + PyArray_Descr *descr = NULL; + int elsize; + + if (flags & PyBUF_WRITABLE) { + PyErr_SetString(PyExc_BufferError, "scalar buffer is readonly"); + goto fail; + } + + /* Fill in information */ + info = _buffer_get_info(self); + if (info == NULL) { + PyErr_SetString(PyExc_BufferError, + "could not get scalar buffer information"); + goto fail; + } + + view->ndim = info->ndim; + view->shape = info->shape; + view->strides = info->strides; + + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { + view->format = info->format; + } else { + view->format = NULL; + } + + descr = PyArray_DescrFromScalar(self); + view->buf = (void *)scalar_value(self, descr); + elsize = descr->elsize; +#ifndef Py_UNICODE_WIDE + if (descr->type_num == NPY_UNICODE) { + elsize >>= 1; + } +#endif + view->len = elsize; + view->itemsize = elsize; + + Py_DECREF(descr); + + view->readonly = 1; + view->suboffsets = NULL; + view->obj = self; + Py_INCREF(self); + return 0; + +fail: + view->obj = NULL; + return -1; +} /* * NOTE: for backward compatibility (esp. with PyArg_ParseTuple("s#", ...)) diff --git a/numpy/core/src/multiarray/buffer.h b/numpy/core/src/multiarray/buffer.h index d2ea01b34..d5da8f440 100644 --- a/numpy/core/src/multiarray/buffer.h +++ b/numpy/core/src/multiarray/buffer.h @@ -9,4 +9,7 @@ _array_dealloc_buffer_info(PyArrayObject *self); NPY_NO_EXPORT PyArray_Descr* _descriptor_from_pep3118_format(char *s); +NPY_NO_EXPORT int +gentype_getbuffer(PyObject *obj, Py_buffer *view, int flags); + #endif diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 9ea26e5e9..e101c3779 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -28,6 +28,7 @@ #include "npy_import.h" #include "dragon4.h" #include "npy_longdouble.h" +#include "buffer.h" #include @@ -2526,22 +2527,6 @@ gentype_getcharbuf(PyObject *self, Py_ssize_t segment, constchar **ptrptr) } #endif /* !defined(NPY_PY3K) */ - -static int -gentype_getbuffer(PyObject *self, Py_buffer *view, int flags) -{ - Py_ssize_t len; - void *buf; - - /* FIXME: XXX: the format is not implemented! -- this needs more work */ - - len = gentype_getreadbuf(self, 0, &buf); - return PyBuffer_FillInfo(view, self, buf, len, 1, flags); -} - -/* releasebuffer is not needed */ - - static PyBufferProcs gentype_as_buffer = { #if !defined(NPY_PY3K) gentype_getreadbuf, /* bf_getreadbuffer*/ diff --git a/numpy/core/tests/test_scalarbuffer.py b/numpy/core/tests/test_scalarbuffer.py new file mode 100644 index 000000000..cd887f2fb --- /dev/null +++ b/numpy/core/tests/test_scalarbuffer.py @@ -0,0 +1,84 @@ +""" +Test scalar buffer interface adheres to PEP 3118 +""" +import sys +import numpy as np +from numpy.testing import run_module_suite, assert_, assert_equal, dec + +# PEP3118 format strings for native (standard alignment and byteorder) types +scalars_and_codes = [ + (np.bool_, '?'), + (np.byte, 'b'), + (np.short, 'h'), + (np.intc, 'i'), + (np.int_, 'l'), + (np.longlong, 'q'), + (np.ubyte, 'B'), + (np.ushort, 'H'), + (np.uintc, 'I'), + (np.uint, 'L'), + (np.ulonglong, 'Q'), + (np.half, 'e'), + (np.single, 'f'), + (np.double, 'd'), + (np.longdouble, 'g'), + (np.csingle, 'Zf'), + (np.cdouble, 'Zd'), + (np.clongdouble, 'Zg'), +] + + +class TestScalarPEP3118(object): + skip_if_no_buffer_interface = dec.skipif(sys.version_info.major < 3, + "scalars do not implement buffer interface in Python 2") + + @skip_if_no_buffer_interface + def test_scalar_match_array(self): + for scalar, _ in scalars_and_codes: + x = scalar() + a = np.array([], dtype=np.dtype(scalar)) + mv_x = memoryview(x) + mv_a = memoryview(a) + assert_equal(mv_x.format, mv_a.format) + + @skip_if_no_buffer_interface + def test_scalar_dim(self): + for scalar, _ in scalars_and_codes: + x = scalar() + mv_x = memoryview(x) + assert_equal(mv_x.itemsize, np.dtype(scalar).itemsize) + assert_equal(mv_x.ndim, 0) + assert_equal(mv_x.shape, ()) + assert_equal(mv_x.strides, ()) + assert_equal(mv_x.suboffsets, ()) + + @skip_if_no_buffer_interface + def test_scalar_known_code(self): + for scalar, code in scalars_and_codes: + x = scalar() + mv_x = memoryview(x) + assert_equal(mv_x.format, code) + + @skip_if_no_buffer_interface + def test_void_scalar_structured_data(self): + dt = np.dtype([('name', np.unicode_, 16), ('grades', np.float64, (2,))]) + x = np.array(('ndarray_scalar', (1.2, 3.0)), dtype=dt)[()] + assert_(isinstance(x, np.void)) + mv_x = memoryview(x) + expected_size = 16 * np.dtype((np.unicode_, 1)).itemsize + expected_size += 2 * np.dtype((np.float64, 1)).itemsize + assert_equal(mv_x.itemsize, expected_size) + assert_equal(mv_x.ndim, 0) + assert_equal(mv_x.shape, ()) + assert_equal(mv_x.strides, ()) + assert_equal(mv_x.suboffsets, ()) + + # check scalar format string against ndarray format string + a = np.array([('Sarah', (8.0, 7.0)), ('John', (6.0, 7.0))], dtype=dt) + assert_(isinstance(a, np.ndarray)) + mv_a = memoryview(a) + assert_equal(mv_x.itemsize, mv_a.itemsize) + assert_equal(mv_x.format, mv_a.format) + +if __name__ == "__main__": + run_module_suite() -- cgit v1.2.3