diff options
Diffstat (limited to 'numpy/core/src/multiarray')
21 files changed, 937 insertions, 175 deletions
diff --git a/numpy/core/src/multiarray/_multiarray_module_test.c b/numpy/core/src/multiarray/_multiarray_module_test.c new file mode 100644 index 000000000..8dc017279 --- /dev/null +++ b/numpy/core/src/multiarray/_multiarray_module_test.c @@ -0,0 +1,129 @@ +#include "Python.h" + +/* + * This is a dummy module. It will be used to ruin the import of multiarray + * during testing. It exports two entry points, one to make the build happy, + * and a multiarray one for the actual test. The content of the module is + * irrelevant to the test. + * + * The code is from + * https://docs.python.org/3/howto/cporting.html + * or + * https://github.com/python/cpython/blob/v3.7.0/Doc/howto/cporting.rst + */ + +#if defined _WIN32 || defined __CYGWIN__ || defined __MINGW32__ + #if defined __GNUC__ || defined __clang__ + #define DLL_PUBLIC __attribute__ ((dllexport)) + #else + #define DLL_PUBLIC __declspec(dllexport) + #endif +#elif defined __GNUC__ || defined __clang__ + #define DLL_PUBLIC __attribute__ ((visibility ("default"))) +#else + /* Enhancement: error now instead ? */ + #define DLL_PUBLIC +#endif + +struct module_state { + PyObject *error; +}; + +#if PY_MAJOR_VERSION >= 3 +#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) +#else +#define GETSTATE(m) (&_state) +static struct module_state _state; +#endif + +static PyObject * +error_out(PyObject *m) { + struct module_state *st = GETSTATE(m); + PyErr_SetString(st->error, "something bad happened"); + return NULL; +} + +static PyMethodDef multiarray_methods[] = { + {"error_out", (PyCFunction)error_out, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +#if PY_MAJOR_VERSION >= 3 + +static int multiarray_traverse(PyObject *m, visitproc visit, void *arg) { + Py_VISIT(GETSTATE(m)->error); + return 0; +} + +static int multiarray_clear(PyObject *m) { + Py_CLEAR(GETSTATE(m)->error); + return 0; +} + + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "multiarray", + NULL, + sizeof(struct module_state), + multiarray_methods, + NULL, + multiarray_traverse, + multiarray_clear, + NULL +}; + +#define INITERROR return NULL + +DLL_PUBLIC PyObject * +PyInit_multiarray(void) + +#else +#define INITERROR return + +void +DLL_PUBLIC initmultiarray(void) +#endif +{ +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); +#else + PyObject *module = Py_InitModule("multiarray", multiarray_methods); +#endif + struct module_state *st; + if (module == NULL) + INITERROR; + st = GETSTATE(module); + + st->error = PyErr_NewException("multiarray.Error", NULL, NULL); + if (st->error == NULL) { + Py_DECREF(module); + INITERROR; + } + +#if PY_MAJOR_VERSION >= 3 + return module; +#endif +} + +/* + * Define a dummy entry point to make MSVC happy + * Python's build system will export this function automatically + */ +#if PY_MAJOR_VERSION >= 3 + +PyObject * +PyInit__multiarray_module_test(void) +{ + return PyInit_multiarray(); +} + +#else + +void +init_multiarray_module_test(void) +{ + initmultiarray(); +} + +#endif diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index 2a8275572..c26bd16ac 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -1871,11 +1871,14 @@ printf_float_g(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) static PyObject * getset_numericops(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) { - PyObject * ops = PyArray_GetNumericOps(); + PyObject *ret; + PyObject *ops = PyArray_GetNumericOps(); if (ops == NULL) { return NULL; } - return PyLong_FromLong(PyArray_SetNumericOps(ops)); + ret = PyLong_FromLong(PyArray_SetNumericOps(ops)); + Py_DECREF(ops); + return ret; } static PyMethodDef Multiarray_TestsMethods[] = { diff --git a/numpy/core/src/multiarray/arrayfunction_override.c b/numpy/core/src/multiarray/arrayfunction_override.c new file mode 100644 index 000000000..e62b32ab2 --- /dev/null +++ b/numpy/core/src/multiarray/arrayfunction_override.c @@ -0,0 +1,376 @@ +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define _MULTIARRAYMODULE + +#include "npy_pycompat.h" +#include "get_attr_string.h" +#include "npy_import.h" +#include "multiarraymodule.h" + + +/* Return the ndarray.__array_function__ method. */ +static PyObject * +get_ndarray_array_function(void) +{ + PyObject* method = PyObject_GetAttrString((PyObject *)&PyArray_Type, + "__array_function__"); + assert(method != NULL); + return method; +} + + +/* + * Get an object's __array_function__ method in the fastest way possible. + * Never raises an exception. Returns NULL if the method doesn't exist. + */ +static PyObject * +get_array_function(PyObject *obj) +{ + static PyObject *ndarray_array_function = NULL; + + if (ndarray_array_function == NULL) { + ndarray_array_function = get_ndarray_array_function(); + } + + /* Fast return for ndarray */ + if (PyArray_CheckExact(obj)) { + Py_INCREF(ndarray_array_function); + return ndarray_array_function; + } + + return PyArray_LookupSpecial(obj, "__array_function__"); +} + + +/* + * Like list.insert(), but for C arrays of PyObject*. Skips error checking. + */ +static void +pyobject_array_insert(PyObject **array, int length, int index, PyObject *item) +{ + int j; + + for (j = length; j > index; j--) { + array[j] = array[j - 1]; + } + array[index] = item; +} + + +/* + * Collects arguments with __array_function__ and their corresponding methods + * in the order in which they should be tried (i.e., skipping redundant types). + * `relevant_args` is expected to have been produced by PySequence_Fast. + * Returns the number of arguments, or -1 on failure. + */ +static int +get_implementing_args_and_methods(PyObject *relevant_args, + PyObject **implementing_args, + PyObject **methods) +{ + int num_implementing_args = 0; + Py_ssize_t i; + int j; + + PyObject **items = PySequence_Fast_ITEMS(relevant_args); + Py_ssize_t length = PySequence_Fast_GET_SIZE(relevant_args); + + for (i = 0; i < length; i++) { + int new_class = 1; + PyObject *argument = items[i]; + + /* Have we seen this type before? */ + for (j = 0; j < num_implementing_args; j++) { + if (Py_TYPE(argument) == Py_TYPE(implementing_args[j])) { + new_class = 0; + break; + } + } + if (new_class) { + PyObject *method = get_array_function(argument); + + if (method != NULL) { + int arg_index; + + if (num_implementing_args >= NPY_MAXARGS) { + PyErr_Format( + PyExc_TypeError, + "maximum number (%d) of distinct argument types " \ + "implementing __array_function__ exceeded", + NPY_MAXARGS); + Py_DECREF(method); + goto fail; + } + + /* "subclasses before superclasses, otherwise left to right" */ + arg_index = num_implementing_args; + for (j = 0; j < num_implementing_args; j++) { + PyObject *other_type; + other_type = (PyObject *)Py_TYPE(implementing_args[j]); + if (PyObject_IsInstance(argument, other_type)) { + arg_index = j; + break; + } + } + Py_INCREF(argument); + pyobject_array_insert(implementing_args, num_implementing_args, + arg_index, argument); + pyobject_array_insert(methods, num_implementing_args, + arg_index, method); + ++num_implementing_args; + } + } + } + return num_implementing_args; + +fail: + for (j = 0; j < num_implementing_args; j++) { + Py_DECREF(implementing_args[j]); + Py_DECREF(methods[j]); + } + return -1; +} + + +/* + * Is this object ndarray.__array_function__? + */ +static int +is_default_array_function(PyObject *obj) +{ + static PyObject *ndarray_array_function = NULL; + + if (ndarray_array_function == NULL) { + ndarray_array_function = get_ndarray_array_function(); + } + return obj == ndarray_array_function; +} + + +/* + * Core implementation of ndarray.__array_function__. This is exposed + * separately so we can avoid the overhead of a Python method call from + * within `implement_array_function`. + */ +NPY_NO_EXPORT PyObject * +array_function_method_impl(PyObject *func, PyObject *types, PyObject *args, + PyObject *kwargs) +{ + Py_ssize_t j; + PyObject *implementation, *result; + + PyObject **items = PySequence_Fast_ITEMS(types); + Py_ssize_t length = PySequence_Fast_GET_SIZE(types); + + for (j = 0; j < length; j++) { + int is_subclass = PyObject_IsSubclass( + items[j], (PyObject *)&PyArray_Type); + if (is_subclass == -1) { + return NULL; + } + if (!is_subclass) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + } + + implementation = PyObject_GetAttr(func, npy_ma_str_wrapped); + if (implementation == NULL) { + return NULL; + } + result = PyObject_Call(implementation, args, kwargs); + Py_DECREF(implementation); + return result; +} + + +/* + * Calls __array_function__ on the provided argument, with a fast-path for + * ndarray. + */ +static PyObject * +call_array_function(PyObject* argument, PyObject* method, + PyObject* public_api, PyObject* types, + PyObject* args, PyObject* kwargs) +{ + if (is_default_array_function(method)) { + return array_function_method_impl(public_api, types, args, kwargs); + } + else { + return PyObject_CallFunctionObjArgs( + method, argument, public_api, types, args, kwargs, NULL); + } +} + + +/* + * Implements the __array_function__ protocol for a function, as described in + * in NEP-18. See numpy.core.overrides for a full docstring. + */ +NPY_NO_EXPORT PyObject * +array_implement_array_function( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args) +{ + PyObject *implementation, *public_api, *relevant_args, *args, *kwargs; + + PyObject *types = NULL; + PyObject *implementing_args[NPY_MAXARGS]; + PyObject *array_function_methods[NPY_MAXARGS]; + + int j, any_overrides; + int num_implementing_args = 0; + PyObject *result = NULL; + + static PyObject *errmsg_formatter = NULL; + + if (!PyArg_UnpackTuple( + positional_args, "implement_array_function", 5, 5, + &implementation, &public_api, &relevant_args, &args, &kwargs)) { + return NULL; + } + + relevant_args = PySequence_Fast( + relevant_args, + "dispatcher for __array_function__ did not return an iterable"); + if (relevant_args == NULL) { + return NULL; + } + + /* Collect __array_function__ implementations */ + num_implementing_args = get_implementing_args_and_methods( + relevant_args, implementing_args, array_function_methods); + if (num_implementing_args == -1) { + goto cleanup; + } + + /* + * Handle the typical case of no overrides. This is merely an optimization + * if some arguments are ndarray objects, but is also necessary if no + * arguments implement __array_function__ at all (e.g., if they are all + * built-in types). + */ + any_overrides = 0; + for (j = 0; j < num_implementing_args; j++) { + if (!is_default_array_function(array_function_methods[j])) { + any_overrides = 1; + break; + } + } + if (!any_overrides) { + result = PyObject_Call(implementation, args, kwargs); + goto cleanup; + } + + /* + * Create a Python object for types. + * We use a tuple, because it's the fastest Python collection to create + * and has the bonus of being immutable. + */ + types = PyTuple_New(num_implementing_args); + if (types == NULL) { + goto cleanup; + } + for (j = 0; j < num_implementing_args; j++) { + PyObject *arg_type = (PyObject *)Py_TYPE(implementing_args[j]); + Py_INCREF(arg_type); + PyTuple_SET_ITEM(types, j, arg_type); + } + + /* Call __array_function__ methods */ + for (j = 0; j < num_implementing_args; j++) { + PyObject *argument = implementing_args[j]; + PyObject *method = array_function_methods[j]; + + /* + * We use `public_api` instead of `implementation` here so + * __array_function__ implementations can do equality/identity + * comparisons. + */ + result = call_array_function( + argument, method, public_api, types, args, kwargs); + + if (result == Py_NotImplemented) { + /* Try the next one */ + Py_DECREF(result); + result = NULL; + } + else { + /* Either a good result, or an exception was raised. */ + goto cleanup; + } + } + + /* No acceptable override found, raise TypeError. */ + npy_cache_import("numpy.core._internal", + "array_function_errmsg_formatter", + &errmsg_formatter); + if (errmsg_formatter != NULL) { + PyObject *errmsg = PyObject_CallFunctionObjArgs( + errmsg_formatter, public_api, types, NULL); + if (errmsg != NULL) { + PyErr_SetObject(PyExc_TypeError, errmsg); + Py_DECREF(errmsg); + } + } + +cleanup: + for (j = 0; j < num_implementing_args; j++) { + Py_DECREF(implementing_args[j]); + Py_DECREF(array_function_methods[j]); + } + Py_XDECREF(types); + Py_DECREF(relevant_args); + return result; +} + + +/* + * Python wrapper for get_implementing_args_and_methods, for testing purposes. + */ +NPY_NO_EXPORT PyObject * +array__get_implementing_args( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args) +{ + PyObject *relevant_args; + int j; + int num_implementing_args = 0; + PyObject *implementing_args[NPY_MAXARGS]; + PyObject *array_function_methods[NPY_MAXARGS]; + PyObject *result = NULL; + + if (!PyArg_ParseTuple(positional_args, "O:array__get_implementing_args", + &relevant_args)) { + return NULL; + } + + relevant_args = PySequence_Fast( + relevant_args, + "dispatcher for __array_function__ did not return an iterable"); + if (relevant_args == NULL) { + return NULL; + } + + num_implementing_args = get_implementing_args_and_methods( + relevant_args, implementing_args, array_function_methods); + if (num_implementing_args == -1) { + goto cleanup; + } + + /* create a Python object for implementing_args */ + result = PyList_New(num_implementing_args); + if (result == NULL) { + goto cleanup; + } + for (j = 0; j < num_implementing_args; j++) { + PyObject *argument = implementing_args[j]; + Py_INCREF(argument); + PyList_SET_ITEM(result, j, argument); + } + +cleanup: + for (j = 0; j < num_implementing_args; j++) { + Py_DECREF(implementing_args[j]); + Py_DECREF(array_function_methods[j]); + } + Py_DECREF(relevant_args); + return result; +} diff --git a/numpy/core/src/multiarray/arrayfunction_override.h b/numpy/core/src/multiarray/arrayfunction_override.h new file mode 100644 index 000000000..0d224e2b6 --- /dev/null +++ b/numpy/core/src/multiarray/arrayfunction_override.h @@ -0,0 +1,16 @@ +#ifndef _NPY_PRIVATE__ARRAYFUNCTION_OVERRIDE_H +#define _NPY_PRIVATE__ARRAYFUNCTION_OVERRIDE_H + +NPY_NO_EXPORT PyObject * +array_implement_array_function( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args); + +NPY_NO_EXPORT PyObject * +array__get_implementing_args( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args); + +NPY_NO_EXPORT PyObject * +array_function_method_impl(PyObject *func, PyObject *types, PyObject *args, + PyObject *kwargs); + +#endif diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index 823ee7115..ca5f5a47b 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -2205,15 +2205,19 @@ static void VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, npy_intp n, int swap, PyArrayObject *arr) { + PyArray_Descr *descr; + if (arr == NULL) { return; } + + descr = PyArray_DESCR(arr); + if (PyArray_HASFIELDS(arr)) { PyObject *key, *value; - PyArray_Descr *descr; + Py_ssize_t pos = 0; - descr = PyArray_DESCR(arr); while (PyDict_Next(descr->fields, &pos, &key, &value)) { npy_intp offset; PyArray_Descr * new; @@ -2236,14 +2240,28 @@ VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, ((PyArrayObject_fields *)arr)->descr = descr; return; } - if (swap && PyArray_DESCR(arr)->subarray != NULL) { - PyArray_Descr *descr, *new; + if (PyDataType_HASSUBARRAY(descr)) { + PyArray_Descr *new; npy_intp num; npy_intp i; int subitemsize; char *dstptr, *srcptr; + /* + * In certain cases subarray copy can be optimized. This is when + * swapping is unecessary and the subarrays data type can certainly + * be simply copied (no object, fields, subarray, and not a user dtype). + */ + npy_bool can_optimize_subarray = (!swap && + !PyDataType_HASFIELDS(descr->subarray->base) && + !PyDataType_HASSUBARRAY(descr->subarray->base) && + !PyDataType_REFCHK(descr->subarray->base) && + (descr->subarray->base->type_num < NPY_NTYPES)); + + if (can_optimize_subarray) { + _basic_copyn(dst, dstride, src, sstride, n, descr->elsize); + return; + } - descr = PyArray_DESCR(arr); new = descr->subarray->base; /* * TODO: temporarily modifying the array like this @@ -2253,6 +2271,10 @@ VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, dstptr = dst; srcptr = src; subitemsize = new->elsize; + if (subitemsize == 0) { + /* There cannot be any elements, so return */ + return; + } num = descr->elsize / subitemsize; for (i = 0; i < n; i++) { new->f->copyswapn(dstptr, subitemsize, srcptr, @@ -2265,22 +2287,26 @@ VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, ((PyArrayObject_fields *)arr)->descr = descr; return; } - _basic_copyn(dst, dstride, src, sstride, n, PyArray_DESCR(arr)->elsize); + /* Must be a naive Void type (e.g. a "V8") so simple copy is sufficient. */ + _basic_copyn(dst, dstride, src, sstride, n, descr->elsize); return; } static void VOID_copyswap (char *dst, char *src, int swap, PyArrayObject *arr) { + PyArray_Descr *descr; + if (arr == NULL) { return; } + + descr = PyArray_DESCR(arr); + if (PyArray_HASFIELDS(arr)) { PyObject *key, *value; - PyArray_Descr *descr; Py_ssize_t pos = 0; - descr = PyArray_DESCR(arr); while (PyDict_Next(descr->fields, &pos, &key, &value)) { npy_intp offset; PyArray_Descr * new; @@ -2303,28 +2329,45 @@ VOID_copyswap (char *dst, char *src, int swap, PyArrayObject *arr) ((PyArrayObject_fields *)arr)->descr = descr; return; } - if (swap && PyArray_DESCR(arr)->subarray != NULL) { - PyArray_Descr *descr, *new; + if (PyDataType_HASSUBARRAY(descr)) { + PyArray_Descr *new; npy_intp num; - int itemsize; + int subitemsize; + /* + * In certain cases subarray copy can be optimized. This is when + * swapping is unecessary and the subarrays data type can certainly + * be simply copied (no object, fields, subarray, and not a user dtype). + */ + npy_bool can_optimize_subarray = (!swap && + !PyDataType_HASFIELDS(descr->subarray->base) && + !PyDataType_HASSUBARRAY(descr->subarray->base) && + !PyDataType_REFCHK(descr->subarray->base) && + (descr->subarray->base->type_num < NPY_NTYPES)); + + if (can_optimize_subarray) { + _basic_copy(dst, src, descr->elsize); + return; + } - descr = PyArray_DESCR(arr); new = descr->subarray->base; /* * TODO: temporarily modifying the array like this * is bad coding style, should be changed. */ ((PyArrayObject_fields *)arr)->descr = new; - itemsize = new->elsize; - num = descr->elsize / itemsize; - new->f->copyswapn(dst, itemsize, src, - itemsize, num, swap, arr); + subitemsize = new->elsize; + if (subitemsize == 0) { + /* There cannot be any elements, so return */ + return; + } + num = descr->elsize / subitemsize; + new->f->copyswapn(dst, subitemsize, src, + subitemsize, num, swap, arr); ((PyArrayObject_fields *)arr)->descr = descr; return; } - - /* copy first if needed */ - _basic_copy(dst, src, PyArray_DESCR(arr)->elsize); + /* Must be a naive Void type (e.g. a "V8") so simple copy is sufficient. */ + _basic_copy(dst, src, descr->elsize); return; } diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 2f66d7f2f..d8ad80266 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -509,6 +509,10 @@ _buffer_info_new(PyObject *obj) PyArray_Descr *descr = NULL; int err = 0; + /* + * Note that the buffer info is cached as pyints making them appear like + * unreachable lost memory to valgrind. + */ info = malloc(sizeof(_buffer_info_t)); if (info == NULL) { PyErr_NoMemory(); @@ -579,9 +583,11 @@ _buffer_info_new(PyObject *obj) err = _buffer_format_string(descr, &fmt, obj, NULL, NULL); Py_DECREF(descr); if (err != 0) { + free(info->shape); goto fail; } if (_append_char(&fmt, '\0') < 0) { + free(info->shape); goto fail; } info->format = fmt.s; diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 2e51cee7e..addb67732 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -164,7 +164,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, if (string_type == NPY_STRING) { if ((temp = PyObject_Str(obj)) == NULL) { - return -1; + goto fail; } #if defined(NPY_PY3K) #if PY_VERSION_HEX >= 0x03030000 @@ -182,7 +182,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, #else if ((temp = PyObject_Unicode(obj)) == NULL) { #endif - return -1; + goto fail; } itemsize = PyUnicode_GET_DATA_SIZE(temp); #ifndef Py_UNICODE_WIDE @@ -216,7 +216,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, if (string_type == NPY_STRING) { if ((temp = PyObject_Str(obj)) == NULL) { - return -1; + goto fail; } #if defined(NPY_PY3K) #if PY_VERSION_HEX >= 0x03030000 @@ -234,7 +234,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, #else if ((temp = PyObject_Unicode(obj)) == NULL) { #endif - return -1; + goto fail; } itemsize = PyUnicode_GET_DATA_SIZE(temp); #ifndef Py_UNICODE_WIDE @@ -511,7 +511,7 @@ promote_types: PyArray_Descr *res_dtype = PyArray_PromoteTypes(dtype, *out_dtype); Py_DECREF(dtype); if (res_dtype == NULL) { - return -1; + goto fail; } if (!string_type && res_dtype->type_num == NPY_UNICODE && diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index 10e3478e2..625028bfb 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -328,6 +328,7 @@ arr_insert(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) } else { Py_XDECREF(values); Py_XDECREF(mask); + PyArray_ResolveWritebackIfCopy(array); Py_XDECREF(array); Py_RETURN_NONE; } @@ -358,6 +359,7 @@ arr_insert(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) fail: Py_XDECREF(mask); + PyArray_ResolveWritebackIfCopy(array); Py_XDECREF(array); Py_XDECREF(values); return NULL; @@ -1575,6 +1577,7 @@ pack_bits(PyObject *input, int axis) if (!PyArray_ISBOOL(inp) && !PyArray_ISINTEGER(inp)) { PyErr_SetString(PyExc_TypeError, "Expected an input array of integer or boolean data type"); + Py_DECREF(inp); goto fail; } @@ -1682,6 +1685,7 @@ unpack_bits(PyObject *input, int axis) if (PyArray_TYPE(inp) != NPY_UBYTE) { PyErr_SetString(PyExc_TypeError, "Expected an input array of unsigned byte data type"); + Py_DECREF(inp); goto fail; } diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index a17621946..b9059ba4d 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1410,6 +1410,7 @@ _array_from_buffer_3118(PyObject *memoryview) * dimensions, so the array is now 0d. */ nd = 0; + Py_DECREF(descr); descr = (PyArray_Descr *)PyObject_CallFunctionObjArgs( (PyObject *)&PyArrayDescr_Type, Py_TYPE(view->obj), NULL); if (descr == NULL) { @@ -2128,12 +2129,15 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) */ /* 2017-Nov-10 1.14 */ - if (DEPRECATE("NPY_ARRAY_UPDATEIFCOPY, NPY_ARRAY_INOUT_ARRAY, and " - "NPY_ARRAY_INOUT_FARRAY are deprecated, use NPY_WRITEBACKIFCOPY, " - "NPY_ARRAY_INOUT_ARRAY2, or NPY_ARRAY_INOUT_FARRAY2 respectively " - "instead, and call PyArray_ResolveWritebackIfCopy before the " - "array is deallocated, i.e. before the last call to Py_DECREF.") < 0) + if (DEPRECATE( + "NPY_ARRAY_UPDATEIFCOPY, NPY_ARRAY_INOUT_ARRAY, and " + "NPY_ARRAY_INOUT_FARRAY are deprecated, use NPY_WRITEBACKIFCOPY, " + "NPY_ARRAY_INOUT_ARRAY2, or NPY_ARRAY_INOUT_FARRAY2 respectively " + "instead, and call PyArray_ResolveWritebackIfCopy before the " + "array is deallocated, i.e. before the last call to Py_DECREF.") < 0) { + Py_DECREF(ret); return NULL; + } Py_INCREF(arr); if (PyArray_SetWritebackIfCopyBase(ret, arr) < 0) { Py_DECREF(ret); @@ -2160,14 +2164,12 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) Py_DECREF(newtype); if (needview) { - PyArray_Descr *dtype = PyArray_DESCR(arr); PyTypeObject *subtype = NULL; if (flags & NPY_ARRAY_ENSUREARRAY) { subtype = &PyArray_Type; } - Py_INCREF(dtype); ret = (PyArrayObject *)PyArray_View(arr, NULL, subtype); if (ret == NULL) { return NULL; @@ -2495,6 +2497,11 @@ PyArray_FromInterface(PyObject *origin) &PyArray_Type, dtype, n, dims, NULL, data, dataflags, NULL, base); + /* + * Ref to dtype was stolen by PyArray_NewFromDescrAndBase + * Prevent DECREFing dtype in fail codepath by setting to NULL + */ + dtype = NULL; if (ret == NULL) { goto fail; } diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index a8550d958..54d19d993 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -3822,18 +3822,26 @@ recursive_find_object_timedelta64_type(PyObject *obj, * single object using [()], but not by using * __getitem__(integer) approaches */ - PyObject *item, *meth, *args; + PyObject *item, *args; - meth = PyObject_GetAttrString(obj, "__getitem__"); - args = Py_BuildValue("(())"); - item = PyObject_CallObject(meth, args); + args = PyTuple_New(0); + if (args == NULL) { + return 0; + } + item = PyObject_GetItem(obj, args); + Py_DECREF(args); + if (item == NULL) { + return 0; + } /* * NOTE: may need other type checks here in the future * for expanded 0 D datetime array conversions? */ if (PyDelta_Check(item)) { + Py_DECREF(item); return delta_checker(meta); } + Py_DECREF(item); } } } diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 3038e4dea..0471a2a3e 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -515,6 +515,7 @@ _convert_from_array_descr(PyObject *obj, int align) #if defined(NPY_PY3K) Py_DECREF(name); #endif + Py_DECREF(conv); goto fail; } dtypeflags |= (conv->flags & NPY_FROM_FIELDS); @@ -837,9 +838,11 @@ _use_inherit(PyArray_Descr *type, PyObject *newobj, int *errflag) else if (new->elsize != conv->elsize) { PyErr_SetString(PyExc_ValueError, "mismatch in size of old and new data-descriptor"); + Py_DECREF(new); goto fail; } else if (invalid_union_object_dtype(new, conv)) { + Py_DECREF(new); goto fail; } @@ -1728,6 +1731,7 @@ PyArray_DescrNew(PyArray_Descr *base) newdescr->c_metadata = NPY_AUXDATA_CLONE(base->c_metadata); if (newdescr->c_metadata == NULL) { PyErr_NoMemory(); + /* TODO: This seems wrong, as the old fields get decref'd? */ Py_DECREF(newdescr); return NULL; } @@ -3336,12 +3340,15 @@ static PyObject * _subscript_by_index(PyArray_Descr *self, Py_ssize_t i) { PyObject *name = PySequence_GetItem(self->names, i); + PyObject *ret; if (name == NULL) { PyErr_Format(PyExc_IndexError, "Field index %zd out of range.", i); return NULL; } - return _subscript_by_name(self, name); + ret = _subscript_by_name(self, name); + Py_DECREF(name); + return ret; } static PyObject * diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 63b1ead25..6347d35eb 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -1572,12 +1572,30 @@ get_cast_transfer_function(int aligned, src_dtype, &tobuffer, &todata); - - /* Get the copy/swap operation to dst */ - PyArray_GetDTypeCopySwapFn(aligned, - dst_itemsize, dst_stride, - dst_dtype, - &frombuffer, &fromdata); + if (!PyDataType_REFCHK(dst_dtype)) { + /* Copying from buffer is a simple copy/swap operation */ + PyArray_GetDTypeCopySwapFn(aligned, + dst_itemsize, dst_stride, + dst_dtype, + &frombuffer, &fromdata); + } + else { + /* + * Since the buffer is initialized to NULL, need to move the + * references in order to DECREF the existing data. + */ + /* Object types cannot be byte swapped */ + assert(PyDataType_ISNOTSWAPPED(dst_dtype)); + /* The loop already needs the python api if this is reached */ + assert(*out_needs_api); + + if (PyArray_GetDTypeTransferFunction( + aligned, dst_itemsize, dst_stride, + dst_dtype, dst_dtype, 1, + &frombuffer, &fromdata, out_needs_api) != NPY_SUCCEED) { + return NPY_FAIL; + } + } if (frombuffer == NULL || tobuffer == NULL) { NPY_AUXDATA_FREE(castdata); @@ -2001,6 +2019,7 @@ typedef struct { _subarray_broadcast_offsetrun offsetruns; } _subarray_broadcast_data; + /* transfer data free function */ static void _subarray_broadcast_data_free(NpyAuxData *data) { diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 7c814e6e6..2fcc2ec22 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -8,6 +8,7 @@ #include "numpy/arrayobject.h" #include "numpy/arrayscalars.h" +#include "arrayfunction_override.h" #include "npy_config.h" #include "npy_pycompat.h" #include "npy_import.h" @@ -1088,13 +1089,29 @@ cleanup: return result; } - static PyObject * -array_function(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_function(PyArrayObject *self, PyObject *c_args, PyObject *c_kwds) { - NPY_FORWARD_NDARRAY_METHOD("_array_function"); -} + PyObject *func, *types, *args, *kwargs, *result; + static char *kwlist[] = {"func", "types", "args", "kwargs", NULL}; + + if (!PyArg_ParseTupleAndKeywords( + c_args, c_kwds, "OOOO:__array_function__", kwlist, + &func, &types, &args, &kwargs)) { + return NULL; + } + types = PySequence_Fast( + types, + "types argument to ndarray.__array_function__ must be iterable"); + if (types == NULL) { + return NULL; + } + + result = array_function_method_impl(func, types, args, kwargs); + Py_DECREF(types); + return result; +} static PyObject * array_copy(PyArrayObject *self, PyObject *args, PyObject *kwds) @@ -1364,6 +1381,7 @@ array_argsort(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } newd = PyArray_DescrNew(saved); + Py_DECREF(newd->names); newd->names = new_name; ((PyArrayObject_fields *)self)->descr = newd; } @@ -1418,6 +1436,7 @@ array_argpartition(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } newd = PyArray_DescrNew(saved); + Py_DECREF(newd->names); newd->names = new_name; ((PyArrayObject_fields *)self)->descr = newd; } @@ -1521,7 +1540,6 @@ array_deepcopy(PyArrayObject *self, PyObject *args) copy = PyImport_ImportModule("copy"); if (copy == NULL) { Py_DECREF(copied_array); - Py_DECREF(copy); return NULL; } deepcopy = PyObject_GetAttrString(copy, "deepcopy"); @@ -1704,129 +1722,150 @@ array_reduce(PyArrayObject *self, PyObject *NPY_UNUSED(args)) } static PyObject * -array_reduce_ex(PyArrayObject *self, PyObject *args) +array_reduce_ex_regular(PyArrayObject *self, int protocol) { - int protocol; - PyObject *ret = NULL, *numeric_mod = NULL, *from_buffer_func = NULL; - PyObject *buffer_tuple = NULL, *pickle_module = NULL, *pickle_class = NULL; - PyObject *class_args = NULL, *class_args_tuple = NULL, *unused = NULL; PyObject *subclass_array_reduce = NULL; + PyObject *ret; + + /* We do not call array_reduce directly but instead lookup and call + * the __reduce__ method to make sure that it's possible to customize + * pickling in sub-classes. */ + subclass_array_reduce = PyObject_GetAttrString((PyObject *)self, + "__reduce__"); + if (subclass_array_reduce == NULL) { + return NULL; + } + ret = PyObject_CallObject(subclass_array_reduce, NULL); + Py_DECREF(subclass_array_reduce); + return ret; +} + +static PyObject * +array_reduce_ex_picklebuffer(PyArrayObject *self, int protocol) +{ + PyObject *numeric_mod = NULL, *from_buffer_func = NULL; + PyObject *pickle_module = NULL, *picklebuf_class = NULL; + PyObject *picklebuf_args = NULL; PyObject *buffer = NULL, *transposed_array = NULL; PyArray_Descr *descr = NULL; char order; - if (PyArg_ParseTuple(args, "i", &protocol)){ - descr = PyArray_DESCR(self); - if ((protocol < 5) || - (!PyArray_IS_C_CONTIGUOUS((PyArrayObject*)self) && - !PyArray_IS_F_CONTIGUOUS((PyArrayObject*)self)) || - PyDataType_FLAGCHK(descr, NPY_ITEM_HASOBJECT) || - (PyType_IsSubtype(((PyObject*)self)->ob_type, &PyArray_Type) && - ((PyObject*)self)->ob_type != &PyArray_Type) || - PyDataType_ISUNSIZED(descr)) { - /* The PickleBuffer class from version 5 of the pickle protocol - * can only be used for arrays backed by a contiguous data buffer. - * For all other cases we fallback to the generic array_reduce - * method that involves using a temporary bytes allocation. However - * we do not call array_reduce directly but instead lookup and call - * the __reduce__ method to make sure that it's possible customize - * pickling in sub-classes. */ - subclass_array_reduce = PyObject_GetAttrString((PyObject *)self, - "__reduce__"); - return PyObject_CallObject(subclass_array_reduce, unused); - } - else if (protocol == 5){ - ret = PyTuple_New(2); - - if (ret == NULL) { - return NULL; - } + descr = PyArray_DESCR(self); - /* if the python version is below 3.8, the pickle module does not provide - * built-in support for protocol 5. We try importing the pickle5 - * backport instead */ + /* if the python version is below 3.8, the pickle module does not provide + * built-in support for protocol 5. We try importing the pickle5 + * backport instead */ #if PY_VERSION_HEX >= 0x03080000 - pickle_module = PyImport_ImportModule("pickle"); -#elif PY_VERSION_HEX < 0x03080000 && PY_VERSION_HEX >= 0x03060000 - pickle_module = PyImport_ImportModule("pickle5"); - if (pickle_module == NULL){ - /* for protocol 5, raise a clear ImportError if pickle5 is not found - */ - PyErr_SetString(PyExc_ImportError, "Using pickle protocol 5 " - "requires the pickle5 module for python versions >=3.6 " - "and <3.8"); - return NULL; - } + /* we expect protocol 5 to be available in Python 3.8 */ + pickle_module = PyImport_ImportModule("pickle"); +#elif PY_VERSION_HEX >= 0x03060000 + pickle_module = PyImport_ImportModule("pickle5"); + if (pickle_module == NULL) { + /* for protocol 5, raise a clear ImportError if pickle5 is not found + */ + PyErr_SetString(PyExc_ImportError, "Using pickle protocol 5 " + "requires the pickle5 module for Python >=3.6 and <3.8"); + return NULL; + } #else - PyErr_SetString(PyExc_ValueError, "pickle protocol 5 is not available " - "for python versions < 3.6"); - return NULL; + PyErr_SetString(PyExc_ValueError, "pickle protocol 5 is not available " + "for Python < 3.6"); + return NULL; #endif - if (pickle_module == NULL){ - return NULL; - } - - pickle_class = PyObject_GetAttrString(pickle_module, - "PickleBuffer"); + if (pickle_module == NULL){ + return NULL; + } + picklebuf_class = PyObject_GetAttrString(pickle_module, "PickleBuffer"); + Py_DECREF(pickle_module); + if (picklebuf_class == NULL) { + return NULL; + } - class_args_tuple = PyTuple_New(1); - if (!PyArray_IS_C_CONTIGUOUS((PyArrayObject*)self) && - PyArray_IS_F_CONTIGUOUS((PyArrayObject*)self)){ + /* Construct a PickleBuffer of the array */ - /* if the array if Fortran-contiguous and not C-contiguous, - * the PickleBuffer instance will hold a view on the transpose - * of the initial array, that is C-contiguous. */ - order = 'F'; - transposed_array = PyArray_Transpose((PyArrayObject*)self, NULL); - PyTuple_SET_ITEM(class_args_tuple, 0, transposed_array); - } - else { - order = 'C'; - PyTuple_SET_ITEM(class_args_tuple, 0, (PyObject *)self); - Py_INCREF(self); - } + if (!PyArray_IS_C_CONTIGUOUS((PyArrayObject*) self) && + PyArray_IS_F_CONTIGUOUS((PyArrayObject*) self)) { + /* if the array if Fortran-contiguous and not C-contiguous, + * the PickleBuffer instance will hold a view on the transpose + * of the initial array, that is C-contiguous. */ + order = 'F'; + transposed_array = PyArray_Transpose((PyArrayObject*)self, NULL); + picklebuf_args = Py_BuildValue("(N)", transposed_array); + } + else { + order = 'C'; + picklebuf_args = Py_BuildValue("(O)", self); + } + if (picklebuf_args == NULL) { + Py_DECREF(picklebuf_class); + return NULL; + } - class_args = Py_BuildValue("O", class_args_tuple); + buffer = PyObject_CallObject(picklebuf_class, picklebuf_args); + Py_DECREF(picklebuf_class); + Py_DECREF(picklebuf_args); + if (buffer == NULL) { + /* Some arrays may refuse to export a buffer, in which case + * just fall back on regular __reduce_ex__ implementation + * (gh-12745). + */ + PyErr_Clear(); + return array_reduce_ex_regular(self, protocol); + } - buffer = PyObject_CallObject(pickle_class, class_args); + /* Get the _frombuffer() function for reconstruction */ - numeric_mod = PyImport_ImportModule("numpy.core.numeric"); - if (numeric_mod == NULL) { - Py_DECREF(ret); - return NULL; - } - from_buffer_func = PyObject_GetAttrString(numeric_mod, - "_frombuffer"); - Py_DECREF(numeric_mod); + numeric_mod = PyImport_ImportModule("numpy.core.numeric"); + if (numeric_mod == NULL) { + Py_DECREF(buffer); + return NULL; + } + from_buffer_func = PyObject_GetAttrString(numeric_mod, + "_frombuffer"); + Py_DECREF(numeric_mod); + if (from_buffer_func == NULL) { + Py_DECREF(buffer); + return NULL; + } - Py_INCREF(descr); + return Py_BuildValue("N(NONN)", + from_buffer_func, buffer, (PyObject *)descr, + PyObject_GetAttrString((PyObject *)self, "shape"), + PyUnicode_FromStringAndSize(&order, 1)); +} - buffer_tuple = PyTuple_New(4); - PyTuple_SET_ITEM(buffer_tuple, 0, buffer); - PyTuple_SET_ITEM(buffer_tuple, 1, (PyObject *)descr); - PyTuple_SET_ITEM(buffer_tuple, 2, - PyObject_GetAttrString((PyObject *)self, - "shape")); - PyTuple_SET_ITEM(buffer_tuple, 3, - PyUnicode_FromStringAndSize(&order, - (Py_ssize_t)1)); +static PyObject * +array_reduce_ex(PyArrayObject *self, PyObject *args) +{ + int protocol; + PyArray_Descr *descr = NULL; - PyTuple_SET_ITEM(ret, 0, from_buffer_func); - PyTuple_SET_ITEM(ret, 1, buffer_tuple); + if (!PyArg_ParseTuple(args, "i", &protocol)) { + return NULL; + } - return ret; - } - else { - PyErr_Format(PyExc_ValueError, - "cannot call __reduce_ex__ with protocol >= %d", - 5); - return NULL; - } + descr = PyArray_DESCR(self); + if ((protocol < 5) || + (!PyArray_IS_C_CONTIGUOUS((PyArrayObject*)self) && + !PyArray_IS_F_CONTIGUOUS((PyArrayObject*)self)) || + PyDataType_FLAGCHK(descr, NPY_ITEM_HASOBJECT) || + (PyType_IsSubtype(((PyObject*)self)->ob_type, &PyArray_Type) && + ((PyObject*)self)->ob_type != &PyArray_Type) || + PyDataType_ISUNSIZED(descr)) { + /* The PickleBuffer class from version 5 of the pickle protocol + * can only be used for arrays backed by a contiguous data buffer. + * For all other cases we fallback to the generic array_reduce + * method that involves using a temporary bytes allocation. */ + return array_reduce_ex_regular(self, protocol); + } + else if (protocol == 5) { + return array_reduce_ex_picklebuffer(self, protocol); } else { + PyErr_Format(PyExc_ValueError, + "__reduce_ex__ called with protocol > 5"); return NULL; } - } static PyObject * diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 8135769d9..166533b3f 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -34,6 +34,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; /* Internal APIs */ +#include "arrayfunction_override.h" #include "arraytypes.h" #include "arrayobject.h" #include "hashdescr.h" @@ -3254,6 +3255,7 @@ array_datetime_data(PyObject *NPY_UNUSED(dummy), PyObject *args) } meta = get_datetime_metadata_from_dtype(dtype); + Py_DECREF(dtype); if (meta == NULL) { return NULL; } @@ -3618,6 +3620,7 @@ _vec_string_with_args(PyArrayObject* char_array, PyArray_Descr* type, if (nargs == -1 || nargs > NPY_MAXARGS) { PyErr_Format(PyExc_ValueError, "len(args) must be < %d", NPY_MAXARGS - 1); + Py_DECREF(type); goto err; } @@ -3625,6 +3628,7 @@ _vec_string_with_args(PyArrayObject* char_array, PyArray_Descr* type, for (i = 1; i < nargs; i++) { PyObject* item = PySequence_GetItem(args, i-1); if (item == NULL) { + Py_DECREF(type); goto err; } broadcast_args[i] = item; @@ -3633,6 +3637,7 @@ _vec_string_with_args(PyArrayObject* char_array, PyArray_Descr* type, in_iter = (PyArrayMultiIterObject*)PyArray_MultiIterFromObjects (broadcast_args, nargs, 0); if (in_iter == NULL) { + Py_DECREF(type); goto err; } n = in_iter->numiter; @@ -3713,6 +3718,7 @@ _vec_string_no_args(PyArrayObject* char_array, in_iter = (PyArrayIterObject*)PyArray_IterNew((PyObject*)char_array); if (in_iter == NULL) { + Py_DECREF(type); goto err; } @@ -3769,7 +3775,7 @@ static PyObject * _vec_string(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) { PyArrayObject* char_array = NULL; - PyArray_Descr *type = NULL; + PyArray_Descr *type; PyObject* method_name; PyObject* args_seq = NULL; @@ -3806,6 +3812,7 @@ _vec_string(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) result = _vec_string_with_args(char_array, type, method, args_seq); } else { + Py_DECREF(type); PyErr_SetString(PyExc_TypeError, "'args' must be a sequence of arguments"); goto err; @@ -4062,6 +4069,9 @@ normalize_axis_index(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) } static struct PyMethodDef array_module_methods[] = { + {"_get_implementing_args", + (PyCFunction)array__get_implementing_args, + METH_VARARGS, NULL}, {"_get_ndarray_c_version", (PyCFunction)array__get_ndarray_c_version, METH_VARARGS|METH_KEYWORDS, NULL}, @@ -4224,6 +4234,9 @@ static struct PyMethodDef array_module_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"_monotonicity", (PyCFunction)arr__monotonicity, METH_VARARGS | METH_KEYWORDS, NULL}, + {"implement_array_function", + (PyCFunction)array_implement_array_function, + METH_VARARGS, NULL}, {"interp", (PyCFunction)arr_interp, METH_VARARGS | METH_KEYWORDS, NULL}, {"interp_complex", (PyCFunction)arr_interp_complex, @@ -4476,6 +4489,7 @@ NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_array_wrap = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_array_finalize = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_buffer = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_ufunc = NULL; +NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_wrapped = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_order = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_copy = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_dtype = NULL; @@ -4492,6 +4506,7 @@ intern_strings(void) npy_ma_str_array_finalize = PyUString_InternFromString("__array_finalize__"); npy_ma_str_buffer = PyUString_InternFromString("__buffer__"); npy_ma_str_ufunc = PyUString_InternFromString("__array_ufunc__"); + npy_ma_str_wrapped = PyUString_InternFromString("__wrapped__"); npy_ma_str_order = PyUString_InternFromString("order"); npy_ma_str_copy = PyUString_InternFromString("copy"); npy_ma_str_dtype = PyUString_InternFromString("dtype"); @@ -4501,7 +4516,7 @@ intern_strings(void) return npy_ma_str_array && npy_ma_str_array_prepare && npy_ma_str_array_wrap && npy_ma_str_array_finalize && - npy_ma_str_buffer && npy_ma_str_ufunc && + npy_ma_str_buffer && npy_ma_str_ufunc && npy_ma_str_wrapped && npy_ma_str_order && npy_ma_str_copy && npy_ma_str_dtype && npy_ma_str_ndmin && npy_ma_str_axis1 && npy_ma_str_axis2; } diff --git a/numpy/core/src/multiarray/multiarraymodule.h b/numpy/core/src/multiarray/multiarraymodule.h index 3de68c549..60a3965c9 100644 --- a/numpy/core/src/multiarray/multiarraymodule.h +++ b/numpy/core/src/multiarray/multiarraymodule.h @@ -7,6 +7,7 @@ NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_array_wrap; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_array_finalize; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_buffer; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_ufunc; +NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_wrapped; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_order; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_copy; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_dtype; diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 90cff4077..18a2cc84f 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -1248,9 +1248,9 @@ npyiter_prepare_operands(int nop, PyArrayObject **op_in, return 1; fail_nop: - iop = nop; + iop = nop - 1; fail_iop: - for (i = 0; i < iop; ++i) { + for (i = 0; i < iop+1; ++i) { Py_XDECREF(op[i]); Py_XDECREF(op_dtype[i]); } @@ -3175,6 +3175,7 @@ npyiter_allocate_transfer_functions(NpyIter *iter) &stransfer, &transferdata, &needs_api) != NPY_SUCCEED) { + iop -= 1; /* This one cannot be cleaned up yet. */ goto fail; } readtransferfn[iop] = stransfer; @@ -3268,7 +3269,7 @@ npyiter_allocate_transfer_functions(NpyIter *iter) return 1; fail: - for (i = 0; i < iop; ++i) { + for (i = 0; i < iop+1; ++i) { if (readtransferdata[iop] != NULL) { NPY_AUXDATA_FREE(readtransferdata[iop]); readtransferdata[iop] = NULL; diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 5a9f3c5fa..30a81e0ca 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -2355,6 +2355,8 @@ npyiter_close(NewNpyArrayIterObject *self) } ret = NpyIter_Deallocate(iter); self->iter = NULL; + Py_XDECREF(self->nested_child); + self->nested_child = NULL; if (ret < 0) { return NULL; } diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index d153a8a64..420501ce2 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -599,15 +599,16 @@ array_positive(PyArrayObject *m1) PyErr_Restore(exc, val, tb); return NULL; } + Py_XDECREF(exc); + Py_XDECREF(val); + Py_XDECREF(tb); + /* 2018-06-28, 1.16.0 */ if (DEPRECATE("Applying '+' to a non-numerical array is " "ill-defined. Returning a copy, but in the future " "this will error.") < 0) { return NULL; } - Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); value = PyArray_Return((PyArrayObject *)PyArray_Copy(m1)); } return value; diff --git a/numpy/core/src/multiarray/refcount.c b/numpy/core/src/multiarray/refcount.c index 4b018b056..b8230c81a 100644 --- a/numpy/core/src/multiarray/refcount.c +++ b/numpy/core/src/multiarray/refcount.c @@ -19,8 +19,12 @@ static void _fillobject(char *optr, PyObject *obj, PyArray_Descr *dtype); -/* Incref all objects found at this record */ + /*NUMPY_API + * XINCREF all objects in a single array item. This is complicated for + * structured datatypes where the position of objects needs to be extracted. + * The function is execute recursively for each nested field or subarrays dtype + * such as as `np.dtype([("field1", "O"), ("field2", "f,O", (3,2))])` */ NPY_NO_EXPORT void PyArray_Item_INCREF(char *data, PyArray_Descr *descr) @@ -51,11 +55,37 @@ PyArray_Item_INCREF(char *data, PyArray_Descr *descr) PyArray_Item_INCREF(data + offset, new); } } + else if (PyDataType_HASSUBARRAY(descr)) { + int size, i, inner_elsize; + + inner_elsize = descr->subarray->base->elsize; + if (inner_elsize == 0) { + /* There cannot be any elements, so return */ + return; + } + /* Subarrays are always contiguous in memory */ + size = descr->elsize / inner_elsize; + + for (i = 0; i < size; i++){ + /* Recursively increment the reference count of subarray elements */ + PyArray_Item_INCREF(data + i * inner_elsize, + descr->subarray->base); + } + } + else { + /* This path should not be reachable. */ + assert(0); + } return; } -/* XDECREF all objects found at this record */ + /*NUMPY_API + * + * XDECREF all objects in a single array item. This is complicated for + * structured datatypes where the position of objects needs to be extracted. + * The function is execute recursively for each nested field or subarrays dtype + * such as as `np.dtype([("field1", "O"), ("field2", "f,O", (3,2))])` */ NPY_NO_EXPORT void PyArray_Item_XDECREF(char *data, PyArray_Descr *descr) @@ -87,6 +117,27 @@ PyArray_Item_XDECREF(char *data, PyArray_Descr *descr) PyArray_Item_XDECREF(data + offset, new); } } + else if (PyDataType_HASSUBARRAY(descr)) { + int size, i, inner_elsize; + + inner_elsize = descr->subarray->base->elsize; + if (inner_elsize == 0) { + /* There cannot be any elements, so return */ + return; + } + /* Subarrays are always contiguous in memory */ + size = descr->elsize / inner_elsize; + + for (i = 0; i < size; i++){ + /* Recursively decrement the reference count of subarray elements */ + PyArray_Item_XDECREF(data + i * inner_elsize, + descr->subarray->base); + } + } + else { + /* This path should not be reachable. */ + assert(0); + } return; } @@ -258,6 +309,10 @@ _fillobject(char *optr, PyObject *obj, PyArray_Descr *dtype) Py_XDECREF(arr); } } + if (dtype->type_num == NPY_OBJECT) { + Py_XINCREF(obj); + NPY_COPY_PYOBJECT_PTR(optr, &obj); + } else if (PyDataType_HASFIELDS(dtype)) { PyObject *key, *value, *title = NULL; PyArray_Descr *new; @@ -274,15 +329,26 @@ _fillobject(char *optr, PyObject *obj, PyArray_Descr *dtype) _fillobject(optr + offset, obj, new); } } - else { - npy_intp i; - npy_intp nsize = dtype->elsize / sizeof(obj); + else if (PyDataType_HASSUBARRAY(dtype)) { + int size, i, inner_elsize; - for (i = 0; i < nsize; i++) { - Py_XINCREF(obj); - NPY_COPY_PYOBJECT_PTR(optr, &obj); - optr += sizeof(obj); + inner_elsize = dtype->subarray->base->elsize; + if (inner_elsize == 0) { + /* There cannot be any elements, so return */ + return; + } + /* Subarrays are always contiguous in memory */ + size = dtype->elsize / inner_elsize; + + /* Call _fillobject on each item recursively. */ + for (i = 0; i < size; i++){ + _fillobject(optr, obj, dtype->subarray->base); + optr += inner_elsize; } - return; } + else { + /* This path should not be reachable. */ + assert(0); + } + return; } diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 2f71c8ae9..52de31289 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -2599,6 +2599,8 @@ NPY_NO_EXPORT PyTypeObject PyGenericArrType_Type = { static void void_dealloc(PyVoidScalarObject *v) { + _dealloc_cached_buffer_info((PyObject *)v); + if (v->flags & NPY_ARRAY_OWNDATA) { npy_free_cache(v->obval, Py_SIZE(v)); } diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index 8e8090002..2e8fb514f 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -40,19 +40,27 @@ maintainer email: oliphant.travis@ieee.org NPY_NO_EXPORT PyArray_Descr **userdescrs=NULL; -static int * -_append_new(int *types, int insert) +static int +_append_new(int **p_types, int insert) { int n = 0; int *newtypes; + int *types = *p_types; while (types[n] != NPY_NOTYPE) { n++; } newtypes = (int *)realloc(types, (n + 2)*sizeof(int)); + if (newtypes == NULL) { + PyErr_NoMemory(); + return -1; + } newtypes[n] = insert; newtypes[n + 1] = NPY_NOTYPE; - return newtypes; + + /* Replace the passed-in pointer */ + *p_types = newtypes; + return 0; } static npy_bool @@ -247,10 +255,13 @@ PyArray_RegisterCanCast(PyArray_Descr *descr, int totype, */ if (descr->f->cancastto == NULL) { descr->f->cancastto = (int *)malloc(1*sizeof(int)); + if (descr->f->cancastto == NULL) { + PyErr_NoMemory(); + return -1; + } descr->f->cancastto[0] = NPY_NOTYPE; } - descr->f->cancastto = _append_new(descr->f->cancastto, - totype); + return _append_new(&descr->f->cancastto, totype); } else { /* register with cancastscalarkindto */ @@ -258,6 +269,10 @@ PyArray_RegisterCanCast(PyArray_Descr *descr, int totype, int i; descr->f->cancastscalarkindto = (int **)malloc(NPY_NSCALARKINDS* sizeof(int*)); + if (descr->f->cancastscalarkindto == NULL) { + PyErr_NoMemory(); + return -1; + } for (i = 0; i < NPY_NSCALARKINDS; i++) { descr->f->cancastscalarkindto[i] = NULL; } @@ -265,11 +280,13 @@ PyArray_RegisterCanCast(PyArray_Descr *descr, int totype, if (descr->f->cancastscalarkindto[scalar] == NULL) { descr->f->cancastscalarkindto[scalar] = (int *)malloc(1*sizeof(int)); + if (descr->f->cancastscalarkindto[scalar] == NULL) { + PyErr_NoMemory(); + return -1; + } descr->f->cancastscalarkindto[scalar][0] = NPY_NOTYPE; } - descr->f->cancastscalarkindto[scalar] = - _append_new(descr->f->cancastscalarkindto[scalar], totype); + return _append_new(&descr->f->cancastscalarkindto[scalar], totype); } - return 0; } |