From 999753f85153ae68536aebe9f51ee3ff02b950bc Mon Sep 17 00:00:00 2001 From: Julian Taylor Date: Sat, 12 Apr 2014 13:29:35 +0200 Subject: MAINT: annotate reference stealing and burrowed refs in multiarray allows for better static analysis report with e.g. cpychecker --- numpy/core/code_generators/genapi.py | 63 +++++++++++++++++++----- numpy/core/code_generators/generate_numpy_api.py | 10 +++- numpy/core/code_generators/generate_ufunc_api.py | 2 +- numpy/core/code_generators/numpy_api.py | 42 ++++++++-------- numpy/core/include/numpy/ndarraytypes.h | 4 +- numpy/core/include/numpy/npy_common.h | 14 ++++++ numpy/core/src/multiarray/convert_datatype.c | 2 +- numpy/core/src/multiarray/methods.c | 3 +- numpy/core/src/multiarray/multiarraymodule.c | 2 +- numpy/core/src/multiarray/scalarapi.c | 5 +- tools/travis-test.sh | 2 +- 11 files changed, 105 insertions(+), 44 deletions(-) diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index ad054920a..bae25f6ef 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -74,6 +74,29 @@ def remove_whitespace(s): def _repl(str): return str.replace('Bool', 'npy_bool') + +class StealRef: + def __init__(self, arg): + self.arg = arg # counting from 1 + + def __str__(self): + try: + return ' '.join('NPY_STEALS_REF_TO_ARG(%d)' % x for x in self.arg) + except TypeError: + return 'NPY_STEALS_REF_TO_ARG(%d)' % self.arg + + +class NonNull: + def __init__(self, arg): + self.arg = arg # counting from 1 + + def __str__(self): + try: + return ' '.join('NPY_GCC_NONNULL(%d)' % x for x in self.arg) + except TypeError: + return 'NPY_GCC_NONNULL(%d)' % self.arg + + class Function(object): def __init__(self, name, return_type, args, doc=''): self.name = name @@ -350,9 +373,10 @@ NPY_NO_EXPORT PyBoolScalarObject _PyArrayScalar_BoolValues[2]; return astr class FunctionApi(object): - def __init__(self, name, index, return_type, args, api_name): + def __init__(self, name, index, annotations, return_type, args, api_name): self.name = name self.index = index + self.annotations = annotations self.return_type = return_type self.args = args self.api_name = api_name @@ -377,17 +401,25 @@ class FunctionApi(object): return " (void *) %s" % self.name def internal_define(self): + annstr = [] + for a in self.annotations: + annstr.append(str(a)) + annstr = ' '.join(annstr) astr = """\ -NPY_NO_EXPORT %s %s \\\n (%s);""" % (self.return_type, - self.name, - self._argtypes_string()) +NPY_NO_EXPORT %s %s %s \\\n (%s);""" % (annstr, self.return_type, + self.name, + self._argtypes_string()) return astr def order_dict(d): - """Order dict by its values.""" + """ order api dict by values + may contain plain integer or (int, annotations) """ o = list(d.items()) def _key(x): - return (x[1], x[0]) + try: + return x[1] + (x[0],) + except TypeError: + return (x[1], x[0]) return sorted(o, key=_key) def merge_api_dicts(dicts): @@ -419,7 +451,13 @@ Same index has been used twice in api definition: %s raise ValueError(msg) # No 'hole' in the indexes may be allowed, and it must starts at 0 - indexes = set(d.values()) + # if its a tuple the first entry is the index, the rest are annotations + indexes = set() + for v in d.values(): + try: + indexes.add(v[0]) + except TypeError: + indexes.add(v) expected = set(range(len(indexes))) if not indexes == expected: diff = expected.symmetric_difference(indexes) @@ -434,7 +472,10 @@ def get_api_functions(tagname, api_dict): functions.extend(find_functions(f, tagname)) dfunctions = [] for func in functions: - o = api_dict[func.name] + try: + o = api_dict[func.name][0] + except TypeError: + o = api_dict[func.name] dfunctions.append( (o, func) ) dfunctions.sort() return [a[1] for a in dfunctions] @@ -444,11 +485,7 @@ def fullapi_hash(api_dicts): of the list of items in the API (as a string).""" a = [] for d in api_dicts: - def sorted_by_values(d): - """Sort a dictionary by its values. Assume the dictionary items is of - the form func_name -> order""" - return sorted(d.items(), key=lambda x_y: (x_y[1], x_y[0])) - for name, index in sorted_by_values(d): + for name, index in order_dict(d): a.extend(name) a.extend(str(index)) diff --git a/numpy/core/code_generators/generate_numpy_api.py b/numpy/core/code_generators/generate_numpy_api.py index ce270a6a0..b2ce473a9 100644 --- a/numpy/core/code_generators/generate_numpy_api.py +++ b/numpy/core/code_generators/generate_numpy_api.py @@ -208,8 +208,14 @@ def do_generate_api(targets, sources): multiarray_api_dict = {} for f in numpyapi_list: name = f.name - index = multiarray_funcs[name] - multiarray_api_dict[f.name] = FunctionApi(f.name, index, f.return_type, + try: + index = multiarray_funcs[name][0] + annotations = multiarray_funcs[name][1:] + except TypeError: + index = multiarray_funcs[name] + annotations = [] + multiarray_api_dict[f.name] = FunctionApi(f.name, index, annotations, + f.return_type, f.args, api_name) for name, index in global_vars.items(): diff --git a/numpy/core/code_generators/generate_ufunc_api.py b/numpy/core/code_generators/generate_ufunc_api.py index 6305385af..4dacd6eb3 100644 --- a/numpy/core/code_generators/generate_ufunc_api.py +++ b/numpy/core/code_generators/generate_ufunc_api.py @@ -173,7 +173,7 @@ def do_generate_api(targets, sources): for f in ufunc_api_list: name = f.name index = ufunc_api_index[name] - ufunc_api_dict[name] = FunctionApi(f.name, index, f.return_type, + ufunc_api_dict[name] = FunctionApi(f.name, index, [], f.return_type, f.args, api_name) for name, index in numpy_api.ufunc_types_api.items(): diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index 791bc6ffc..2a0da9c6e 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -14,6 +14,8 @@ exception, so it should hopefully not get unnoticed). """ from __future__ import division, absolute_import, print_function +from code_generators.genapi import StealRef, NonNull + multiarray_global_vars = { 'NPY_NUMUSERTYPES': 7, 'NPY_DEFAULT_ASSIGN_CASTING': 292, @@ -90,7 +92,7 @@ multiarray_funcs_api = { 'PyArray_TypeObjectFromType': 46, 'PyArray_Zero': 47, 'PyArray_One': 48, - 'PyArray_CastToType': 49, + 'PyArray_CastToType': (49, StealRef(2), NonNull(2)), 'PyArray_CastTo': 50, 'PyArray_CastAnyTo': 51, 'PyArray_CanCastSafely': 52, @@ -102,24 +104,24 @@ multiarray_funcs_api = { 'PyArray_DescrFromTypeObject': 58, 'PyArray_Size': 59, 'PyArray_Scalar': 60, - 'PyArray_FromScalar': 61, + 'PyArray_FromScalar': (61, StealRef(2)), 'PyArray_ScalarAsCtype': 62, 'PyArray_CastScalarToCtype': 63, 'PyArray_CastScalarDirect': 64, 'PyArray_ScalarFromObject': 65, 'PyArray_GetCastFunc': 66, 'PyArray_FromDims': 67, - 'PyArray_FromDimsAndDataAndDescr': 68, - 'PyArray_FromAny': 69, - 'PyArray_EnsureArray': 70, - 'PyArray_EnsureAnyArray': 71, + 'PyArray_FromDimsAndDataAndDescr': (68, StealRef(3)), + 'PyArray_FromAny': (69, StealRef(2)), + 'PyArray_EnsureArray': (70, StealRef(1)), + 'PyArray_EnsureAnyArray': (71, StealRef(1)), 'PyArray_FromFile': 72, 'PyArray_FromString': 73, 'PyArray_FromBuffer': 74, - 'PyArray_FromIter': 75, - 'PyArray_Return': 76, - 'PyArray_GetField': 77, - 'PyArray_SetField': 78, + 'PyArray_FromIter': (75, StealRef(2)), + 'PyArray_Return': (76, StealRef(1)), + 'PyArray_GetField': (77, StealRef(2), NonNull(2)), + 'PyArray_SetField': (78, StealRef(2), NonNull(2)), 'PyArray_Byteswap': 79, 'PyArray_Resize': 80, 'PyArray_MoveInto': 81, @@ -135,7 +137,7 @@ multiarray_funcs_api = { 'PyArray_ValidType': 91, 'PyArray_UpdateFlags': 92, 'PyArray_New': 93, - 'PyArray_NewFromDescr': 94, + 'PyArray_NewFromDescr': (94, StealRef(2)), 'PyArray_DescrNew': 95, 'PyArray_DescrNewFromType': 96, 'PyArray_GetPriority': 97, @@ -149,8 +151,8 @@ multiarray_funcs_api = { 'PyArray_CheckStrides': 105, 'PyArray_DescrNewByteorder': 106, 'PyArray_IterAllButAxis': 107, - 'PyArray_CheckFromAny': 108, - 'PyArray_FromArray': 109, + 'PyArray_CheckFromAny': (108, StealRef(2)), + 'PyArray_FromArray': (109, StealRef(2)), 'PyArray_FromInterface': 110, 'PyArray_FromStructInterface': 111, 'PyArray_FromArrayAttr': 112, @@ -178,7 +180,7 @@ multiarray_funcs_api = { 'PyArray_Reshape': 134, 'PyArray_Newshape': 135, 'PyArray_Squeeze': 136, - 'PyArray_View': 137, + 'PyArray_View': (137, StealRef(2)), 'PyArray_SwapAxes': 138, 'PyArray_Max': 139, 'PyArray_Min': 140, @@ -203,7 +205,7 @@ multiarray_funcs_api = { 'PyArray_MultiplyIntList': 159, 'PyArray_GetPtr': 160, 'PyArray_CompareLists': 161, - 'PyArray_AsCArray': 162, + 'PyArray_AsCArray': (162, StealRef(5)), 'PyArray_As1D': 163, 'PyArray_As2D': 164, 'PyArray_Free': 165, @@ -224,8 +226,8 @@ multiarray_funcs_api = { 'PyArray_ByteorderConverter': 180, 'PyArray_OrderConverter': 181, 'PyArray_EquivTypes': 182, - 'PyArray_Zeros': 183, - 'PyArray_Empty': 184, + 'PyArray_Zeros': (183, StealRef(3)), + 'PyArray_Empty': (184, StealRef(3)), 'PyArray_Where': 185, 'PyArray_Arange': 186, 'PyArray_ArangeObj': 187, @@ -316,18 +318,18 @@ multiarray_funcs_api = { 'PyArray_CanCastArrayTo': 274, 'PyArray_CanCastTypeTo': 275, 'PyArray_EinsteinSum': 276, - 'PyArray_NewLikeArray': 277, + 'PyArray_NewLikeArray': (277, StealRef(3)), 'PyArray_GetArrayParamsFromObject': 278, 'PyArray_ConvertClipmodeSequence': 279, 'PyArray_MatrixProduct2': 280, # End 1.6 API 'NpyIter_IsFirstVisit': 281, - 'PyArray_SetBaseObject': 282, + 'PyArray_SetBaseObject': (282, StealRef(2)), 'PyArray_CreateSortedStridePerm': 283, 'PyArray_RemoveAxesInPlace': 284, 'PyArray_DebugPrint': 285, 'PyArray_FailUnlessWriteable': 286, - 'PyArray_SetUpdateIfCopyBase': 287, + 'PyArray_SetUpdateIfCopyBase': (287, StealRef(2)), 'PyDataMem_NEW': 288, 'PyDataMem_FREE': 289, 'PyDataMem_RENEW': 290, diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index ce348ed11..21ff8cd1a 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1492,13 +1492,13 @@ PyArray_STRIDE(const PyArrayObject *arr, int istride) return ((PyArrayObject_fields *)arr)->strides[istride]; } -static NPY_INLINE PyObject * +static NPY_INLINE NPY_RETURNS_BORROWED_REF PyObject * PyArray_BASE(PyArrayObject *arr) { return ((PyArrayObject_fields *)arr)->base; } -static NPY_INLINE PyArray_Descr * +static NPY_INLINE NPY_RETURNS_BORROWED_REF PyArray_Descr * PyArray_DESCR(PyArrayObject *arr) { return ((PyArrayObject_fields *)arr)->descr; diff --git a/numpy/core/include/numpy/npy_common.h b/numpy/core/include/numpy/npy_common.h index 44d7e1010..6244783f5 100644 --- a/numpy/core/include/numpy/npy_common.h +++ b/numpy/core/include/numpy/npy_common.h @@ -73,6 +73,20 @@ #define NPY_INLINE #endif +#ifdef WITH_CPYCHECKER_RETURNS_BORROWED_REF_ATTRIBUTE + #define NPY_RETURNS_BORROWED_REF \ + __attribute__((cpychecker_returns_borrowed_ref)) +#else + #define NPY_RETURNS_BORROWED_REF +#endif + +#ifdef WITH_CPYCHECKER_STEALS_REFERENCE_TO_ARG_ATTRIBUTE + #define NPY_STEALS_REF_TO_ARG(n) \ + __attribute__((cpychecker_steals_reference_to_arg(n))) +#else + #define NPY_STEALS_REF_TO_ARG(n) +#endif + /* 64 bit file position support, also on win-amd64. Ticket #1660 */ #if defined(_MSC_VER) && defined(_WIN64) && (_MSC_VER > 1400) || \ defined(__MINGW32__) || defined(__MINGW64__) diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index 849386504..1db3bfe85 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -36,7 +36,7 @@ NPY_NO_EXPORT npy_intp REQUIRED_STR_LEN[] = {0, 3, 5, 10, 10, 20, 20, 20, 20}; * For backward compatibility * * Cast an array using typecode structure. - * steals reference to at --- cannot be NULL + * steals reference to dtype --- cannot be NULL * * This function always makes a copy of arr, even if the dtype * doesn't change. diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index abfa04cb6..9104ffd03 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -352,9 +352,9 @@ array_swapaxes(PyArrayObject *self, PyObject *args) } -/* steals typed reference */ /*NUMPY_API Get a subset of bytes from each element of the array + steals reference to typed, must not be NULL */ NPY_NO_EXPORT PyObject * PyArray_GetField(PyArrayObject *self, PyArray_Descr *typed, int offset) @@ -410,6 +410,7 @@ array_getfield(PyArrayObject *self, PyObject *args, PyObject *kwds) /*NUMPY_API Set a subset of bytes from each element of the array + steals reference to dtype, must not be NULL */ NPY_NO_EXPORT int PyArray_SetField(PyArrayObject *self, PyArray_Descr *dtype, diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index c6d3c632f..a05e87e84 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1540,7 +1540,7 @@ PyArray_EquivTypenums(int typenum1, int typenum2) * NPY_RELAXED_STRIDES_CHECKING: If the strides logic is changed, the * order specific stride setting is not necessary. */ -static PyObject * +static NPY_STEALS_REF_TO_ARG(1) PyObject * _prepend_ones(PyArrayObject *arr, int nd, int ndmin, NPY_ORDER order) { npy_intp newdims[NPY_MAXDIMS]; diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index 8bbfb41fa..d71823566 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -831,8 +831,9 @@ PyArray_Scalar(void *data, PyArray_Descr *descr, PyObject *base) /*NUMPY_API * - *Return either an array or the appropriate Python object if the array - *is 0d and matches a Python type. + * Return either an array or the appropriate Python object if the array + * is 0d and matches a Python type. + * steals reference to mp */ NPY_NO_EXPORT PyObject * PyArray_Return(PyArrayObject *mp) diff --git a/tools/travis-test.sh b/tools/travis-test.sh index df9dedeca..f06aea8f6 100755 --- a/tools/travis-test.sh +++ b/tools/travis-test.sh @@ -24,7 +24,7 @@ if [ -z "$USE_DEBUG" ]; then else sysflags="$($PYTHON -c "from distutils import sysconfig; print (sysconfig.get_config_var('CFLAGS'))")" # windows compilers have this requirement - CFLAGS="$sysflags -Werror=declaration-after-statement" $PYTHON setup.py build_ext --inplace + CFLAGS="$sysflags -Werror=declaration-after-statement -Werror=nonnull" $PYTHON setup.py build_ext --inplace fi } -- cgit v1.2.3