Merge pull request #10740 from charris/add-python-3.7-tests

MAINT, TST: Fixes for Python 3.7

10 files changed, 198 insertions, 37 deletions

diff --git a/.travis.yml b/.travis.yml
index 1aa033caa..168a7a385 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -35,6 +35,7 @@ python:
   - 3.4
   - 3.5
   - 3.6
+  - 3.7-dev
 matrix:
   include:
     - python: 3.6
diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src
index e8aa19416..5e6804a5c 100644
--- a/numpy/core/src/multiarray/arraytypes.c.src
+++ b/numpy/core/src/multiarray/arraytypes.c.src
@@ -998,11 +998,25 @@ VOID_setitem(PyObject *op, void *input, void *vap)
      * undiscerning case: It interprets any object as a buffer
      * and reads as many bytes as possible, padding with 0.
      */
+#if defined(NPY_PY3K)
+    {
+        Py_buffer view;
+
+        if (PyObject_GetBuffer(op, &view, PyBUF_SIMPLE) < 0) {
+            return -1;
+        }
+        memcpy(ip, view.buf, PyArray_MIN(view.len, itemsize));
+        if (itemsize > view.len) {
+            memset(ip + view.len, 0, itemsize - view.len);
+        }
+        PyBuffer_Release(&view);
+    }
+#else
     {
         const void *buffer;
         Py_ssize_t buflen;
-        res = PyObject_AsReadBuffer(op, &buffer, &buflen);
-        if (res == -1) {
+
+        if (PyObject_AsReadBuffer(op, &buffer, &buflen) < 0) {
             return -1;
         }
         memcpy(ip, buffer, PyArray_MIN(buflen, itemsize));
@@ -1010,6 +1024,7 @@ VOID_setitem(PyObject *op, void *input, void *vap)
             memset(ip + buflen, 0, itemsize - buflen);
         }
     }
+#endif
     return 0;
 }
 
diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c
index 10efdc4c8..f191f8db4 100644
--- a/numpy/core/src/multiarray/common.c
+++ b/numpy/core/src/multiarray/common.c
@@ -305,7 +305,8 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims,
         memset(&buffer_view, 0, sizeof(Py_buffer));
         if (PyObject_GetBuffer(obj, &buffer_view,
                                PyBUF_FORMAT|PyBUF_STRIDES) == 0 ||
-            PyObject_GetBuffer(obj, &buffer_view, PyBUF_FORMAT) == 0) {
+            PyObject_GetBuffer(obj, &buffer_view,
+                               PyBUF_FORMAT|PyBUF_SIMPLE) == 0) {
 
             PyErr_Clear();
             dtype = _descriptor_from_pep3118_format(buffer_view.format);
@@ -633,8 +634,12 @@ NPY_NO_EXPORT npy_bool
 _IsWriteable(PyArrayObject *ap)
 {
     PyObject *base=PyArray_BASE(ap);
+#if defined(NPY_PY3K)
+    Py_buffer view;
+#else
     void *dummy;
     Py_ssize_t n;
+#endif
 
     /* If we own our own data, then no-problem */
     if ((base == NULL) || (PyArray_FLAGS(ap) & NPY_ARRAY_OWNDATA)) {
@@ -668,9 +673,18 @@ _IsWriteable(PyArrayObject *ap)
     if (PyString_Check(base)) {
         return NPY_TRUE;
     }
+#if defined(NPY_PY3K)
+    if (PyObject_GetBuffer(base, &view, PyBUF_WRITABLE|PyBUF_SIMPLE) < 0) {
+        PyErr_Clear();
+        return NPY_FALSE;
+    }
+    PyBuffer_Release(&view);
+#else
     if (PyObject_AsWriteBuffer(base, &dummy, &n) < 0) {
+        PyErr_Clear();
         return NPY_FALSE;
     }
+#endif
     return NPY_TRUE;
 }
 
diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c
index 56bd65eed..5ee385c46 100644
--- a/numpy/core/src/multiarray/compiled_base.c
+++ b/numpy/core/src/multiarray/compiled_base.c
@@ -1402,7 +1402,11 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args)
 {
     PyObject *obj;
     PyObject *str;
+    #if (PY_VERSION_HEX >= 0x030700A2)
+    const char *docstr;
+    #else
     char *docstr;
+    #endif
     static char *msg = "already has a docstring";
     PyObject *tp_dict = PyArrayDescr_Type.tp_dict;
     PyObject *myobj;
diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c
index 2bb1cbfc1..b39834266 100644
--- a/numpy/core/src/multiarray/conversion_utils.c
+++ b/numpy/core/src/multiarray/conversion_utils.c
@@ -165,10 +165,12 @@ PyArray_BufferConverter(PyObject *obj, PyArray_Chunk *buf)
     }
 
 #if defined(NPY_PY3K)
-    if (PyObject_GetBuffer(obj, &view, PyBUF_ANY_CONTIGUOUS|PyBUF_WRITABLE) != 0) {
+    if (PyObject_GetBuffer(obj, &view,
+                PyBUF_ANY_CONTIGUOUS|PyBUF_WRITABLE|PyBUF_SIMPLE) != 0) {
         PyErr_Clear();
         buf->flags &= ~NPY_ARRAY_WRITEABLE;
-        if (PyObject_GetBuffer(obj, &view, PyBUF_ANY_CONTIGUOUS) != 0) {
+        if (PyObject_GetBuffer(obj, &view,
+                PyBUF_ANY_CONTIGUOUS|PyBUF_SIMPLE) != 0) {
             return NPY_FAIL;
         }
     }
@@ -177,8 +179,10 @@ PyArray_BufferConverter(PyObject *obj, PyArray_Chunk *buf)
     buf->len = (npy_intp) view.len;
 
     /*
-     * XXX: PyObject_AsWriteBuffer does also this, but it is unsafe, as there is
-     * no strict guarantee that the buffer sticks around after being released.
+     * In Python 3 both of the deprecated functions PyObject_AsWriteBuffer and
+     * PyObject_AsReadBuffer that this code replaces release the buffer. It is
+     * up to the object that supplies the buffer to guarantee that the buffer
+     * sticks around after the release.
      */
     PyBuffer_Release(&view);
 
diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c
index 3d6b161b1..0eba077da 100644
--- a/numpy/core/src/multiarray/ctors.c
+++ b/numpy/core/src/multiarray/ctors.c
@@ -729,13 +729,16 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it,
     /* PEP 3118 buffer interface */
     if (PyObject_CheckBuffer(obj) == 1) {
         memset(&buffer_view, 0, sizeof(Py_buffer));
-        if (PyObject_GetBuffer(obj, &buffer_view, PyBUF_STRIDES) == 0 ||
-            PyObject_GetBuffer(obj, &buffer_view, PyBUF_ND) == 0) {
+        if (PyObject_GetBuffer(obj, &buffer_view,
+                    PyBUF_STRIDES|PyBUF_SIMPLE) == 0 ||
+                PyObject_GetBuffer(obj, &buffer_view,
+                    PyBUF_ND|PyBUF_SIMPLE) == 0) {
             int nd = buffer_view.ndim;
+
             if (nd < *maxndim) {
                 *maxndim = nd;
             }
-            for (i=0; i<*maxndim; i++) {
+            for (i = 0; i < *maxndim; i++) {
                 d[i] = buffer_view.shape[i];
             }
             PyBuffer_Release(&buffer_view);
@@ -756,6 +759,7 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it,
     e = PyArray_LookupSpecial_OnInstance(obj, "__array_struct__");
     if (e != NULL) {
         int nd = -1;
+
         if (NpyCapsule_Check(e)) {
             PyArrayInterface *inter;
             inter = (PyArrayInterface *)NpyCapsule_AsVoidPtr(e);
@@ -2187,7 +2191,11 @@ PyArray_FromInterface(PyObject *origin)
     PyArrayObject *ret;
     PyArray_Descr *dtype = NULL;
     char *data = NULL;
+#if defined(NPY_PY3K)
+    Py_buffer view;
+#else
     Py_ssize_t buffer_len;
+#endif
     int res, i, n;
     npy_intp dims[NPY_MAXDIMS], strides[NPY_MAXDIMS];
     int dataflags = NPY_ARRAY_BEHAVED;
@@ -2217,7 +2225,7 @@ PyArray_FromInterface(PyObject *origin)
     if (PyUnicode_Check(attr)) {
         PyObject *tmp = PyUnicode_AsASCIIString(attr);
         if (tmp == NULL) {
-            goto fail; 
+            goto fail;
         }
         attr = tmp;
     }
@@ -2251,7 +2259,7 @@ PyArray_FromInterface(PyObject *origin)
             dtype = new_dtype;
         }
     }
-  
+
 #if defined(NPY_PY3K)
     Py_DECREF(attr);  /* Pairs with the unicode handling above */
 #endif
@@ -2335,6 +2343,25 @@ PyArray_FromInterface(PyObject *origin)
         else {
             base = origin;
         }
+#if defined(NPY_PY3K)
+        if (PyObject_GetBuffer(base, &view,
+                    PyBUF_WRITABLE|PyBUF_SIMPLE) < 0) {
+            PyErr_Clear();
+            if (PyObject_GetBuffer(base, &view,
+                        PyBUF_SIMPLE) < 0) {
+                goto fail;
+            }
+            dataflags &= ~NPY_ARRAY_WRITEABLE;
+        }
+        data = (char *)view.buf;
+        /*
+         * In Python 3 both of the deprecated functions PyObject_AsWriteBuffer and
+         * PyObject_AsReadBuffer that this code replaces release the buffer. It is
+         * up to the object that supplies the buffer to guarantee that the buffer
+         * sticks around after the release.
+         */
+        PyBuffer_Release(&view);
+#else
         res = PyObject_AsWriteBuffer(base, (void **)&data, &buffer_len);
         if (res < 0) {
             PyErr_Clear();
@@ -2345,6 +2372,7 @@ PyArray_FromInterface(PyObject *origin)
             }
             dataflags &= ~NPY_ARRAY_WRITEABLE;
         }
+#endif
         /* Get offset number from interface specification */
         attr = PyDict_GetItemString(origin, "offset");
         if (attr) {
@@ -3480,6 +3508,9 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type,
 {
     PyArrayObject *ret;
     char *data;
+#if defined(NPY_PY3K)
+    Py_buffer view;
+#endif
     Py_ssize_t ts;
     npy_intp s, n;
     int itemsize;
@@ -3519,6 +3550,26 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type,
         Py_INCREF(buf);
     }
 
+#if defined(NPY_PY3K)
+    if (PyObject_GetBuffer(buf, &view, PyBUF_WRITABLE|PyBUF_SIMPLE) < 0) {
+        writeable = 0;
+        PyErr_Clear();
+        if (PyObject_GetBuffer(buf, &view, PyBUF_SIMPLE) < 0) {
+            Py_DECREF(buf);
+            Py_DECREF(type);
+            return NULL;
+        }
+    }
+    data = (char *)view.buf;
+    ts = view.len;
+    /*
+     * In Python 3 both of the deprecated functions PyObject_AsWriteBuffer and
+     * PyObject_AsReadBuffer that this code replaces release the buffer. It is
+     * up to the object that supplies the buffer to guarantee that the buffer
+     * sticks around after the release.
+     */
+    PyBuffer_Release(&view);
+#else
     if (PyObject_AsWriteBuffer(buf, (void *)&data, &ts) == -1) {
         writeable = 0;
         PyErr_Clear();
@@ -3528,6 +3579,7 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type,
             return NULL;
         }
     }
+#endif
 
     if ((offset < 0) || (offset > ts)) {
         PyErr_Format(PyExc_ValueError,
diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c
index 825363f19..86e6e7a2f 100644
--- a/numpy/core/src/multiarray/getset.c
+++ b/numpy/core/src/multiarray/getset.c
@@ -106,8 +106,12 @@ array_strides_set(PyArrayObject *self, PyObject *obj)
     npy_intp offset = 0;
     npy_intp lower_offset = 0;
     npy_intp upper_offset = 0;
+#if defined(NPY_PY3K)
+    Py_buffer view;
+#else
     Py_ssize_t buf_len;
     char *buf;
+#endif
 
     if (obj == NULL) {
         PyErr_SetString(PyExc_AttributeError,
@@ -132,12 +136,21 @@ array_strides_set(PyArrayObject *self, PyObject *obj)
      * Get the available memory through the buffer interface on
      * PyArray_BASE(new) or if that fails from the current new
      */
-    if (PyArray_BASE(new) && PyObject_AsReadBuffer(PyArray_BASE(new),
-                                           (const void **)&buf,
-                                           &buf_len) >= 0) {
+#if defined(NPY_PY3K)
+    if (PyArray_BASE(new) &&
+            PyObject_GetBuffer(PyArray_BASE(new), &view, PyBUF_SIMPLE) >= 0) {
+        offset = PyArray_BYTES(self) - (char *)view.buf;
+        numbytes = view.len + offset;
+        PyBuffer_Release(&view);
+    }
+#else
+    if (PyArray_BASE(new) &&
+            PyObject_AsReadBuffer(PyArray_BASE(new), (const void **)&buf,
+                                  &buf_len) >= 0) {
         offset = PyArray_BYTES(self) - buf;
         numbytes = buf_len + offset;
     }
+#endif
     else {
         PyErr_Clear();
         offset_bounds_from_strides(PyArray_ITEMSIZE(new), PyArray_NDIM(new),
@@ -328,6 +341,9 @@ array_data_set(PyArrayObject *self, PyObject *op)
     void *buf;
     Py_ssize_t buf_len;
     int writeable=1;
+#if defined(NPY_PY3K)
+    Py_buffer view;
+#endif
 
     /* 2016-19-02, 1.12 */
     int ret = DEPRECATE("Assigning the 'data' attribute is an "
@@ -342,18 +358,38 @@ array_data_set(PyArrayObject *self, PyObject *op)
                 "Cannot delete array data");
         return -1;
     }
+#if defined(NPY_PY3K)
+    if (PyObject_GetBuffer(op, &view, PyBUF_WRITABLE|PyBUF_SIMPLE) < 0) {
+        writeable = 0;
+        PyErr_Clear();
+        if (PyObject_GetBuffer(op, &view, PyBUF_SIMPLE) < 0) {
+            return -1;
+        }
+    }
+    buf = view.buf;
+    buf_len = view.len;
+    /*
+     * In Python 3 both of the deprecated functions PyObject_AsWriteBuffer and
+     * PyObject_AsReadBuffer that this code replaces release the buffer. It is
+     * up to the object that supplies the buffer to guarantee that the buffer
+     * sticks around after the release.
+     */
+    PyBuffer_Release(&view);
+#else
     if (PyObject_AsWriteBuffer(op, &buf, &buf_len) < 0) {
+        PyErr_Clear();
         writeable = 0;
         if (PyObject_AsReadBuffer(op, (const void **)&buf, &buf_len) < 0) {
+            PyErr_Clear();
             PyErr_SetString(PyExc_AttributeError,
-                            "object does not have single-segment " \
-                            "buffer interface");
+                    "object does not have single-segment buffer interface");
             return -1;
         }
     }
+#endif
     if (!PyArray_ISONESEGMENT(self)) {
-        PyErr_SetString(PyExc_AttributeError, "cannot set single-" \
-                        "segment buffer for discontiguous array");
+        PyErr_SetString(PyExc_AttributeError,
+                "cannot set single-segment buffer for discontiguous array");
         return -1;
     }
     if (PyArray_NBYTES(self) > buf_len) {
diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c
index cd88ab76b..004af8a70 100644
--- a/numpy/core/src/multiarray/methods.c
+++ b/numpy/core/src/multiarray/methods.c
@@ -2305,7 +2305,7 @@ array_setflags(PyArrayObject *self, PyObject *args, PyObject *kwds)
         }
         else {
             PyErr_SetString(PyExc_ValueError,
-                            "cannot set aligned flag of mis-"\
+                            "cannot set aligned flag of mis-"
                             "aligned array to True");
             return NULL;
         }
@@ -2315,7 +2315,7 @@ array_setflags(PyArrayObject *self, PyObject *args, PyObject *kwds)
         if (PyObject_IsTrue(uic)) {
             fa->flags = flagback;
             PyErr_SetString(PyExc_ValueError,
-                            "cannot set WRITEBACKIFCOPY "       \
+                            "cannot set WRITEBACKIFCOPY "
                             "flag to True");
             return NULL;
         }
diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src
index ee83206de..9ea26e5e9 100644
--- a/numpy/core/src/multiarray/scalartypes.c.src
+++ b/numpy/core/src/multiarray/scalartypes.c.src
@@ -1842,6 +1842,9 @@ static PyObject *
 gentype_reduce(PyObject *self, PyObject *NPY_UNUSED(args))
 {
     PyObject *ret = NULL, *obj = NULL, *mod = NULL;
+#if defined(NPY_PY3K)
+    Py_buffer view;
+#endif
     const char *buffer;
     Py_ssize_t buflen;
 
@@ -1850,18 +1853,35 @@ gentype_reduce(PyObject *self, PyObject *NPY_UNUSED(args))
     if (ret == NULL) {
         return NULL;
     }
+
 #if defined(NPY_PY3K)
     if (PyArray_IsScalar(self, Unicode)) {
         /* Unicode on Python 3 does not expose the buffer interface */
         buffer = PyUnicode_AS_DATA(self);
         buflen = PyUnicode_GET_DATA_SIZE(self);
     }
-    else
-#endif
+    else if (PyObject_GetBuffer(self, &view, PyBUF_SIMPLE) >= 0) {
+        buffer = view.buf;
+        buflen = view.len;
+        /*
+         * In Python 3 both of the deprecated functions PyObject_AsWriteBuffer and
+         * PyObject_AsReadBuffer that this code replaces release the buffer. It is
+         * up to the object that supplies the buffer to guarantee that the buffer
+         * sticks around after the release.
+         */
+        PyBuffer_Release(&view);
+    }
+    else {
+        Py_DECREF(ret);
+        return NULL;
+    }
+#else
     if (PyObject_AsReadBuffer(self, (const void **)&buffer, &buflen)<0) {
         Py_DECREF(ret);
         return NULL;
     }
+#endif
+
     mod = PyImport_ImportModule("numpy.core.multiarray");
     if (mod == NULL) {
         return NULL;
diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py
index a97b627f9..33b3555b0 100644
--- a/numpy/testing/tests/test_utils.py
+++ b/numpy/testing/tests/test_utils.py
@@ -1114,18 +1114,28 @@ class TestStringEqual(unittest.TestCase):
                           lambda: assert_string_equal("foo", "hello"))
 
 
-def assert_warn_len_equal(mod, n_in_context, py3_n_in_context=None):
+def assert_warn_len_equal(mod, n_in_context, py34=None, py37=None):
     mod_warns = mod.__warningregistry__
+    num_warns = len(mod_warns)
     # Python 3.4 appears to clear any pre-existing warnings of the same type,
     # when raising warnings inside a catch_warnings block. So, there is a
     # warning generated by the tests within the context manager, but no
     # previous warnings.
     if 'version' in mod_warns:
-        if py3_n_in_context is None:
-            py3_n_in_context = n_in_context
-        assert_equal(len(mod_warns) - 1, py3_n_in_context)
-    else:
-        assert_equal(len(mod_warns), n_in_context)
+        # Python 3 adds a 'version' entry to the registry,
+        # do not count it.
+        num_warns -= 1
+
+        # Behavior of warnings is Python version dependent. Adjust the
+        # expected result to compensate. In particular, Python 3.7 does
+        # not make an entry for ignored warnings.
+        if sys.version_info[:2] >= (3, 7):
+            if py37 is not None:
+                n_in_context = py37
+        elif sys.version_info[:2] >= (3, 4):
+            if py34 is not None:
+                n_in_context = py34
+    assert_equal(num_warns, n_in_context)
 
 
 def _get_fresh_mod():
@@ -1134,6 +1144,8 @@ def _get_fresh_mod():
     try:
         my_mod.__warningregistry__.clear()
     except AttributeError:
+        # will not have a __warningregistry__ unless warning has been
+        # raised in the module at some point
         pass
     return my_mod
 
@@ -1147,21 +1159,23 @@ def test_clear_and_catch_warnings():
         warnings.warn('Some warning')
     assert_equal(my_mod.__warningregistry__, {})
     # Without specified modules, don't clear warnings during context
+    # Python 3.7 catch_warnings doesn't make an entry for 'ignore'.
     with clear_and_catch_warnings():
         warnings.simplefilter('ignore')
         warnings.warn('Some warning')
-    assert_warn_len_equal(my_mod, 1)
+    assert_warn_len_equal(my_mod, 1, py37=0)
     # Confirm that specifying module keeps old warning, does not add new
     with clear_and_catch_warnings(modules=[my_mod]):
         warnings.simplefilter('ignore')
         warnings.warn('Another warning')
-    assert_warn_len_equal(my_mod, 1)
+    assert_warn_len_equal(my_mod, 1, py37=0)
     # Another warning, no module spec does add to warnings dict, except on
     # Python 3.4 (see comments in `assert_warn_len_equal`)
+    # Python 3.7 catch_warnings doesn't make an entry for 'ignore'.
     with clear_and_catch_warnings():
         warnings.simplefilter('ignore')
         warnings.warn('Another warning')
-    assert_warn_len_equal(my_mod, 2, 1)
+    assert_warn_len_equal(my_mod, 2, py34=1, py37=0)
 
 
 def test_suppress_warnings_module():
@@ -1178,6 +1192,7 @@ def test_suppress_warnings_module():
         np.apply_along_axis(warn, 0, [0])
 
     # Test module based warning suppression:
+    assert_warn_len_equal(my_mod, 0)
     with suppress_warnings() as sup:
         sup.record(UserWarning)
         # suppress warning from other module (may have .pyc ending),
@@ -1189,8 +1204,7 @@ def test_suppress_warnings_module():
     # got filtered)
     assert_(len(sup.log) == 1)
     assert_(sup.log[0].message.args[0] == "Some warning")
-
-    assert_warn_len_equal(my_mod, 0)
+    assert_warn_len_equal(my_mod, 0, py37=0)
     sup = suppress_warnings()
     # Will have to be changed if apply_along_axis is moved:
     sup.filter(module=my_mod)
@@ -1204,11 +1218,11 @@ def test_suppress_warnings_module():
     assert_warn_len_equal(my_mod, 0)
 
     # Without specified modules, don't clear warnings during context
+    # Python 3.7 does not add ignored warnings.
     with suppress_warnings():
         warnings.simplefilter('ignore')
         warnings.warn('Some warning')
-    assert_warn_len_equal(my_mod, 1)
-
+    assert_warn_len_equal(my_mod, 1, py37=0)
 
 def test_suppress_warnings_type():
     # Initial state of module, no warnings
@@ -1232,10 +1246,11 @@ def test_suppress_warnings_type():
     assert_warn_len_equal(my_mod, 0)
 
     # Without specified modules, don't clear warnings during context
+    # Python 3.7 does not add ignored warnings.
     with suppress_warnings():
         warnings.simplefilter('ignore')
         warnings.warn('Some warning')
-    assert_warn_len_equal(my_mod, 1)
+    assert_warn_len_equal(my_mod, 1, py37=0)
 
 
 def test_suppress_warnings_decorate_no_record():