diff options
| author | DongHun Kwak <dh0128.kwak@samsung.com> | 2020-12-31 09:38:55 +0900 |
|---|---|---|
| committer | DongHun Kwak <dh0128.kwak@samsung.com> | 2020-12-31 09:38:55 +0900 |
| commit | 92fba4b9b454bc82b27770074a248dd685053832 (patch) | |
| tree | 6ee53dfa05d52fbd824da6abc7d190485505f646 | |
| parent | 295fa02e974b890f98bb7bf6a94045c2cd3f5f68 (diff) | |
| download | python-numpy-92fba4b9b454bc82b27770074a248dd685053832.tar.gz python-numpy-92fba4b9b454bc82b27770074a248dd685053832.tar.bz2 python-numpy-92fba4b9b454bc82b27770074a248dd685053832.zip | |
Imported Upstream version 1.18.0upstream/1.18.0
454 files changed, 15166 insertions, 7056 deletions
diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 079496d93..000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,154 +0,0 @@ -# As config was originally based on an example by Olivier Grisel. Thanks! -# https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor.yml -clone_depth: 50 - -# No reason for us to restrict the number concurrent jobs -max_jobs: 100 - -cache: - - '%LOCALAPPDATA%\pip\Cache' - -environment: - global: - MINGW_32: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin - MINGW_64: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin - OPENBLAS_32: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-5f998ef_gcc7_1_0_win32.zip - OPENBLAS_64: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-5f998ef_gcc7_1_0_win64.zip - APPVEYOR_SAVE_CACHE_ON_ERROR: true - APPVEYOR_SKIP_FINALIZE_ON_EXIT: true - TEST_TIMEOUT: 1000 - NPY_NUM_BUILD_JOBS: 4 - - matrix: - - PYTHON: C:\Python36 - PYTHON_VERSION: 3.6 - PYTHON_ARCH: 32 - TEST_MODE: fast - - - PYTHON: C:\Python37 - PYTHON_VERSION: 3.7 - PYTHON_ARCH: 32 - TEST_MODE: fast - - - PYTHON: C:\Python36-x64 - PYTHON_VERSION: 3.6 - PYTHON_ARCH: 64 - TEST_MODE: full - INSTALL_PICKLE5: 1 - - - PYTHON: C:\Python37-x64 - PYTHON_VERSION: 3.7 - PYTHON_ARCH: 64 - TEST_MODE: full - INSTALL_PICKLE5: 1 - -init: - - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" - - "ECHO \"%APPVEYOR_SCHEDULED_BUILD%\"" - # If there is a newer build queued for the same PR, cancel this one. - # The AppVeyor 'rollout builds' option is supposed to serve the same - # purpose but it is problematic because it tends to cancel builds pushed - # directly to master instead of just PR builds (or the converse). - # credits: JuliaLang developers. - - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` - https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` - Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` - raise "There are newer queued builds for this pull request, skipping build." - } - -install: - # Prepend newly installed Python to the PATH of this build (this cannot be - # done from inside the powershell script as it would require to restart - # the parent CMD process). - - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - - if [%PYTHON_ARCH%]==[32] SET PATH=%MINGW_32%;%PATH% & SET OPENBLAS=%OPENBLAS_32% - - if [%PYTHON_ARCH%]==[64] SET PATH=%MINGW_64%;%PATH% & SET OPENBLAS=%OPENBLAS_64% - - # Check that we have the expected version and architecture for Python - - python --version - - >- - %CMD_IN_ENV% - python -c "import sys,platform,struct; - print(sys.platform, platform.machine(), struct.calcsize('P') * 8, )" - - # Install "openblas.a" to PYTHON\lib - # Library provided by Matthew Brett at https://github.com/matthew-brett/build-openblas - - ps: | - $clnt = new-object System.Net.WebClient - $file = "$(New-TemporaryFile).zip" - $tmpdir = New-TemporaryFile | %{ rm $_; mkdir $_ } - $destination = "$env:PYTHON\lib\openblas.a" - - echo $file - echo $tmpdir - echo $env:OPENBLAS - - $clnt.DownloadFile($env:OPENBLAS, $file) - Get-FileHash $file | Format-List - - Expand-Archive $file $tmpdir - - rm $tmpdir\$env:PYTHON_ARCH\lib\*.dll.a - $lib = ls $tmpdir\$env:PYTHON_ARCH\lib\*.a | ForEach { ls $_ } | Select-Object -first 1 - echo $lib - - cp $lib $destination - ls $destination - - # Upgrade to the latest pip. - - 'python -m pip install -U pip setuptools wheel' - - - if [%INSTALL_PICKLE5%]==[1] echo pickle5 >> tools/ci/appveyor/requirements.txt - - # Install the numpy test dependencies. - - 'pip install -U --timeout 5 --retries 2 -r tools/ci/appveyor/requirements.txt' - -build_script: - # Here, we add MinGW to the path to be able to link an OpenBLAS.dll - # We then use the import library from the DLL to compile with MSVC - - ps: | - pip wheel -v -v -v --wheel-dir=dist . - - # For each wheel that pip has placed in the "dist" directory - # First, upload the wheel to the "artifacts" tab and then - # install the wheel. If we have only built numpy (as is the case here), - # then there will be one wheel to install. - - # This method is more representative of what will be distributed, - # because it actually tests what the built wheels will be rather than - # what 'setup.py install' will do and at it uploads the wheels so that - # they can be inspected. - - ls dist -r | Foreach-Object { - Push-AppveyorArtifact $_.FullName - pip install $_.FullName - } - -test_script: - python runtests.py -v -n -m %TEST_MODE% -- --junitxml=%cd%\junit-results.xml - -after_build: - # Remove old or huge cache files to hopefully not exceed the 1GB cache limit. - # - # If the cache limit is reached, the cache will not be updated (of not even - # created in the first run). So this is a trade of between keeping the cache - # current and having a cache at all. - # NB: This is done only `on_success` since the cache in uploaded only on - # success anyway. - - C:\cygwin\bin\find "%LOCALAPPDATA%\pip" -type f -mtime +360 -delete - - C:\cygwin\bin\find "%LOCALAPPDATA%\pip" -type f -size +10M -delete - - C:\cygwin\bin\find "%LOCALAPPDATA%\pip" -empty -delete - # Show size of cache - - C:\cygwin\bin\du -hs "%LOCALAPPDATA%\pip\Cache" - -on_finish: - # We can get a nice display of test results in the "test" tab with py.test - # For now, this does nothing. - - ps: | - If (Test-Path .\junit-results.xml) { - (new-object net.webclient).UploadFile( - "https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", - (Resolve-Path .\junit-results.xml) - ) - } - $LastExitCode = 0 diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b4ab812f..772c3fbfd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: python3 -m venv venv ln -s $(which python3) venv/bin/python3.6 . venv/bin/activate - pip install cython sphinx==1.8.5 matplotlib ipython + pip install cython sphinx==2.2.0 matplotlib ipython sudo apt-get update sudo apt-get install -y graphviz texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-generic-extra latexmk texlive-xetex @@ -35,6 +35,14 @@ jobs: pip install scipy - run: + name: create release notes + command: | + . venv/bin/activate + pip install git+https://github.com/hawkowl/towncrier.git@master + VERSION=$(python -c "import setup; print(setup.VERSION)") + towncrier --version $VERSION --yes + ./tools/ci/test_all_newsfragments_used.py + - run: name: build devdocs command: | . venv/bin/activate diff --git a/.codecov.yml b/.codecov.yml index 35584a188..d92d54c9d 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,6 +1,4 @@ codecov: - ci: - - !appveyor notify: require_ci_to_pass: no after_n_builds: 1 diff --git a/.coveragerc b/.coveragerc index 1f61c25a4..9048b9cc4 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,4 @@ [run] branch = True include = */numpy/* +disable_warnings = include-ignored diff --git a/.dependabot/config.yml b/.dependabot/config.yml new file mode 100644 index 000000000..160ec85cf --- /dev/null +++ b/.dependabot/config.yml @@ -0,0 +1,9 @@ +version: 1 +update_configs: + - package_manager: "python" + directory: "/" + update_schedule: "weekly" + commit_message: + prefix: "MAINT" + default_labels: + - "03 - Maintenance" diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 01d9a537e..22113b913 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -16,12 +16,12 @@ Thanks for your interest in contributing code to numpy! + If this is your first time contributing to a project on GitHub, please read through our -[guide to contributing to numpy](https://docs.scipy.org/doc/numpy/dev/index.html) +[guide to contributing to numpy](https://numpy.org/devdocs/dev/index.html) + If you have contributed to other projects on GitHub you can go straight to our -[development workflow](https://docs.scipy.org/doc/numpy/dev/gitwash/development_workflow.html) +[development workflow](https://numpy.org/devdocs/dev/development_workflow.html) Either way, please be sure to follow our -[convention for commit messages](https://docs.scipy.org/doc/numpy/dev/gitwash/development_workflow.html#writing-the-commit-message). +[convention for commit messages](https://numpy.org/devdocs/dev/development_workflow.html#writing-the-commit-message). If you are writing new C code, please follow the style described in ``doc/C_STYLE_GUIDE``. diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 4c868d735..25322bd34 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,2 +1,3 @@ +github: [numfocus] tidelift: pypi/numpy custom: https://www.numpy.org/#support-numpy diff --git a/.gitignore b/.gitignore index 9283cb477..2ad02b560 100644 --- a/.gitignore +++ b/.gitignore @@ -146,10 +146,6 @@ numpy/core/src/npysort/radixsort.c numpy/core/src/npysort/selection.c numpy/core/src/npysort/timsort.c numpy/core/src/npysort/sort.c -numpy/core/src/common/npy_binsearch.h -numpy/core/src/common/npy_partition.h -numpy/core/src/common/npy_sort.h -numpy/core/src/common/templ_common.h numpy/core/src/private/npy_binsearch.h numpy/core/src/private/npy_partition.h numpy/core/src/private/templ_common.h @@ -16,6 +16,9 @@ extraction: index: build_command: - python3 setup.py build + after_prepare: + - pip3 install --upgrade --user cython + - export PATH="$HOME/.local/bin:$PATH" queries: - include: py/file-not-closed @@ -63,6 +63,7 @@ Christopher Hanley <chanley@gmail.com> chanley <chanley@gmail.com> Christoph Gohlke <cgohlke@uci.edu> cgholke <?@?> Christoph Gohlke <cgohlke@uci.edu> cgohlke <cgohlke@uci.edu> Christoph Gohlke <cgohlke@uci.edu> Christolph Gohlke <cgohlke@uci.edu> +Colin Snyder <47012605+colinsnyder@users.noreply.github.com> colinsnyder <47012605+colinsnyder@users.noreply.github.com> Daniel B Allan <daniel.b.allan@gmail.com> danielballan <daniel.b.allan@gmail.com> Daniel da Silva <mail@danieldasilva.org> Daniel da Silva <daniel@meltingwax.net> Daniel da Silva <mail@danieldasilva.org> Daniel da Silva <var.mail.daniel@gmail.com> @@ -126,6 +127,7 @@ Jérémie du Boisberranger <jeremie.du-boisberranger@inria.fr> jeremiedbb <34657 Jerome Kelleher <jerome.kelleher@ed.ac.uk> jeromekelleher <jerome.kelleher@ed.ac.uk> Johannes Hampp <johannes.hampp@zeu.uni-giessen.de> euronion <42553970+euronion@users.noreply.github.com> Johannes Schönberger <hannesschoenberger@gmail.com> Johannes Schönberger <jschoenberger@demuc.de> +Johann Faouzi <johann.faouzi@gmail.com> johann.faouzi <johann.faouzi@icm-institute.org> John Darbyshire <24256554+attack68@users.noreply.github.com> attack68 <24256554+attack68@users.noreply.github.com> Joseph Fox-Rabinovitz <jfoxrabinovitz@gmail.com> Joseph Fox-Rabinovitz <joseph.r.fox-rabinovitz@nasa.gov> Joseph Fox-Rabinovitz <jfoxrabinovitz@gmail.com> Joseph Fox-Rabinovitz <madphysicist@users.noreply.github.com> @@ -135,6 +137,7 @@ Julian Taylor <juliantaylor108@gmail.com> Julian Taylor <jtaylor.debian@googlema Julian Taylor <juliantaylor108@gmail.com> Julian Taylor <juliantaylor108@googlemail.com> Julien Lhermitte <jrmlhermitte@gmail.com> Julien Lhermitte <lhermitte@bnl.gov> Julien Schueller <julien.schueller@gmail.com> jschueller <julien.schueller@gmail.com> +Justus Magin <keewis@posteo.de> keewis <keewis@users.noreply.github.com> Kai Striega <kaistriega@gmail.com> kai <kaistriega@gmail.com> Kai Striega <kaistriega@gmail.com> kai-striega <kaistriega@gmail.com> Kai Striega <kaistriega@gmail.com> kai-striega <kaistriega+github@gmail.com> @@ -149,6 +152,8 @@ Lars Buitinck <larsmans@gmail.com> Lars Buitinck <L.J.Buitinck@uva.nl> Lars Grüter <lagru@mailbox.org> Lars G <lagru@mailbox.org> Luis Pedro Coelho <luis@luispedro.org> Luis Pedro Coelho <lpc@cmu.edu> Luke Zoltan Kelley <lkelley@cfa.harvard.edu> lzkelley <lkelley@cfa.harvard.edu> +Magdalena Proszewska <magdalena.proszewska@gmail.com> mpro <magdalena.proszewska@gmail.com> +Magdalena Proszewska <magdalena.proszewska@gmail.com> mproszewska <38814059+mproszewska@users.noreply.github.com> Manoj Kumar <manojkumarsivaraj334@gmail.com> MechCoder <manojkumarsivaraj334@gmail.com> Mark DePristo <mdepristo@synapdx.com> markdepristo <mdepristo@synapdx.com> Mark Weissman <mw9050@gmail.com> m-d-w <mw9050@gmail.com> @@ -157,6 +162,7 @@ Mark Wiebe <mwwiebe@gmail.com> Mark Wiebe <mwiebe@continuum.io> Mark Wiebe <mwwiebe@gmail.com> Mark Wiebe <mwiebe@enthought.com> Mark Wiebe <mwwiebe@gmail.com> Mark Wiebe <mwiebe@georg.(none)> Martin Goodson <martingoodson@gmail.com> martingoodson <martingoodson@gmail.com> +Martin Reinecke <martin@mpa-garching.mpg.de> mreineck <martin@mpa-garching.mpg.de> Martin Teichmann <martin.teichmann@xfel.eu> Martin Teichmann <lkb.teichmann@gmail.com> Martino Sorbaro <martino.sorbaro@ed.ac.uk> martinosorb <martino.sorbaro@ed.ac.uk> Mattheus Ueckermann <empeeu@yahoo.com> empeeu <empeeu@yahoo.com> @@ -232,7 +238,12 @@ Wojtek Ruszczewski <git@wr.waw.pl> wrwrwr <git@wr.waw.pl> Yuji Kanagawa <yuji.kngw.80s.revive@gmail.com> kngwyu <yuji.kngw.80s.revive@gmail.com> Yury Kirienko <yury.kirienko@gmail.com> kirienko <yury.kirienko@gmail.com> Zixu Zhao <zixu.zhao.tireless@gmail.com> ZZhaoTireless <zixu.zhao.tireless@gmail.com> -Ziyan Zhou<ziyan.zhou@mujin.co.jp> Ziyan <ziyan.zhou@mujin.co.jp> +Ziyan Zhou <ziyan.zhou@mujin.co.jp> Ziyan <ziyan.zhou@mujin.co.jp> +Zieji Pohz <poh.ziji@gmail.com> jpoh <poh.zijie@gmail.com> +Zieji Pohz <poh.ziji@gmail.com> zjpoh <poh.zijie@gmail.com> +Zieji Pohz <poh.ziji@gmail.com> Zijie (ZJ) Poh <8103276+zjpoh@users.noreply.github.com> +Zolisa Bleki <zolisa.bleki@gmail.com> zoj613 <44142765+zoj613@users.noreply.github.com> +Zolisa Bleki <zolisa.bleki@gmail.com> RedRuM <44142765+zoj613@users.noreply.github.com> luzpaz <kunda@scribus.net> luz.paz <luzpaz@users.noreply.github.com> luzpaz <kunda@scribus.net> luzpaz <luzpaz@users.noreply.github.com> spacescientist <aspacescientist@protonmail.com> spacescientist <spacescientist@pm.me> diff --git a/.travis.yml b/.travis.yml index ebfa713b2..64b7be051 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,9 +18,17 @@ cache: directories: - $HOME/.cache/pip +stage: Comprehensive tests + +stages: + # Do the style check and a single test job, don't proceed if it fails + - name: Initial tests + # Do the rest of the tests + - name: Comprehensive tests + env: global: - - OpenBLAS_version=0.3.5 + - OpenBLAS_version=0.3.7 - WHEELHOUSE_UPLOADER_USERNAME=travis.numpy # The following is generated with the command: # travis encrypt -r numpy/numpy WHEELHOUSE_UPLOADER_SECRET=tH3AP1KeY @@ -29,15 +37,18 @@ env: iFWt9Ka92CaqYdU7nqfWp9VImSndPmssjmCXJ1v1IjZPAM\ ahp7Qnm0rWRmA0z9SomuRUQOJQ6s684vU=" -python: - - 3.5 - - 3.6 - - 3.8-dev matrix: include: - - python: 3.7 - env: INSTALL_PICKLE5=1 + # Do all python versions without environment variables set + - stage: Initial tests + python: 3.8 + - python: 3.5 + - python: 3.6 + - python: 3.7 + + - python: 3.6 + dist: bionic env: USE_DEBUG=1 addons: apt: @@ -47,11 +58,14 @@ matrix: - python3-dbg - python3-dev - python3-setuptools - - python: 3.6 + + - python: 3.7 env: USE_WHEEL=1 RUN_FULL_TESTS=1 RUN_COVERAGE=1 INSTALL_PICKLE5=1 - - python: 3.6 + + - python: 3.7 env: USE_SDIST=1 - - python: 3.6 + + - python: 3.7 env: - PYTHONOPTIMIZE=2 - BLAS=None @@ -60,25 +74,37 @@ matrix: - NPY_BLAS_ORDER=mkl,blis,openblas,atlas,accelerate,blas - NPY_LAPACK_ORDER=MKL,OPENBLAS,ATLAS,ACCELERATE,LAPACK - USE_ASV=1 - - python: 3.5 + + - python: 3.7 env: NPY_RELAXED_STRIDES_CHECKING=0 - - python: 3.6 + + - python: 3.7 env: USE_WHEEL=1 NPY_RELAXED_STRIDES_DEBUG=1 - - python: 3.6 + + - python: 3.7 env: NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=0 - - python: 3.6 + + - python: 3.7 env: - BLAS=None - LAPACK=None - ATLAS=None - - os: linux-ppc64le - python: 3.6 + + - python: 3.7 + os: linux + arch: ppc64le env: - # for matrix annotation only - - PPC64_LE=1 - # use POWER8 OpenBLAS build, not system ATLAS + # use ppc64le OpenBLAS build, not system ATLAS - ATLAS=None + - python: 3.7 + os: linux + arch: s390x + env: + # use s390x OpenBLAS build, not system ATLAS + - ATLAS=None + + before_install: - ./tools/travis-before-install.sh diff --git a/INSTALL.rst.txt b/INSTALL.rst.txt index 640ddafc7..d6c42bad7 100644 --- a/INSTALL.rst.txt +++ b/INSTALL.rst.txt @@ -12,7 +12,7 @@ https://scipy.org/install.html. Prerequisites ============= -Building NumPy requires the following software installed: +Building NumPy requires the following installed software: 1) For Python 3, Python__ 3.5.x or newer. @@ -28,6 +28,7 @@ Building NumPy requires the following software installed: 2) Cython >= 0.29.2 (for development versions of numpy, not for released versions) + 3) pytest__ (optional) 1.15 or later This is required for testing numpy, but not for using it. @@ -36,7 +37,7 @@ Python__ http://www.python.org pytest__ http://pytest.readthedocs.io -.. note:: +.. note:: If you want to build NumPy in order to work on NumPy itself, use ``runtests.py``. For more details, see @@ -76,11 +77,7 @@ skipped when running the test suite if no Fortran compiler is available. For building Scipy a Fortran compiler is needed though, so we include some details on Fortran compilers in the rest of this section. -On OS X and Linux, all common compilers will work. Note that C99 support is -required. For compilers that don't support the C99 language standard by -default (such as ``gcc`` versions < 5.0), it should be enabled. For ``gcc``:: - - export CFLAGS='-std=c99' +On OS X and Linux, all common compilers will work. For Fortran, ``gfortran`` works, ``g77`` does not. In case ``g77`` is installed then ``g77`` will be detected and used first. To explicitly select @@ -91,7 +88,7 @@ installed then ``g77`` will be detected and used first. To explicitly select Windows ------- -On Windows, building from source can be difficult. Currently the most robust +On Windows, building from source can be difficult. Currently, the most robust option is to use the Intel compilers, or alternatively MSVC (the same version as used to build Python itself) with Intel ifort. Intel itself maintains a good `application note <https://software.intel.com/en-us/articles/numpyscipy-with-intel-mkl>`_ @@ -116,9 +113,9 @@ details. Windows ------- -The Intel compilers work with Intel MKL, see the application note linked above. +The Intel compilers work with Intel MKL, see the application note linked above. MingwPy__ works with OpenBLAS. -For an overview of the state of BLAS/LAPACK libraries on Windows, see +For an overview of the state of BLAS/LAPACK libraries on Windows, see `here <https://mingwpy.github.io/blas_lapack.html>`_. OS X @@ -131,7 +128,7 @@ ATLAS) will also work. Ubuntu/Debian ------------- -For best performance a development package providing BLAS and CBLAS should be +For best performance, a development package providing BLAS and CBLAS should be installed. Some of the options available are: - ``libblas-dev``: reference BLAS (not very optimized) diff --git a/MANIFEST.in b/MANIFEST.in index 7c8f3b6ef..b58f85d4d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,19 +5,24 @@ # Avoid using MANIFEST.in for that. # include MANIFEST.in +include pyproject.toml include pytest.ini include *.txt include README.md include site.cfg.example +include runtests.py +include tox.ini +include .coveragerc +include test_requirements.txt recursive-include numpy/random *.pyx *.pxd *.pyx.in *.pxd.in +include numpy/random/include/* +include numpy/__init__.pxd # Add build support that should go in sdist, but not go in bdist/be installed # Note that sub-directories that don't have __init__ are apparently not # included by 'recursive-include', so list those separately recursive-include numpy * recursive-include numpy/_build_utils * recursive-include numpy/linalg/lapack_lite * -include runtests.py -include tox.ini pytest.ini .coveragerc recursive-include tools * # Add sdist files whose use depends on local configuration. include numpy/core/src/common/cblasfuncs.c @@ -2,8 +2,6 @@ []( https://travis-ci.org/numpy/numpy) -[]( - https://ci.appveyor.com/project/charris/numpy) []( https://dev.azure.com/numpy/numpy/_build/latest?definitionId=5) []( @@ -12,7 +10,7 @@ NumPy is the fundamental package needed for scientific computing with Python. - **Website:** https://www.numpy.org -- **Documentation:** http://docs.scipy.org/ +- **Documentation:** https://docs.scipy.org/ - **Mailing list:** https://mail.python.org/mailman/listinfo/numpy-discussion - **Source code:** https://github.com/numpy/numpy - **Contributing:** https://www.numpy.org/devdocs/dev/index.html @@ -35,4 +33,17 @@ Tests can then be run after installation with: python -c 'import numpy; numpy.test()' + +Call for Contributions +---------------------- + +NumPy appreciates help from a wide range of different backgrounds. +Work such as high level documentation or website improvements are valuable +and we would like to grow our team with people filling these roles. +Small improvements or fixes are always appreciated and issues labeled as easy +may be a good starting point. +If you are considering larger contributions outside the traditional coding work, +please contact us through the mailing list. + + [](https://numfocus.org) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index edb577cdb..9e1e52952 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,216 +8,207 @@ trigger: variables: # OpenBLAS_version should be updated # to match numpy-wheels repo - OpenBLAS_version: 0.3.7.dev + OpenBLAS_version: 0.3.7 -jobs: -- job: Linux_Python_36_32bit_full_with_asserts - pool: - vmImage: 'ubuntu-16.04' - steps: - - script: | - docker pull i386/ubuntu:bionic - docker run -v $(pwd):/numpy i386/ubuntu:bionic /bin/bash -c "cd numpy && \ - apt-get -y update && \ - apt-get -y install python3.6-dev python3-pip locales python3-certifi && \ - locale-gen fr_FR && update-locale && \ - pip3 install setuptools nose cython==0.29.0 pytest pytz pickle5 && \ - apt-get -y install gfortran-5 wget && \ - target=\$(python3 tools/openblas_support.py) && \ - cp -r \$target/usr/local/lib/* /usr/lib && \ - cp \$target/usr/local/include/* /usr/include && \ - python3 -m pip install . && \ - F77=gfortran-5 F90=gfortran-5 \ - CFLAGS='-UNDEBUG -std=c99' python3 runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml && \ - python3 tools/openblas_support.py --check_version $(OpenBLAS_version)" - displayName: 'Run 32-bit Ubuntu Docker Build / Tests' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.6-32 bit full Linux' -- job: macOS - pool: - # NOTE: at time of writing, there is a danger - # that using an invalid vmIMage string for macOS - # image silently redirects to a Windows build on Azure; - # for now, use the only image name officially present in - # the docs even though i.e., numba uses another in their - # azure config for mac os -- Microsoft has indicated - # they will patch this issue - vmImage: macOS-10.13 - steps: - # the @0 refers to the (major) version of the *task* on Microsoft's - # end, not the order in the build matrix nor anything to do - # with version of Python selected - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.6' - addToPath: true - architecture: 'x64' - # NOTE: do we have a compelling reason to use older / newer - # versions of Xcode toolchain for testing? - - script: /bin/bash -c "sudo xcode-select -s /Applications/Xcode_10.app/Contents/Developer" - displayName: 'select Xcode version' - # NOTE: might be better if we could avoid installing - # two C compilers, but with homebrew looks like we're - # now stuck getting the full gcc toolchain instead of - # just pulling in gfortran - - script: | - # same version of gfortran as the wheel builds - brew install gcc49 - # manually link critical gfortran libraries - ln -s /usr/local/Cellar/gcc@4.9/4.9.4_1/lib/gcc/4.9/libgfortran.3.dylib /usr/local/lib/libgfortran.3.dylib - ln -s /usr/local/Cellar/gcc@4.9/4.9.4_1/lib/gcc/4.9/libquadmath.0.dylib /usr/local/lib/libquadmath.0.dylib - # manually symlink gfortran-4.9 to plain gfortran - # for f2py - ln -s /usr/local/bin/gfortran-4.9 /usr/local/bin/gfortran - displayName: 'make gfortran available on mac os vm' - # use the pre-built openblas binary that most closely - # matches our MacOS wheel builds -- currently based - # primarily on file size / name details - - script: | - target=$(python tools/openblas_support.py) - # manually link to appropriate system paths - cp $target/usr/local/lib/* /usr/local/lib/ - cp $target/usr/local/include/* /usr/local/include/ - displayName: 'install pre-built openblas' - - script: python -m pip install --upgrade pip setuptools wheel - displayName: 'Install tools' - - script: python -m pip install cython nose pytz pytest pickle5 vulture docutils sphinx==1.8.5 numpydoc - displayName: 'Install dependencies; some are optional to avoid test skips' - - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'" - displayName: 'Check for unreachable code paths in Python modules' - # prefer usage of clang over gcc proper - # to match likely scenario on many user mac machines - - script: python setup.py build -j 4 install - displayName: 'Build NumPy' - env: - BLAS: None - LAPACK: None - ATLAS: None - ACCELERATE: None - CC: /usr/bin/clang - # wait until after dev build of NumPy to pip - # install matplotlib to avoid pip install of older numpy - - script: python -m pip install matplotlib - displayName: 'Install matplotlib before refguide run' - - script: python runtests.py -g --refguide-check - displayName: 'Run Refuide Check' - - script: python runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml - displayName: 'Run Full NumPy Test Suite' - - bash: python tools/openblas_support.py --check_version $(OpenBLAS_version) - displayName: 'Verify OpenBLAS version' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.6 64-bit full Mac OS' -- job: Windows - pool: - vmImage: 'VS2017-Win2016' - strategy: - maxParallel: 6 - matrix: - Python36-32bit-fast: - PYTHON_VERSION: '3.6' - PYTHON_ARCH: 'x86' - TEST_MODE: fast - BITS: 32 - Python37-32bit-fast: - PYTHON_VERSION: '3.7' - PYTHON_ARCH: 'x86' - TEST_MODE: fast - BITS: 32 - Python35-64bit-full: - PYTHON_VERSION: '3.5' - PYTHON_ARCH: 'x64' - TEST_MODE: full - BITS: 64 - Python36-64bit-full: - PYTHON_VERSION: '3.6' - PYTHON_ARCH: 'x64' - TEST_MODE: full - INSTALL_PICKLE5: 1 - BITS: 64 - Python37-64bit-full: - PYTHON_VERSION: '3.7' - PYTHON_ARCH: 'x64' - TEST_MODE: full - INSTALL_PICKLE5: 1 - BITS: 64 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PYTHON_VERSION) - addToPath: true - architecture: $(PYTHON_ARCH) - - script: python -m pip install --upgrade pip setuptools wheel - displayName: 'Install tools' - - script: python -m pip install cython nose pytz pytest - displayName: 'Install dependencies; some are optional to avoid test skips' - - script: if [%INSTALL_PICKLE5%]==[1] python -m pip install pickle5 - displayName: 'Install optional pickle5 backport (only for python3.6 and 3.7)' +stages: +- stage: InitialTests + jobs: + - job: WindowsFast + pool: + vmImage: 'VS2017-Win2016' + strategy: + matrix: + Python36-64bit-fast: + PYTHON_VERSION: '3.6' + PYTHON_ARCH: 'x64' + TEST_MODE: fast + BITS: 64 + steps: + - template: azure-steps-windows.yml - - powershell: | - $pyversion = python -c "from __future__ import print_function; import sys; print(sys.version.split()[0])" - Write-Host "Python Version: $pyversion" - $target = "C:\\hostedtoolcache\\windows\\Python\\$pyversion\\$(PYTHON_ARCH)\\lib\\openblas.a" - Write-Host "target path: $target" - $openblas = python tools/openblas_support.py - cp $openblas $target - displayName: 'Download / Install OpenBLAS' +- stage: ComprehensiveTests + jobs: + - job: Linux_Python_36_32bit_full_with_asserts + pool: + vmImage: 'ubuntu-16.04' + steps: + - script: | + docker pull i386/ubuntu:bionic + docker run -v $(pwd):/numpy i386/ubuntu:bionic /bin/bash -c "cd numpy && \ + apt-get -y update && \ + apt-get -y install python3.6-dev python3-pip locales python3-certifi && \ + locale-gen fr_FR && update-locale && \ + apt-get -y install gfortran-5 wget && \ + target=\$(python3 tools/openblas_support.py) && \ + cp -r \$target/usr/local/lib/* /usr/lib && \ + cp \$target/usr/local/include/* /usr/include && \ + python3 -m pip install --user --upgrade pip setuptools && \ + python3 -m pip install --user -r test_requirements.txt && \ + python3 -m pip install . && \ + F77=gfortran-5 F90=gfortran-5 \ + CFLAGS=-UNDEBUG python3 runtests.py -n --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml && \ + python3 tools/openblas_support.py --check_version $(OpenBLAS_version)" + displayName: 'Run 32-bit Ubuntu Docker Build / Tests' + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-*.xml' + failTaskOnFailedTests: true + testRunTitle: 'Publish test results for Python 3.6-32 bit full Linux' + - job: macOS + pool: + # NOTE: at time of writing, there is a danger + # that using an invalid vmIMage string for macOS + # image silently redirects to a Windows build on Azure; + # for now, use the only image name officially present in + # the docs even though i.e., numba uses another in their + # azure config for mac os -- Microsoft has indicated + # they will patch this issue + vmImage: macOS-10.14 + steps: + # the @0 refers to the (major) version of the *task* on Microsoft's + # end, not the order in the build matrix nor anything to do + # with version of Python selected + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.6' + addToPath: true + architecture: 'x64' + # NOTE: do we have a compelling reason to use older / newer + # versions of Xcode toolchain for testing? + - script: /bin/bash -c "sudo xcode-select -s /Applications/Xcode_10.app/Contents/Developer" + displayName: 'select Xcode version' + # NOTE: might be better if we could avoid installing + # two C compilers, but with homebrew looks like we're + # now stuck getting the full gcc toolchain instead of + # just pulling in gfortran + - script: | + # same version of gfortran as the wheel builds + brew install gcc49 + # manually link critical gfortran libraries + ln -s /usr/local/Cellar/gcc@4.9/4.9.4_1/lib/gcc/4.9/libgfortran.3.dylib /usr/local/lib/libgfortran.3.dylib + ln -s /usr/local/Cellar/gcc@4.9/4.9.4_1/lib/gcc/4.9/libquadmath.0.dylib /usr/local/lib/libquadmath.0.dylib + # manually symlink gfortran-4.9 to plain gfortran + # for f2py + ln -s /usr/local/bin/gfortran-4.9 /usr/local/bin/gfortran + displayName: 'make gfortran available on mac os vm' + # use the pre-built openblas binary that most closely + # matches our MacOS wheel builds -- currently based + # primarily on file size / name details + - script: | + target=$(python tools/openblas_support.py) + # manually link to appropriate system paths + cp $target/usr/local/lib/* /usr/local/lib/ + cp $target/usr/local/include/* /usr/local/include/ + displayName: 'install pre-built openblas' + - script: python -m pip install --upgrade pip setuptools wheel + displayName: 'Install tools' + - script: | + python -m pip install -r test_requirements.txt + python -m pip install vulture docutils sphinx==2.2.0 numpydoc + displayName: 'Install dependencies; some are optional to avoid test skips' + - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'" + displayName: 'Check for unreachable code paths in Python modules' + # prefer usage of clang over gcc proper + # to match likely scenario on many user mac machines + - script: python setup.py build -j 4 build_src --verbose-cfg install + displayName: 'Build NumPy' + env: + BLAS: None + LAPACK: None + ATLAS: None + ACCELERATE: None + CC: /usr/bin/clang + # wait until after dev build of NumPy to pip + # install matplotlib to avoid pip install of older numpy + - script: python -m pip install matplotlib + displayName: 'Install matplotlib before refguide run' + - script: python runtests.py -g --refguide-check + displayName: 'Run Refuide Check' + - script: python runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml + displayName: 'Run Full NumPy Test Suite' + - bash: python tools/openblas_support.py --check_version $(OpenBLAS_version) + displayName: 'Verify OpenBLAS version' + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-*.xml' + failTaskOnFailedTests: true + testRunTitle: 'Publish test results for Python 3.6 64-bit full Mac OS' + - job: Windows + pool: + vmImage: 'VS2017-Win2016' + strategy: + maxParallel: 6 + matrix: + Python36-32bit-fast: + PYTHON_VERSION: '3.6' + PYTHON_ARCH: 'x86' + TEST_MODE: fast + BITS: 32 + Python37-32bit-fast: + PYTHON_VERSION: '3.7' + PYTHON_ARCH: 'x86' + TEST_MODE: fast + BITS: 32 + Python38-32bit-fast: + PYTHON_VERSION: '3.8' + PYTHON_ARCH: 'x86' + TEST_MODE: fast + BITS: 32 + Python35-64bit-full: + PYTHON_VERSION: '3.5' + PYTHON_ARCH: 'x64' + TEST_MODE: full + BITS: 64 + Python36-64bit-full: + PYTHON_VERSION: '3.6' + PYTHON_ARCH: 'x64' + TEST_MODE: full + BITS: 64 + Python37-64bit-full: + PYTHON_VERSION: '3.7' + PYTHON_ARCH: 'x64' + TEST_MODE: full + BITS: 64 + Python38-64bit-full: + PYTHON_VERSION: '3.8' + PYTHON_ARCH: 'x64' + TEST_MODE: full + BITS: 64 + steps: + - template: azure-steps-windows.yml + - job: Linux_PyPy3 + pool: + vmIMage: 'ubuntu-16.04' + steps: + - script: source tools/pypy-test.sh + displayName: 'Run PyPy3 Build / Tests' + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-*.xml' + testRunTitle: 'Publish test results for PyPy3' + failTaskOnFailedTests: true + - job: Linux_gcc48 + pool: + vmImage: 'ubuntu-18.04' + steps: + - script: | + if ! `gcc-4.8 2>/dev/null`; then + sudo apt install gcc-4.8 + fi + displayName: 'add gcc 4.8' + - script: | + python3 -m pip install --user --upgrade pip setuptools + python3 -m pip install --user -r test_requirements.txt + CPPFLAGS='' CC=gcc-4.8 F77=gfortran-5 F90=gfortran-5 \ + python3 runtests.py --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml + displayName: 'Run gcc4.8 Build / Tests' + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-*.xml' + failTaskOnFailedTests: true + testRunTitle: 'Publish test results for gcc 4.8' - - powershell: | - choco install -y mingw --forcex86 --force --version=5.3.0 - displayName: 'Install 32-bit mingw for 32-bit builds' - condition: eq(variables['BITS'], 32) - # NOTE: for Windows builds it seems much more tractable to use runtests.py - # vs. manual setup.py and then runtests.py for testing only - - powershell: | - If ($(BITS) -eq 32) { - $env:NPY_DISTUTILS_APPEND_FLAGS = 1 - $env:CFLAGS = "-m32" - $env:LDFLAGS = "-m32" - $env:PATH = "C:\\tools\\mingw32\\bin;" + $env:PATH - refreshenv - } - python -c "from tools import openblas_support; openblas_support.make_init('numpy')" - pip wheel -v -v -v --wheel-dir=dist . - - ls dist -r | Foreach-Object { - pip install $_.FullName - } - displayName: 'Build NumPy' - - bash: | - pushd . && cd .. && target=$(python -c "import numpy, os; print(os.path.abspath(os.path.join(os.path.dirname(numpy.__file__), '.libs')))") && popd - pip download -d destination --only-binary --no-deps numpy==1.14 - cd destination && unzip numpy*.whl && cp numpy/.libs/*.dll $target - ls $target - displayName: 'Add extraneous & older DLL to numpy/.libs to probe DLL handling robustness' - condition: eq(variables['PYTHON_VERSION'], '3.6') - - script: pushd . && cd .. && python -c "from ctypes import windll; windll.kernel32.SetDefaultDllDirectories(0x00000800); import numpy" && popd - displayName: 'For gh-12667; Windows DLL resolution' - - script: python runtests.py -n --show-build-log --mode=$(TEST_MODE) -- -rsx --junitxml=junit/test-results.xml - displayName: 'Run NumPy Test Suite' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python $(PYTHON_VERSION) $(BITS)-bit $(TEST_MODE) Windows' - -- job: Linux_PyPy3 - pool: - vmIMage: 'ubuntu-16.04' - steps: - - script: source tools/pypy-test.sh - displayName: 'Run PyPy3 Build / Tests' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - testRunTitle: 'Publish test results for PyPy3' - failTaskOnFailedTests: true diff --git a/azure-steps-windows.yml b/azure-steps-windows.yml new file mode 100644 index 000000000..26d7a667d --- /dev/null +++ b/azure-steps-windows.yml @@ -0,0 +1,56 @@ +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: $(PYTHON_VERSION) + addToPath: true + architecture: $(PYTHON_ARCH) +- script: python -m pip install --upgrade pip setuptools wheel + displayName: 'Install tools' +- script: python -m pip install -r test_requirements.txt + displayName: 'Install dependencies; some are optional to avoid test skips' +- powershell: | + $pyversion = python -c "from __future__ import print_function; import sys; print(sys.version.split()[0])" + Write-Host "Python Version: $pyversion" + $target = "C:\\hostedtoolcache\\windows\\Python\\$pyversion\\$(PYTHON_ARCH)\\lib\\openblas.a" + Write-Host "target path: $target" + $openblas = python tools/openblas_support.py + cp $openblas $target + displayName: 'Download / Install OpenBLAS' + +- powershell: | + choco install -y mingw --forcex86 --force --version=5.3.0 + displayName: 'Install 32-bit mingw for 32-bit builds' + condition: eq(variables['BITS'], 32) +# NOTE: for Windows builds it seems much more tractable to use runtests.py +# vs. manual setup.py and then runtests.py for testing only +- powershell: | + If ($(BITS) -eq 32) { + $env:CFLAGS = "-m32" + $env:LDFLAGS = "-m32" + $env:PATH = "C:\\tools\\mingw32\\bin;" + $env:PATH + refreshenv + } + python -c "from tools import openblas_support; openblas_support.make_init('numpy')" + pip wheel -v -v -v --wheel-dir=dist . + + ls dist -r | Foreach-Object { + pip install $_.FullName + } + displayName: 'Build NumPy' +- bash: | + pushd . && cd .. && target=$(python -c "import numpy, os; print(os.path.abspath(os.path.join(os.path.dirname(numpy.__file__), '.libs')))") && popd + pip download -d destination --only-binary --no-deps numpy==1.14 + cd destination && unzip numpy*.whl && cp numpy/.libs/*.dll $target + ls $target + displayName: 'Add extraneous & older DLL to numpy/.libs to probe DLL handling robustness' + condition: eq(variables['PYTHON_VERSION'], '3.6') +- script: pushd . && cd .. && python -c "from ctypes import windll; windll.kernel32.SetDefaultDllDirectories(0x00000800); import numpy" && popd + displayName: 'For gh-12667; Windows DLL resolution' +- script: python runtests.py -n --show-build-log --mode=$(TEST_MODE) -- -rsx --junitxml=junit/test-results.xml + displayName: 'Run NumPy Test Suite' +- task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-*.xml' + failTaskOnFailedTests: true + testRunTitle: 'Publish test results for Python $(PYTHON_VERSION) $(BITS)-bit $(TEST_MODE) Windows'
\ No newline at end of file diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 45da9533f..b3c7f9f20 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -35,7 +35,7 @@ // The Pythons you'd like to test against. If not provided, defaults // to the current version of Python used to run `asv`. - "pythons": ["3.6"], + "pythons": ["3.7"], // The matrix of dependencies to test. Each key is the name of a // package (in PyPI) and the values are version numbers. An empty @@ -43,6 +43,7 @@ // version. "matrix": { "six": [], + "Cython": [], }, // The directory (relative to the current directory) that benchmarks are @@ -68,7 +69,7 @@ // `asv` will cache wheels of the recent builds in each // environment, making them faster to install next time. This is // number of builds to keep, per environment. - "wheel_cache_size": 2, + "build_cache_size": 8, // The commits after which the regression search in `asv publish` // should start looking for regressions. Dictionary whose keys are diff --git a/benchmarks/benchmarks/bench_avx.py b/benchmarks/benchmarks/bench_avx.py new file mode 100644 index 000000000..f7b524e43 --- /dev/null +++ b/benchmarks/benchmarks/bench_avx.py @@ -0,0 +1,34 @@ +from __future__ import absolute_import, division, print_function + +from .common import Benchmark + +import numpy as np + +avx_ufuncs = ['sqrt', + 'absolute', + 'reciprocal', + 'square', + 'rint', + 'floor', + 'ceil' , + 'trunc'] +stride = [1, 2, 4] +dtype = ['f', 'd'] + +class AVX_UFunc(Benchmark): + params = [avx_ufuncs, stride, dtype] + param_names = ['avx_based_ufunc', 'stride', 'dtype'] + timeout = 10 + + def setup(self, ufuncname, stride, dtype): + np.seterr(all='ignore') + try: + self.f = getattr(np, ufuncname) + except AttributeError: + raise NotImplementedError() + N = 10000 + self.arr = np.ones(stride*N, dtype) + + def time_ufunc(self, ufuncname, stride, dtype): + self.f(self.arr[::stride]) + diff --git a/doc/DISTUTILS.rst.txt b/doc/DISTUTILS.rst.txt index eadde63f8..bcef82500 100644 --- a/doc/DISTUTILS.rst.txt +++ b/doc/DISTUTILS.rst.txt @@ -243,7 +243,7 @@ in writing setup scripts: after processing all source generators, no extension module will be built. This is the recommended way to conditionally define extension modules. Source generator functions are called by the - ``build_src`` command of ``numpy.distutils``. + ``build_src`` sub-command of ``numpy.distutils``. For example, here is a typical source generator function:: diff --git a/doc/HOWTO_RELEASE.rst.txt b/doc/HOWTO_RELEASE.rst.txt index e2aea12b7..53359f0a2 100644 --- a/doc/HOWTO_RELEASE.rst.txt +++ b/doc/HOWTO_RELEASE.rst.txt @@ -11,7 +11,7 @@ useful info can be found. Source tree ----------- -* INSTALL.txt +* INSTALL.rst.txt * release.sh * pavement.py @@ -197,17 +197,6 @@ best to read the pavement.py script. .. note:: The following steps are repeated for the beta(s), release candidates(s) and the final release. -Check that docs can be built ----------------------------- -Do:: - - cd doc/ - make dist - -to check that the documentation is in a buildable state. See -doc/HOWTO_BUILD_DOCS.rst.txt for more details and for how to update -https://docs.scipy.org. - Check deprecations ------------------ Before the release branch is made, it should be checked that all deprecated @@ -257,12 +246,19 @@ updated for a major release. Check the release notes ----------------------- -Check that the release notes are up-to-date. +Use `towncrier`_ to build the release note and +commit the changes. This will remove all the fragments from +``doc/release/upcoming_changes`` and add ``doc/release/<version>-note.rst``. +Note that currently towncrier must be installed from its master branch as the +last release (19.2.0) is outdated. + + towncrier --version "<version>" + git commit -m"Create release note" -Write or update the release notes in a file named for the release, such as -``doc/release/1.11.0-notes.rst``. +Check that the release notes are up-to-date. -Mention at least the following: +Update the release notes with a Highlights section. Mention some of the +following: - major new features - deprecated and removed features @@ -270,8 +266,7 @@ Mention at least the following: - for SciPy, supported NumPy version(s) - outlook for the near future -Also make sure that as soon as the branch is made, there is a new release -notes file in the master branch for the next release. +.. _towncrier: https://github.com/hawkowl/towncrier Update the release status and create a release "tag" ---------------------------------------------------- @@ -307,7 +302,7 @@ changes:: And make sure the ``VERSION`` variable is set properly. Now you can make the release commit and tag. We recommend you don't push -the commit or tag immediately, just in case you need to do more cleanup. We +the commit or tag immediately, just in case you need to do more cleanup. We prefer to defer the push of the tag until we're confident this is the exact form of the released code (see: :ref:`push-tag-and-commit`): @@ -383,14 +378,24 @@ Build the changelog and notes for upload with:: paver write_release_and_log -The tar-files and binary releases for distribution should be uploaded to SourceForge, -together with the Release Notes and the Changelog. Uploading can be done -through a web interface or, more efficiently, through scp/sftp/rsync as -described in the SourceForge -`upload guide <https://sourceforge.net/apps/trac/sourceforge/wiki/Release%20files%20for%20download>`_ (dead link). -For example:: +Build and archive documentation +------------------------------- +Do:: - scp <filename> <username>,numpy@frs.sourceforge.net:/home/frs/project/n/nu/numpy/NumPy/<releasedir>/ + cd doc/ + make dist + +to check that the documentation is in a buildable state. Then, after tagging, +create an archive of the documentation in the numpy/doc repo:: + + # This checks out github.com/numpy/doc and adds (``git add``) the + # documentation to the checked out repo. + make merge-doc + # Now edit the ``index.html`` file in the repo to reflect the new content, + # and commit the changes + git -C dist/merge commit -a "Add documentation for <version>" + # Push to numpy/doc repo + git -C push Update PyPI ----------- @@ -443,28 +448,6 @@ you released you can push the tag and release commit up to github:: where ``upstream`` points to the main https://github.com/numpy/numpy.git repository. -Update docs.scipy.org ---------------------- - -All documentation for a release can be updated on https://docs.scipy.org/ with: - - make dist - make upload USERNAME=<yourname> RELEASE=1.11.0 - -Note that ``<username>`` must have SSH credentials on the server. If you don't -have those, ask someone who does (the list currently includes @rgommers, -@juliantaylor and @pv). - -Also rebuild and upload ``docs.scipy.org`` front page, if the release -series is a new one. The front page sources have their own repo: -https://github.com/scipy/docs.scipy.org. Do the following: - -- Update ``index.rst`` for the new version. -- ``make dist`` -- Check that the built documentation is OK. -- ``touch output-is-fine`` -- ``make upload USERNAME=<username> RELEASE=1.x.y`` - Update scipy.org ---------------- diff --git a/doc/Makefile b/doc/Makefile index 00393abc6..3c32cb811 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -14,6 +14,10 @@ PYTHON = python$(PYVER) SPHINXOPTS ?= SPHINXBUILD ?= LANG=C sphinx-build PAPER ?= +# For merging a documentation archive into a git checkout of numpy/doc +# Turn a tag like v1.18.0 into 1.18 +# Use sed -n -e 's/patttern/match/p' to return a blank value if no match +TAG ?= $(shell git describe --tag | sed -n -e's,v\([1-9]\.[0-9]*\)\.[0-9].*,\1,p') FILES= @@ -24,7 +28,8 @@ ALLSPHINXOPTS = -WT --keep-going -d build/doctrees $(PAPEROPT_$(PAPER)) \ $(SPHINXOPTS) source .PHONY: help clean html web pickle htmlhelp latex changes linkcheck \ - dist dist-build gitwash-update version-check html-build latex-build + dist dist-build gitwash-update version-check html-build latex-build \ + merge-doc #------------------------------------------------------------------------------ @@ -40,6 +45,7 @@ help: @echo " dist PYVER=... to make a distribution-ready tree" @echo " gitwash-update GITWASH=path/to/gitwash update gitwash developer docs" @echo " upload USERNAME=... RELEASE=... to upload built docs to docs.scipy.org" + @echo " merge-doc TAG=... to clone numpy/doc and archive documentation into it" clean: -rm -rf build/* @@ -92,7 +98,9 @@ else endif -dist: +dist: build/dist.tar.gz + +build/dist.tar.gz: make $(DIST_VARS) real-dist real-dist: dist-build html-build html-scipyorg @@ -113,7 +121,7 @@ dist-build: install -d $(subst :, ,$(INSTALL_PPH)) $(PYTHON) `which easy_install` --prefix=$(INSTALL_DIR) ../dist/*.egg -upload: +upload: build/dist.tar.gz # SSH must be correctly configured for this to work. # Assumes that ``make dist`` was already run # Example usage: ``make upload USERNAME=rgommers RELEASE=1.10.1`` @@ -130,6 +138,32 @@ upload: ssh $(USERNAME)@docs.scipy.org rm $(UPLOAD_DIR)/dist.tar.gz ssh $(USERNAME)@docs.scipy.org ln -snf numpy-$(RELEASE) /srv/docs_scipy_org/doc/numpy + +merge-doc: build/dist.tar.gz +ifeq "$(TAG)" "" + echo tag "$(TAG)" not of the form 1.18; + exit 1; +endif + @# Only clone if the directory does not exist + @if ! test -d build/merge; then \ + git clone https://github.com/numpy/doc build/merge; \ + fi; + @# Remove any old content and copy in the new, add it to git + -rm -rf build/merge/$(TAG)/* + -mkdir -p build/merge/$(TAG) + @# -C changes working directory + tar -C build/merge/$(TAG) -xf build/dist.tar.gz + git -C build/merge add $(TAG) + @# For now, the user must do this. If it is onerous, automate it and change + @# the instructions in doc/HOWTO_RELEASE.rst.txt + @echo " " + @echo New documentation archive added to ./build/merge. + @echo Now add/modify the appropiate section after + @echo " <!-- insert here -->" + @echo in build/merge/index.html, + @echo then \"git commit\", \"git push\" + + #------------------------------------------------------------------------------ # Basic Sphinx generation rules for different formats #------------------------------------------------------------------------------ diff --git a/doc/Py3K.rst.txt b/doc/Py3K.rst.txt index f78b9e5db..b23536ca5 100644 --- a/doc/Py3K.rst.txt +++ b/doc/Py3K.rst.txt @@ -812,20 +812,20 @@ Types with tp_as_sequence defined PySequenceMethods in py3k are binary compatible with py2k, but some of the slots have gone away. I suspect this means some functions need redefining so -the semantics of the slots needs to be checked. - -PySequenceMethods foo_sequence_methods = { - (lenfunc)0, /* sq_length */ - (binaryfunc)0, /* sq_concat */ - (ssizeargfunc)0, /* sq_repeat */ - (ssizeargfunc)0, /* sq_item */ - (void *)0, /* nee sq_slice */ - (ssizeobjargproc)0, /* sq_ass_item */ - (void *)0, /* nee sq_ass_slice */ - (objobjproc)0, /* sq_contains */ - (binaryfunc)0, /* sq_inplace_concat */ - (ssizeargfunc)0 /* sq_inplace_repeat */ -}; +the semantics of the slots needs to be checked:: + + PySequenceMethods foo_sequence_methods = { + (lenfunc)0, /* sq_length */ + (binaryfunc)0, /* sq_concat */ + (ssizeargfunc)0, /* sq_repeat */ + (ssizeargfunc)0, /* sq_item */ + (void *)0, /* nee sq_slice */ + (ssizeobjargproc)0, /* sq_ass_item */ + (void *)0, /* nee sq_ass_slice */ + (objobjproc)0, /* sq_contains */ + (binaryfunc)0, /* sq_inplace_concat */ + (ssizeargfunc)0 /* sq_inplace_repeat */ + }; PyMappingMethods @@ -840,13 +840,13 @@ Types with tp_as_mapping defined * multiarray/arrayobject.c PyMappingMethods in py3k look to be the same as in py2k. The semantics -of the slots needs to be checked. +of the slots needs to be checked:: -PyMappingMethods foo_mapping_methods = { - (lenfunc)0, /* mp_length */ - (binaryfunc)0, /* mp_subscript */ - (objobjargproc)0 /* mp_ass_subscript */ -}; + PyMappingMethods foo_mapping_methods = { + (lenfunc)0, /* mp_length */ + (binaryfunc)0, /* mp_subscript */ + (objobjargproc)0 /* mp_ass_subscript */ + }; PyFile diff --git a/doc/RELEASE_WALKTHROUGH.rst.txt b/doc/RELEASE_WALKTHROUGH.rst.txt index 6987dd6c1..0a761e350 100644 --- a/doc/RELEASE_WALKTHROUGH.rst.txt +++ b/doc/RELEASE_WALKTHROUGH.rst.txt @@ -38,6 +38,11 @@ to the maintenance branch, and later will be forward ported to master. Finish the Release Note ----------------------- +.. note: + + This has changed now that we use ``towncrier``. See the instructions for + creating the release note in ``doc/release/upcoming_changes/README.rst``. + Fill out the release note ``doc/release/1.14.5-notes.rst`` calling out significant changes. @@ -51,7 +56,7 @@ repository:: $ git checkout maintenance/1.14.x $ git pull upstream maintenance/1.14.x $ git submodule update - $ git clean -xdf > /dev/null + $ git clean -xdfq Edit pavement.py and setup.py as detailed in HOWTO_RELEASE:: @@ -78,7 +83,7 @@ Paver is used to build the source releases. It will create the ``release`` and ``release/installers`` directories and put the ``*.zip`` and ``*.tar.gz`` source releases in the latter. :: - $ cython --version # check that you have the correct cython version + $ python3 -m cython --version # check for correct cython version $ paver sdist # sdist will do a git clean -xdf, so we omit that @@ -227,28 +232,39 @@ add files, using an editable text window and as binary uploads. - Hit the ``{Publish,Update} release`` button at the bottom. -Upload documents to docs.scipy.org ----------------------------------- +Upload documents to numpy.org +----------------------------- This step is only needed for final releases and can be skipped for -pre-releases. You will also need upload permission for the document server, if -you do not have permission ping Pauli Virtanen or Ralf Gommers to generate and -upload the documentation. Otherwise:: +pre-releases. ``make merge-doc`` clones the ``numpy/doc`` repo into +``doc/build/merge`` and updates it with the new documentation:: $ pushd doc $ make dist - $ make upload USERNAME=<yourname> RELEASE=v1.14.5 + $ make merge-doc $ popd -If the release series is a new one, you will need to rebuild and upload the -``docs.scipy.org`` front page:: +If the release series is a new one, you will need to add a new section to the +``doc/build/merge/index.html`` front page just after the "insert here" comment:: + + $ gvim doc/build/merge/index.html +/'insert here' + +Otherwise, only the ``zip`` and ``pdf`` links should be updated with the +new tag name:: - $ cd ../docs.scipy.org - $ gvim index.rst + $ gvim doc/build/merge/index.html +/'tag v1.14' -Note: there is discussion about moving the docs to github. This section will be -updated when/if that happens. +You can "test run" the new documentation in a browser to make sure the links +work:: + $ firefox doc/build/merge/index.html + +Once everything seems satisfactory, commit and upload the changes:: + + $ pushd doc/build/merge + $ git commit -am"Add documentation for v1.14.5" + $ git push + $ popd Announce the release on scipy.org --------------------------------- diff --git a/doc/changelog/1.16.6-changelog.rst b/doc/changelog/1.16.6-changelog.rst deleted file mode 100644 index 62ff46c34..000000000 --- a/doc/changelog/1.16.6-changelog.rst +++ /dev/null @@ -1,36 +0,0 @@ - -Contributors -============ - -A total of 10 people contributed to this release. - -* CakeWithSteak -* Charles Harris -* Chris Burr -* Eric Wieser -* Fernando Saravia -* Lars Grueter -* Matti Picus -* Maxwell Aladago -* Qiming Sun -* Warren Weckesser - -Pull requests merged -==================== - -A total of 14 pull requests were merged for this release. - -* `#14211 <https://github.com/numpy/numpy/pull/14211>`__: BUG: Fix uint-overflow if padding with linear_ramp and negative... -* `#14275 <https://github.com/numpy/numpy/pull/14275>`__: BUG: fixing to allow unpickling of PY3 pickles from PY2 -* `#14340 <https://github.com/numpy/numpy/pull/14340>`__: BUG: Fix misuse of .names and .fields in various places (backport... -* `#14423 <https://github.com/numpy/numpy/pull/14423>`__: BUG: test, fix regression in converting to ctypes. -* `#14434 <https://github.com/numpy/numpy/pull/14434>`__: BUG: Fixed maximum relative error reporting in assert_allclose -* `#14509 <https://github.com/numpy/numpy/pull/14509>`__: BUG: Fix regression in boolean matmul. -* `#14686 <https://github.com/numpy/numpy/pull/14686>`__: BUG: properly define PyArray_DescrCheck -* `#14853 <https://github.com/numpy/numpy/pull/14853>`__: BLD: add 'apt update' to shippable -* `#14854 <https://github.com/numpy/numpy/pull/14854>`__: BUG: Fix _ctypes class circular reference. (#13808) -* `#14856 <https://github.com/numpy/numpy/pull/14856>`__: BUG: Fix `np.einsum` errors on Power9 Linux and z/Linux -* `#14863 <https://github.com/numpy/numpy/pull/14863>`__: BLD: Prevent -flto from optimising long double representation... -* `#14864 <https://github.com/numpy/numpy/pull/14864>`__: BUG: lib: Fix histogram problem with signed integer arrays. -* `#15172 <https://github.com/numpy/numpy/pull/15172>`__: ENH: Backport improvements to testing functions. -* `#15191 <https://github.com/numpy/numpy/pull/15191>`__: REL: Prepare for 1.16.6 release. diff --git a/doc/changelog/1.17.5-changelog.rst b/doc/changelog/1.17.5-changelog.rst deleted file mode 100644 index 7ac758075..000000000 --- a/doc/changelog/1.17.5-changelog.rst +++ /dev/null @@ -1,26 +0,0 @@ - -Contributors -============ - -A total of 6 people contributed to this release. People with a "+" by their -names contributed a patch for the first time. - -* Charles Harris -* Eric Wieser -* Ilhan Polat -* Matti Picus -* Michael Hudson-Doyle -* Ralf Gommers - -Pull requests merged -==================== - -A total of 7 pull requests were merged for this release. - -* `#14593 <https://github.com/numpy/numpy/pull/14593>`__: MAINT: backport Cython API cleanup to 1.17.x, remove docs -* `#14937 <https://github.com/numpy/numpy/pull/14937>`__: BUG: fix integer size confusion in handling array's ndmin argument -* `#14939 <https://github.com/numpy/numpy/pull/14939>`__: BUILD: remove SSE2 flag from numpy.random builds -* `#14993 <https://github.com/numpy/numpy/pull/14993>`__: MAINT: Added Python3.8 branch to dll lib discovery -* `#15038 <https://github.com/numpy/numpy/pull/15038>`__: BUG: Fix refcounting in ufunc object loops -* `#15067 <https://github.com/numpy/numpy/pull/15067>`__: BUG: Exceptions tracebacks are dropped -* `#15175 <https://github.com/numpy/numpy/pull/15175>`__: ENH: Backport improvements to testing functions. diff --git a/doc/changelog/1.18.0-changelog.rst b/doc/changelog/1.18.0-changelog.rst new file mode 100644 index 000000000..b86b3614a --- /dev/null +++ b/doc/changelog/1.18.0-changelog.rst @@ -0,0 +1,533 @@ + +Contributors +============ + +A total of 114 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Abhinav Sagar +* Alex Henrie + +* Alexander Jung + +* Allan Haldane +* Andrea Pattori +* Andrew Liu + +* Anis Ladram + +* Anne Bonner + +* Antoine Dechaume + +* Aryan Naraghi + +* Bastian Eichenberger + +* Brian Wignall + +* Brigitta Sipocz +* CakeWithSteak + +* Charles Harris +* Chris Barker +* Chris Burr + +* Chris Markiewicz + +* Christoph Gohlke +* Christopher Whelan +* Colin Snyder +* Dan Allan +* Daniel Ching +* David Stansby + +* David Zwicker + +* Dieter Werthmüller +* Disconnect3d + +* Dmytro + +* Doug Davis + +* Eric Larson +* Eric Wieser +* Esben Haabendal + +* Eugene Prilepin + +* Felix Divo + +* Gary Gurlaskie +* Gina + +* Giuseppe Cuccu + +* Grzegorz Bokota + +* Guanqun Lu + +* Guilherme Leobas + +* Guillaume Horel +* Géraud Le Falher + +* Hameer Abbasi +* Harmon +* Hiroyuki V. Yamazaki +* Huang, Guangtai + +* Hugo + +* Hyeonguk Ryu + +* Ilhan Polat + +* Isaac Virshup +* Jack J. Woehr + +* Jack Woehr + +* Jackie Leng +* Jaime Fernandez +* Jeff Hale + +* Johann Faouzi + +* Jon Dufresne + +* Joseph Fox-Rabinovitz +* Joseph R. Fox-Rabinovitz + +* João Marcos Gris + +* Justus Magin + +* Jérémie du Boisberranger +* Kai Striega +* Kevin Sheppard +* Kexuan Sun +* Kmol Yuan + +* Kriti Singh +* Larry Bradley + +* Lars Grueter +* Luis Pedro Coelho +* MSeifert04 +* Magdalena Proszewska + +* Manny + +* Mark Harfouche +* Martin Reinecke +* Martin Thoma +* Matt Haberland + +* Matt McCormick + +* Matthias Bussonnier +* Matti Picus +* Max Bolingbroke + +* Maxwell Aladago + +* Michael Hudson-Doyle + +* Oleksandr Pavlyk +* Omar Merghany + +* Pauli Virtanen +* Peter Andreas Entschev +* Peter Bell +* Peter Cock + +* Pradeep Reddy Raamana + +* Qiming Sun + +* Raghuveer Devulapalli +* Ralf Gommers +* Samesh + +* Samesh Lakhotia + +* Sebastian Berg +* Sergei Lebedev +* Seth Troisi + +* Siddhesh Poyarekar + +* Simon + +* Simon Notley + +* Stefan van der Walt +* Stephan Hoyer +* Steve Stagg +* Thomas A Caswell +* Thomas Kluyver +* Tim Hoffmann + +* Tirth Patel + +* Tyler Reddy +* Vladimir Pershin + +* Warren Weckesser +* Yadong Zhang + +* Zieji Pohz + +* Zolisa Bleki + + +Pull requests merged +==================== + +A total of 406 pull requests were merged for this release. + +* `#9301 <https://github.com/numpy/numpy/pull/9301>`__: DOC: added note to docstring of numpy.savez +* `#10151 <https://github.com/numpy/numpy/pull/10151>`__: BUG: Numpy scalar types sometimes have the same name +* `#12129 <https://github.com/numpy/numpy/pull/12129>`__: DOC: Improve axes shift description and example in np.tensordot +* `#12205 <https://github.com/numpy/numpy/pull/12205>`__: MAINT: avoid relying on `np.generic.__name__` in `np.dtype.name` +* `#12284 <https://github.com/numpy/numpy/pull/12284>`__: ENH: supply our version of numpy.pxd, requires cython>=0.29 +* `#12633 <https://github.com/numpy/numpy/pull/12633>`__: BUG: General fixes to f2py reference counts (dereferencing) +* `#12658 <https://github.com/numpy/numpy/pull/12658>`__: BUG: NaT now sorts to ends of arrays +* `#12828 <https://github.com/numpy/numpy/pull/12828>`__: DOC: Updates to nditer usage instructions +* `#13003 <https://github.com/numpy/numpy/pull/13003>`__: BUG: Do not crash on recursive `.dtype` attribute lookup. +* `#13368 <https://github.com/numpy/numpy/pull/13368>`__: ENH: Use AVX for float32 implementation of np.sin & np.cos +* `#13605 <https://github.com/numpy/numpy/pull/13605>`__: DEP: Deprecate silent ignoring of bad data in fromfile/fromstring +* `#13610 <https://github.com/numpy/numpy/pull/13610>`__: ENH: Always produce a consistent shape in the result of `argwhere` +* `#13673 <https://github.com/numpy/numpy/pull/13673>`__: DOC: array(obj, dtype=dt) can downcast +* `#13698 <https://github.com/numpy/numpy/pull/13698>`__: DOC: Document ma.filled behavior with non-scalar fill_value +* `#13710 <https://github.com/numpy/numpy/pull/13710>`__: DOC: Add note to irfft-like functions about the default sizes +* `#13739 <https://github.com/numpy/numpy/pull/13739>`__: BUG: Don't produce undefined behavior for a << b if b >= bitsof(a) +* `#13766 <https://github.com/numpy/numpy/pull/13766>`__: MAINT: Update NEP template. +* `#13794 <https://github.com/numpy/numpy/pull/13794>`__: ENH: random: Add the multivariate hypergeometric distribution. +* `#13799 <https://github.com/numpy/numpy/pull/13799>`__: DOC: Fix unrendered links +* `#13812 <https://github.com/numpy/numpy/pull/13812>`__: MAINT: Rewrite Floyd algorithm +* `#13825 <https://github.com/numpy/numpy/pull/13825>`__: DOC: Add missing macros to C-API documentation +* `#13829 <https://github.com/numpy/numpy/pull/13829>`__: ENH: Add axis argument to random.permutation and random.shuffle +* `#13847 <https://github.com/numpy/numpy/pull/13847>`__: DOC: Adds documentation of functions exposed in numpy namespace +* `#13860 <https://github.com/numpy/numpy/pull/13860>`__: BUG: Refcount fixes +* `#13871 <https://github.com/numpy/numpy/pull/13871>`__: MAINT: Ensure array_dealloc does not modify refcount of self +* `#13874 <https://github.com/numpy/numpy/pull/13874>`__: MAINT: Prepare master for 1.18.0 development. +* `#13876 <https://github.com/numpy/numpy/pull/13876>`__: MAINT,BUG,DOC: Fix errors in _add_newdocs +* `#13880 <https://github.com/numpy/numpy/pull/13880>`__: MAINT: Remove an unnessary backslash between two string literals +* `#13881 <https://github.com/numpy/numpy/pull/13881>`__: MAINT: Update pavement to use python3 in shell commands. +* `#13882 <https://github.com/numpy/numpy/pull/13882>`__: MAINT: Remove unnecessary backslashes (and replace others by... +* `#13883 <https://github.com/numpy/numpy/pull/13883>`__: MAINT: Replace integers in places where booleans are expected +* `#13884 <https://github.com/numpy/numpy/pull/13884>`__: DOC: Add missing parameter description for keepdims in MaskedArray +* `#13885 <https://github.com/numpy/numpy/pull/13885>`__: ENH: use AVX for float32 and float64 implementation of sqrt,... +* `#13886 <https://github.com/numpy/numpy/pull/13886>`__: DOC: reformat top-level release index +* `#13892 <https://github.com/numpy/numpy/pull/13892>`__: DOC : Refactor Array API documentation -- Array Structure and... +* `#13895 <https://github.com/numpy/numpy/pull/13895>`__: DOC: Fix typo in "make_mask" documentation +* `#13896 <https://github.com/numpy/numpy/pull/13896>`__: MAINT: Delete unused _aliased_types.py +* `#13901 <https://github.com/numpy/numpy/pull/13901>`__: BLD: Remove Trusty dist in Travis CI build +* `#13907 <https://github.com/numpy/numpy/pull/13907>`__: BUG: Handle weird bytestrings in dtype() +* `#13908 <https://github.com/numpy/numpy/pull/13908>`__: ENH: use towncrier to build the release note +* `#13913 <https://github.com/numpy/numpy/pull/13913>`__: ENH: improve error message for ragged-array creation failure +* `#13914 <https://github.com/numpy/numpy/pull/13914>`__: DOC: Update the description of byteswap +* `#13916 <https://github.com/numpy/numpy/pull/13916>`__: BUG: i0 Bessel function regression on array-likes supporting... +* `#13920 <https://github.com/numpy/numpy/pull/13920>`__: ENH, BUILD: refactor all OpenBLAS downloads into a single, testable... +* `#13922 <https://github.com/numpy/numpy/pull/13922>`__: MAINT: Remove unnecessary parenthesis in numpy.ma.core +* `#13925 <https://github.com/numpy/numpy/pull/13925>`__: MAINT: Fix wrong spelling of ufunc +* `#13926 <https://github.com/numpy/numpy/pull/13926>`__: DOC: Remove explicit .next method calls with built-in next function... +* `#13928 <https://github.com/numpy/numpy/pull/13928>`__: DOC: Don't override MaskedArray.view documentation with the one... +* `#13930 <https://github.com/numpy/numpy/pull/13930>`__: BUG: Fix incorrect GIL release in array.nonzero +* `#13935 <https://github.com/numpy/numpy/pull/13935>`__: MAINT: Warn if `_add_newdocs.py` is used to add docstrings to... +* `#13943 <https://github.com/numpy/numpy/pull/13943>`__: MAINT: Revert #13876, "MAINT,BUG,DOC: Fix errors in _add_newdocs" +* `#13944 <https://github.com/numpy/numpy/pull/13944>`__: MAINT,BUG,DOC: Fix errors in _add_newdocs +* `#13945 <https://github.com/numpy/numpy/pull/13945>`__: DOC, MAINT: emphasize random API changes, remove Generator.randint +* `#13946 <https://github.com/numpy/numpy/pull/13946>`__: DOC: Add a numpy-doc docstring to add_newdoc +* `#13947 <https://github.com/numpy/numpy/pull/13947>`__: DOC: Fix rst rendering in data types +* `#13948 <https://github.com/numpy/numpy/pull/13948>`__: DOC:Update the description of set_printoptions in quickstart... +* `#13950 <https://github.com/numpy/numpy/pull/13950>`__: Fixing failure on Python 2.7 on Windows 7 +* `#13952 <https://github.com/numpy/numpy/pull/13952>`__: Fix a typo related to the range of indices +* `#13959 <https://github.com/numpy/numpy/pull/13959>`__: DOC: add space between words across lines +* `#13964 <https://github.com/numpy/numpy/pull/13964>`__: BUG, DOC: add new recfunctions to `__all__` +* `#13967 <https://github.com/numpy/numpy/pull/13967>`__: DOC: Change (old) range() to np.arange() +* `#13968 <https://github.com/numpy/numpy/pull/13968>`__: DOC: improve np.sort docstring +* `#13970 <https://github.com/numpy/numpy/pull/13970>`__: DOC: spellcheck numpy/doc/broadcasting.py +* `#13976 <https://github.com/numpy/numpy/pull/13976>`__: MAINT, TST: remove test-installed-numpy.py +* `#13979 <https://github.com/numpy/numpy/pull/13979>`__: DOC: Document array_function at a higher level. +* `#13985 <https://github.com/numpy/numpy/pull/13985>`__: DOC: show workaround for backward compatibility +* `#13988 <https://github.com/numpy/numpy/pull/13988>`__: DOC: Add a call for contribution paragraph to the readme +* `#13989 <https://github.com/numpy/numpy/pull/13989>`__: BUG: Missing warnings import in polyutils +* `#13990 <https://github.com/numpy/numpy/pull/13990>`__: BUILD: adapt "make version-check" to "make dist" +* `#13991 <https://github.com/numpy/numpy/pull/13991>`__: DOC: emphasize need for matching numpy, git versions +* `#14002 <https://github.com/numpy/numpy/pull/14002>`__: TST, MAINT, BUG: expand OpenBLAS version checking +* `#14004 <https://github.com/numpy/numpy/pull/14004>`__: ENH: Chain exception for typed item assignment +* `#14005 <https://github.com/numpy/numpy/pull/14005>`__: MAINT: Fix spelling error in npy_tempita kwarg +* `#14010 <https://github.com/numpy/numpy/pull/14010>`__: DOC: Array API : Directory restructure and code cleanup +* `#14011 <https://github.com/numpy/numpy/pull/14011>`__: [DOC] Remove unused/deprecated functions +* `#14022 <https://github.com/numpy/numpy/pull/14022>`__: Update system_info.py +* `#14025 <https://github.com/numpy/numpy/pull/14025>`__: DOC:Link between the two indexing documentation pages +* `#14026 <https://github.com/numpy/numpy/pull/14026>`__: DOC: Update NumFOCUS subcommittee replacing Nathaniel with Sebastian +* `#14027 <https://github.com/numpy/numpy/pull/14027>`__: DOC: update "Contributing to NumPy" with more activities/roles +* `#14028 <https://github.com/numpy/numpy/pull/14028>`__: DOC: Improve quickstart documentation of new random Generator +* `#14030 <https://github.com/numpy/numpy/pull/14030>`__: DEP: Speed up WarnOnWrite deprecation in buffer interface +* `#14032 <https://github.com/numpy/numpy/pull/14032>`__: NEP: numpy.org website redesign +* `#14035 <https://github.com/numpy/numpy/pull/14035>`__: DOC: Fix docstring of numpy.allclose regarding NaNs +* `#14036 <https://github.com/numpy/numpy/pull/14036>`__: DEP: Raise warnings for deprecated functions PyArray_As1D, PyArray_As2D +* `#14039 <https://github.com/numpy/numpy/pull/14039>`__: DEP: Remove np.rank which has been deprecated for more than 5... +* `#14048 <https://github.com/numpy/numpy/pull/14048>`__: BUG, TEST: Adding validation test suite to validate float32 exp +* `#14051 <https://github.com/numpy/numpy/pull/14051>`__: ENH,DEP: Allow multiple axes in expand_dims +* `#14053 <https://github.com/numpy/numpy/pull/14053>`__: ENH: add pyproject.toml +* `#14060 <https://github.com/numpy/numpy/pull/14060>`__: DOC: Update cversions.py links and wording +* `#14062 <https://github.com/numpy/numpy/pull/14062>`__: DOC, BUILD: cleanups and fix (again) 'make dist' +* `#14063 <https://github.com/numpy/numpy/pull/14063>`__: BUG: Fix file-like object check when saving arrays +* `#14064 <https://github.com/numpy/numpy/pull/14064>`__: DOC: Resolve bad references in Sphinx warnings +* `#14068 <https://github.com/numpy/numpy/pull/14068>`__: MAINT: bump ARMv8 / POWER8 OpenBLAS in CI +* `#14069 <https://github.com/numpy/numpy/pull/14069>`__: DOC: Emphasize the need to run tests when building from source +* `#14070 <https://github.com/numpy/numpy/pull/14070>`__: DOC:Add example to clarify "numpy.save" behavior on already open... +* `#14072 <https://github.com/numpy/numpy/pull/14072>`__: DEP: Deprecate full and economic modes for linalg.qr +* `#14073 <https://github.com/numpy/numpy/pull/14073>`__: DOC: Doc release +* `#14074 <https://github.com/numpy/numpy/pull/14074>`__: BUG: fix build issue on icc 2016 +* `#14076 <https://github.com/numpy/numpy/pull/14076>`__: TST: Add 3.8-dev to travisCI testing. +* `#14085 <https://github.com/numpy/numpy/pull/14085>`__: DOC: Add blank line above doctest for intersect1d +* `#14086 <https://github.com/numpy/numpy/pull/14086>`__: ENH: Propose standard policy for dropping support of old Python... +* `#14089 <https://github.com/numpy/numpy/pull/14089>`__: DOC: Use `pip install .` where possible instead of calling setup.py +* `#14091 <https://github.com/numpy/numpy/pull/14091>`__: MAINT: adjustments to test_ufunc_noncontigous +* `#14092 <https://github.com/numpy/numpy/pull/14092>`__: MAINT: Improve NEP template +* `#14096 <https://github.com/numpy/numpy/pull/14096>`__: DOC: fix documentation of i and j for tri. +* `#14097 <https://github.com/numpy/numpy/pull/14097>`__: MAINT: Lazy import testing on python >=3.7 +* `#14100 <https://github.com/numpy/numpy/pull/14100>`__: DEP: Deprecate PyArray_FromDimsAndDataAndDescr, PyArray_FromDims +* `#14101 <https://github.com/numpy/numpy/pull/14101>`__: MAINT: Clearer error message while padding with stat_length=0 +* `#14106 <https://github.com/numpy/numpy/pull/14106>`__: MAINT: remove duplicate variable assignments +* `#14108 <https://github.com/numpy/numpy/pull/14108>`__: BUG: initialize variable that is passed by pointer +* `#14110 <https://github.com/numpy/numpy/pull/14110>`__: DOC: fix typo in c-api/array.rst doc +* `#14121 <https://github.com/numpy/numpy/pull/14121>`__: BUG: Add gcd/lcm definitions to npy_math.h +* `#14122 <https://github.com/numpy/numpy/pull/14122>`__: MAINT: Mark umath accuracy test xfail. +* `#14124 <https://github.com/numpy/numpy/pull/14124>`__: MAINT: Use equality instead of identity check with literal +* `#14130 <https://github.com/numpy/numpy/pull/14130>`__: MAINT: Fix small typo in quickstart docs +* `#14134 <https://github.com/numpy/numpy/pull/14134>`__: DOC, MAINT: Update master after 1.17.0 release. +* `#14141 <https://github.com/numpy/numpy/pull/14141>`__: ENH: add c-imported modules for freeze analysis in np.random +* `#14143 <https://github.com/numpy/numpy/pull/14143>`__: BUG: Fix DeprecationWarning in python 3.8 +* `#14144 <https://github.com/numpy/numpy/pull/14144>`__: BUG: Remove stray print that causes a SystemError on python 3.7... +* `#14145 <https://github.com/numpy/numpy/pull/14145>`__: BUG: Remove the broken clip wrapper +* `#14152 <https://github.com/numpy/numpy/pull/14152>`__: BUG: avx2_scalef_ps must be static +* `#14153 <https://github.com/numpy/numpy/pull/14153>`__: TST: Allow fuss in testing strided/non-strided exp/log loops +* `#14170 <https://github.com/numpy/numpy/pull/14170>`__: NEP: Proposal for __duckarray__ protocol +* `#14171 <https://github.com/numpy/numpy/pull/14171>`__: BUG: Make advanced indexing result on read-only subclass writeable +* `#14178 <https://github.com/numpy/numpy/pull/14178>`__: TST: Clean up of test_pocketfft.py +* `#14181 <https://github.com/numpy/numpy/pull/14181>`__: DEP: Deprecate np.alen +* `#14185 <https://github.com/numpy/numpy/pull/14185>`__: MAINT: Workaround for Intel compiler bug leading to failing test +* `#14190 <https://github.com/numpy/numpy/pull/14190>`__: DOC: Fix hermitian argument docs in `svd` +* `#14195 <https://github.com/numpy/numpy/pull/14195>`__: MAINT: Fix a docstring typo. +* `#14196 <https://github.com/numpy/numpy/pull/14196>`__: DOC: Fix links in `/.github/CONTRIBUTING.md`. +* `#14197 <https://github.com/numpy/numpy/pull/14197>`__: ENH: Multivariate normal speedups +* `#14203 <https://github.com/numpy/numpy/pull/14203>`__: MAINT: Improve mismatch message of np.testing.assert_array_equal +* `#14204 <https://github.com/numpy/numpy/pull/14204>`__: DOC,MAINT: Move towncrier files and fixup categories +* `#14207 <https://github.com/numpy/numpy/pull/14207>`__: BUG: Fixed default BitGenerator name +* `#14209 <https://github.com/numpy/numpy/pull/14209>`__: BUG: Fix uint-overflow if padding with linear_ramp and negative... +* `#14216 <https://github.com/numpy/numpy/pull/14216>`__: ENH: Enable huge pages in all Linux builds +* `#14217 <https://github.com/numpy/numpy/pull/14217>`__: BUG: Fix leak in the f2py-generated module init and `PyMem_Del`... +* `#14219 <https://github.com/numpy/numpy/pull/14219>`__: DOC: new nan_to_num keywords are from 1.17 onwards +* `#14223 <https://github.com/numpy/numpy/pull/14223>`__: TST: Add tests for deprecated C functions (PyArray_As1D, PyArray_As1D) +* `#14224 <https://github.com/numpy/numpy/pull/14224>`__: DOC: mention `take_along_axis` in `choose` +* `#14227 <https://github.com/numpy/numpy/pull/14227>`__: ENH: Parse complex number from string +* `#14231 <https://github.com/numpy/numpy/pull/14231>`__: DOC: update or remove outdated sourceforge links +* `#14234 <https://github.com/numpy/numpy/pull/14234>`__: MAINT: Better error message for norm +* `#14235 <https://github.com/numpy/numpy/pull/14235>`__: DOC: add backlinks to numpy.org +* `#14240 <https://github.com/numpy/numpy/pull/14240>`__: BUG: Don't fail when lexsorting some empty arrays. +* `#14241 <https://github.com/numpy/numpy/pull/14241>`__: BUG: Fix segfault in `random.permutation(x)` when x is a string. +* `#14245 <https://github.com/numpy/numpy/pull/14245>`__: Doc: fix a typo in NEP21 +* `#14249 <https://github.com/numpy/numpy/pull/14249>`__: DOC: set status of NEP 28 (website redesign) to Accepted +* `#14250 <https://github.com/numpy/numpy/pull/14250>`__: BLD: MAINT: change default behavior of build flag appending. +* `#14252 <https://github.com/numpy/numpy/pull/14252>`__: BUG: Fixes StopIteration error from 'np.genfromtext' for empty... +* `#14255 <https://github.com/numpy/numpy/pull/14255>`__: BUG: fix inconsistent axes ordering for axis in function `unique` +* `#14256 <https://github.com/numpy/numpy/pull/14256>`__: DEP: Deprecate load/dump functions in favour of pickle methods +* `#14257 <https://github.com/numpy/numpy/pull/14257>`__: MAINT: Update NEP-30 +* `#14259 <https://github.com/numpy/numpy/pull/14259>`__: DEP: Deprecate arrayprint formatting functions +* `#14266 <https://github.com/numpy/numpy/pull/14266>`__: DOC: remove scipy.org from the breadcrumb formattiong +* `#14270 <https://github.com/numpy/numpy/pull/14270>`__: BUG: Fix formatting error in exception message +* `#14272 <https://github.com/numpy/numpy/pull/14272>`__: DOC: Address typos in dispatch docs +* `#14279 <https://github.com/numpy/numpy/pull/14279>`__: BUG: Fix ZeroDivisionError for zero length arrays in pocketfft. +* `#14290 <https://github.com/numpy/numpy/pull/14290>`__: BUG: Fix misuse of .names and .fields in various places +* `#14291 <https://github.com/numpy/numpy/pull/14291>`__: TST, BUG: Use python3.6-dbg. +* `#14295 <https://github.com/numpy/numpy/pull/14295>`__: BUG: core: Handle large negative np.int64 args in binary_repr. +* `#14298 <https://github.com/numpy/numpy/pull/14298>`__: BUG: Fix numpy.random bug in platform detection +* `#14303 <https://github.com/numpy/numpy/pull/14303>`__: MAINT: random: Match type of SeedSequence.pool_size to DEFAULT_POOL_SIZE. +* `#14310 <https://github.com/numpy/numpy/pull/14310>`__: Bug: Fix behavior of structured_to_unstructured on non-trivial... +* `#14311 <https://github.com/numpy/numpy/pull/14311>`__: DOC: add two commas, move one word +* `#14313 <https://github.com/numpy/numpy/pull/14313>`__: DOC: Clarify rules about broadcasting when empty arrays are involved. +* `#14321 <https://github.com/numpy/numpy/pull/14321>`__: TST, MAINT: bump to OpenBLAS 0.3.7 stable +* `#14325 <https://github.com/numpy/numpy/pull/14325>`__: DEP: numpy.testing.rand +* `#14335 <https://github.com/numpy/numpy/pull/14335>`__: DEP: Deprecate class `SafeEval` +* `#14341 <https://github.com/numpy/numpy/pull/14341>`__: BUG: revert detecting and raising error on ragged arrays +* `#14342 <https://github.com/numpy/numpy/pull/14342>`__: DOC: Improve documentation of `isscalar`. +* `#14349 <https://github.com/numpy/numpy/pull/14349>`__: MAINT: Fix bloated mismatch error percentage in array comparisons. +* `#14351 <https://github.com/numpy/numpy/pull/14351>`__: DOC: Fix a minor typo in dispatch documentation. +* `#14352 <https://github.com/numpy/numpy/pull/14352>`__: MAINT: Remove redundant deprecation checks +* `#14353 <https://github.com/numpy/numpy/pull/14353>`__: MAINT: polynomial: Add an N-d vander implementation used under... +* `#14355 <https://github.com/numpy/numpy/pull/14355>`__: DOC: clarify that PytestTester is non-public +* `#14356 <https://github.com/numpy/numpy/pull/14356>`__: DOC: support and require sphinx>=2.2 +* `#14360 <https://github.com/numpy/numpy/pull/14360>`__: DOC: random: fix doc linking, was referencing private submodules. +* `#14364 <https://github.com/numpy/numpy/pull/14364>`__: MAINT: Fixes for prospective Python 3.10 and 4.0 +* `#14365 <https://github.com/numpy/numpy/pull/14365>`__: DOC: lib: Add more explanation of the weighted average calculation. +* `#14368 <https://github.com/numpy/numpy/pull/14368>`__: MAINT: Avoid BytesWarning in PyArray_DescrConverter() +* `#14369 <https://github.com/numpy/numpy/pull/14369>`__: MAINT: Post NumPy 1.17.1 update. +* `#14370 <https://github.com/numpy/numpy/pull/14370>`__: DOC: Fixed dtype docs for var, nanvar. +* `#14372 <https://github.com/numpy/numpy/pull/14372>`__: DOC: Document project as Python 3 only with a trove classifier +* `#14378 <https://github.com/numpy/numpy/pull/14378>`__: BUILD: move all test dependencies to ./test_requirements.txt +* `#14381 <https://github.com/numpy/numpy/pull/14381>`__: BUG: lib: Fix histogram problem with signed integer arrays. +* `#14385 <https://github.com/numpy/numpy/pull/14385>`__: REL: Update master after NumPy 1.16.5 release. +* `#14387 <https://github.com/numpy/numpy/pull/14387>`__: BUG: test, fix regression in converting to ctypes +* `#14389 <https://github.com/numpy/numpy/pull/14389>`__: NEP: Add initial draft of NEP-31: Context-local and global overrides... +* `#14390 <https://github.com/numpy/numpy/pull/14390>`__: DOC: document numpy/doc update process +* `#14392 <https://github.com/numpy/numpy/pull/14392>`__: DOC: update np.around docstring with note about floating-point... +* `#14393 <https://github.com/numpy/numpy/pull/14393>`__: BUG: view with fieldless dtype should raise if itemsize != 0 +* `#14395 <https://github.com/numpy/numpy/pull/14395>`__: DOC: fix issue with __new__ usage in subclassing doc. +* `#14398 <https://github.com/numpy/numpy/pull/14398>`__: DOC: Fix release notes table of contents +* `#14399 <https://github.com/numpy/numpy/pull/14399>`__: NEP 32: Remove the financial functions from NumPy +* `#14404 <https://github.com/numpy/numpy/pull/14404>`__: BLD: Update RELEASE_WALKTHROUGH and cythonize. +* `#14407 <https://github.com/numpy/numpy/pull/14407>`__: Bump pytest from 5.1.1 to 5.1.2 +* `#14408 <https://github.com/numpy/numpy/pull/14408>`__: TST: Remove build job since we now use Dependabot +* `#14410 <https://github.com/numpy/numpy/pull/14410>`__: BLD: Only allow using Cython module when cythonizing. +* `#14411 <https://github.com/numpy/numpy/pull/14411>`__: TST: Add dependabot config file. +* `#14416 <https://github.com/numpy/numpy/pull/14416>`__: BUG: Fix format statement associated with AttributeError. +* `#14417 <https://github.com/numpy/numpy/pull/14417>`__: BUG: Fix aradixsort indirect indexing. +* `#14426 <https://github.com/numpy/numpy/pull/14426>`__: DOC: add the reference to 'printoptions' +* `#14429 <https://github.com/numpy/numpy/pull/14429>`__: BUG: Do not show Override module in private error classes. +* `#14444 <https://github.com/numpy/numpy/pull/14444>`__: DOC: Make implementation bullet points consistent in NEP 29 +* `#14447 <https://github.com/numpy/numpy/pull/14447>`__: MAINT: Clarify policy language in NEP-29. +* `#14448 <https://github.com/numpy/numpy/pull/14448>`__: REL: Update master after 1.17.2 release. +* `#14452 <https://github.com/numpy/numpy/pull/14452>`__: MAINT: clean up pocketfft modules inside numpy.fft namespace +* `#14453 <https://github.com/numpy/numpy/pull/14453>`__: BLD: remove generated Cython files from sdist +* `#14454 <https://github.com/numpy/numpy/pull/14454>`__: MAINT: add test to prevent new public-looking modules being added +* `#14458 <https://github.com/numpy/numpy/pull/14458>`__: BUG: random.hypergeometic assumes npy_long is npy_int64, hangs... +* `#14459 <https://github.com/numpy/numpy/pull/14459>`__: ENH: Print the amount of memory that would be used by a failed... +* `#14460 <https://github.com/numpy/numpy/pull/14460>`__: MAINT: use test_requirements.txt in tox and shippable, ship it... +* `#14464 <https://github.com/numpy/numpy/pull/14464>`__: BUG: add a specialized loop for boolean matmul +* `#14469 <https://github.com/numpy/numpy/pull/14469>`__: BUG: Fix _ctypes class circular reference. (#13808) +* `#14472 <https://github.com/numpy/numpy/pull/14472>`__: BUG: core: Fix the str function of the rational dtype. +* `#14475 <https://github.com/numpy/numpy/pull/14475>`__: DOC: add timedelta64 signature +* `#14477 <https://github.com/numpy/numpy/pull/14477>`__: MAINT: Extract raising of MemoryError to a helper function +* `#14483 <https://github.com/numpy/numpy/pull/14483>`__: BUG,MAINT: Some fixes and minor cleanup based on clang analysis +* `#14484 <https://github.com/numpy/numpy/pull/14484>`__: MAINT: Add `NPY_UNUSED` and `const` qualified suggested by clang +* `#14485 <https://github.com/numpy/numpy/pull/14485>`__: MAINT: Silence integer comparison build warnings in assert statements +* `#14486 <https://github.com/numpy/numpy/pull/14486>`__: MAINT: distutils: Add newline at the end of printed warnings. +* `#14490 <https://github.com/numpy/numpy/pull/14490>`__: BUG: random: Revert gh-14458 and refix gh-14557. +* `#14493 <https://github.com/numpy/numpy/pull/14493>`__: DOC: Fix reference NPY_ARRAY_OWNDATA instead of NPY_OWNDATA. +* `#14495 <https://github.com/numpy/numpy/pull/14495>`__: ENH: Allow NPY_PKG_CONFIG_PATH environment variable override +* `#14498 <https://github.com/numpy/numpy/pull/14498>`__: MAINT: remove the entropy c-extension module +* `#14499 <https://github.com/numpy/numpy/pull/14499>`__: DOC: Add backslashes so PyUFunc_FromFuncAndDataAndSignatureAndIdentity... +* `#14500 <https://github.com/numpy/numpy/pull/14500>`__: DOC: Fix a minor typo in changelog readme +* `#14501 <https://github.com/numpy/numpy/pull/14501>`__: BUG: Fix randint when range is 2**32 +* `#14503 <https://github.com/numpy/numpy/pull/14503>`__: DOC: tweak np.round docstring to clarify floating-point error +* `#14508 <https://github.com/numpy/numpy/pull/14508>`__: DOC: Add warning to NPV function +* `#14510 <https://github.com/numpy/numpy/pull/14510>`__: API: Do not return None from recfunctions.drop_fields +* `#14511 <https://github.com/numpy/numpy/pull/14511>`__: BUG: Fix flatten_dtype so that nested 0-field structs are flattened... +* `#14514 <https://github.com/numpy/numpy/pull/14514>`__: DOC: Build release notes during CircleCI step +* `#14518 <https://github.com/numpy/numpy/pull/14518>`__: BUILD: Hide platform configuration probe behind --debug-configure +* `#14520 <https://github.com/numpy/numpy/pull/14520>`__: Mention that split() returns views into the original array +* `#14521 <https://github.com/numpy/numpy/pull/14521>`__: MAINT: Simplify lookfor function +* `#14523 <https://github.com/numpy/numpy/pull/14523>`__: MAINT: random: Remove a few duplicated C function prototypes. +* `#14525 <https://github.com/numpy/numpy/pull/14525>`__: BUILD, MAINT: run tests with verbose for PyPY, also do not leak... +* `#14526 <https://github.com/numpy/numpy/pull/14526>`__: BUG: fix release snippet failures caught only after merging +* `#14527 <https://github.com/numpy/numpy/pull/14527>`__: BLD: add warn-error option, adds -Werror to compiler +* `#14531 <https://github.com/numpy/numpy/pull/14531>`__: BUG: random: Create a legacy implementation of random.binomial. +* `#14534 <https://github.com/numpy/numpy/pull/14534>`__: MAINT: remove unused functions, rearrange headers (from CC=clang) +* `#14535 <https://github.com/numpy/numpy/pull/14535>`__: DOC: Fix a bit of code in 'Beyond the Basics' C API user guide. +* `#14536 <https://github.com/numpy/numpy/pull/14536>`__: MAINT: Cleanup old_defines in DOC +* `#14540 <https://github.com/numpy/numpy/pull/14540>`__: DOC: Added missing versionadded to diff(prepend) +* `#14543 <https://github.com/numpy/numpy/pull/14543>`__: BUG: Avoid ctypes in Generators +* `#14545 <https://github.com/numpy/numpy/pull/14545>`__: Changing ImportWarning to DeprecationWarning +* `#14548 <https://github.com/numpy/numpy/pull/14548>`__: MAINT: handle case where GIT_VERSION is empty string +* `#14554 <https://github.com/numpy/numpy/pull/14554>`__: MAINT: core: Remove duplicated inner loop ee->e from log, exp,... +* `#14555 <https://github.com/numpy/numpy/pull/14555>`__: DOC: clarify input types in basics.io.genfromtxt.rst +* `#14557 <https://github.com/numpy/numpy/pull/14557>`__: DOC: remove note about Pocketfft license file (non-existing here). +* `#14558 <https://github.com/numpy/numpy/pull/14558>`__: DOC: Fix code that generates the table in the 'Casting Rules'... +* `#14562 <https://github.com/numpy/numpy/pull/14562>`__: MAINT: don't install partial numpy.random C/Cython API. +* `#14564 <https://github.com/numpy/numpy/pull/14564>`__: TST: ensure coercion tables aren't printed on failing public... +* `#14567 <https://github.com/numpy/numpy/pull/14567>`__: DEP: remove deprecated (and private) numpy.testing submodules. +* `#14568 <https://github.com/numpy/numpy/pull/14568>`__: BLD, DOC: fix gh-14518, add release note +* `#14570 <https://github.com/numpy/numpy/pull/14570>`__: BUG: importing build_src breaks setuptools monkeypatch for msvc14 +* `#14572 <https://github.com/numpy/numpy/pull/14572>`__: DOC: Note runtests.py `-- -s` method to use pytests `-s` +* `#14573 <https://github.com/numpy/numpy/pull/14573>`__: DOC: update submodule docstrings, remove info.py files +* `#14576 <https://github.com/numpy/numpy/pull/14576>`__: DOC: Document the NPY_SCALARKIND values as C variables. +* `#14582 <https://github.com/numpy/numpy/pull/14582>`__: MAINT: Bump pytest from 5.1.2 to 5.1.3 +* `#14583 <https://github.com/numpy/numpy/pull/14583>`__: DEP: remove deprecated select behaviour +* `#14585 <https://github.com/numpy/numpy/pull/14585>`__: BUG: Add missing check for 0-sized array in ravel_multi_index +* `#14586 <https://github.com/numpy/numpy/pull/14586>`__: BUG: dtype refcount cleanups +* `#14587 <https://github.com/numpy/numpy/pull/14587>`__: DOC: Fix a minor typo in changelog entry +* `#14592 <https://github.com/numpy/numpy/pull/14592>`__: MAINT: Fix typo: remoge → remove +* `#14595 <https://github.com/numpy/numpy/pull/14595>`__: DOC: Change the promotion table checkmark to 'Y'. +* `#14596 <https://github.com/numpy/numpy/pull/14596>`__: DEP: Complete deprecation of invalid array/memory order +* `#14598 <https://github.com/numpy/numpy/pull/14598>`__: DOC: Add to doc that interp cannot contain NaN +* `#14600 <https://github.com/numpy/numpy/pull/14600>`__: NEP: Accept NEP 32. +* `#14601 <https://github.com/numpy/numpy/pull/14601>`__: NEP: Fix discrepancies in NEPs +* `#14603 <https://github.com/numpy/numpy/pull/14603>`__: NEP: Only list "Active" NEPs under "Meta-NEPs" +* `#14604 <https://github.com/numpy/numpy/pull/14604>`__: API: restructure and document numpy.random C-API +* `#14605 <https://github.com/numpy/numpy/pull/14605>`__: BUG: properly define PyArray_DescrCheck{,Exact} +* `#14607 <https://github.com/numpy/numpy/pull/14607>`__: MAINT: Remove duplicate files from .gitignore +* `#14608 <https://github.com/numpy/numpy/pull/14608>`__: API: rearrange the cython files in numpy.random +* `#14614 <https://github.com/numpy/numpy/pull/14614>`__: MAINT: Bump pytest from 5.1.3 to 5.2.0 +* `#14615 <https://github.com/numpy/numpy/pull/14615>`__: MAINT: Add "MAINT" tag to dependabot commit msg +* `#14616 <https://github.com/numpy/numpy/pull/14616>`__: DOC: Updated sphinx directive formatting +* `#14620 <https://github.com/numpy/numpy/pull/14620>`__: DEP: Finish deprecation of non-integer `num` in linspace +* `#14621 <https://github.com/numpy/numpy/pull/14621>`__: DOC: s/OR/AND/ in np.logical_and docstring +* `#14623 <https://github.com/numpy/numpy/pull/14623>`__: DOC: misleading np.sinc() documentation +* `#14629 <https://github.com/numpy/numpy/pull/14629>`__: DOC: clarify residual in np.polyfit +* `#14630 <https://github.com/numpy/numpy/pull/14630>`__: BUILD: change to build_src --verbose-cfg, runtests.py --debug-info +* `#14631 <https://github.com/numpy/numpy/pull/14631>`__: BUG: always free clean_sep +* `#14634 <https://github.com/numpy/numpy/pull/14634>`__: DOC: Create `class Extension` docstring and add it to documentation. +* `#14636 <https://github.com/numpy/numpy/pull/14636>`__: DOC: add `printoptions` as a context manager to `set_printoptions` +* `#14639 <https://github.com/numpy/numpy/pull/14639>`__: DOC: Fix typo in NEP 29 +* `#14643 <https://github.com/numpy/numpy/pull/14643>`__: MAINT: Use scalar math power function directly +* `#14649 <https://github.com/numpy/numpy/pull/14649>`__: DOC: Add IPython to dependencies needed to build docs. +* `#14652 <https://github.com/numpy/numpy/pull/14652>`__: MAINT: Bump pytest-cov from 2.7.1 to 2.8.1 +* `#14653 <https://github.com/numpy/numpy/pull/14653>`__: MAINT: Bump pytest from 5.2.0 to 5.2.1 +* `#14654 <https://github.com/numpy/numpy/pull/14654>`__: MAINT: Bump pytz from 2019.2 to 2019.3 +* `#14656 <https://github.com/numpy/numpy/pull/14656>`__: MAINT: Use `extract_unit` throughout datetime +* `#14657 <https://github.com/numpy/numpy/pull/14657>`__: BUG: fix fromfile behavior when reading sub-array dtypes +* `#14662 <https://github.com/numpy/numpy/pull/14662>`__: BUG: random: Use correct length when axis is given to shuffle. +* `#14669 <https://github.com/numpy/numpy/pull/14669>`__: BUG: Do not rely on undefined behaviour to cast from float to... +* `#14674 <https://github.com/numpy/numpy/pull/14674>`__: NEP: add default-dtype-object-deprecation nep 34 +* `#14681 <https://github.com/numpy/numpy/pull/14681>`__: MAINT: Remove unused boolean negative/subtract loops +* `#14682 <https://github.com/numpy/numpy/pull/14682>`__: DEP: ufunc `out` argument must be a tuple for multiple outputs +* `#14693 <https://github.com/numpy/numpy/pull/14693>`__: BUG: Fix `np.einsum` errors on Power9 Linux and z/Linux +* `#14696 <https://github.com/numpy/numpy/pull/14696>`__: DOC: Note release notes process changes on devdocs start page +* `#14699 <https://github.com/numpy/numpy/pull/14699>`__: Doc warnings +* `#14705 <https://github.com/numpy/numpy/pull/14705>`__: DOC: Switch Markdown link to RST in NEP 29 +* `#14709 <https://github.com/numpy/numpy/pull/14709>`__: TST: Divide Azure CI Pipelines into stages. +* `#14710 <https://github.com/numpy/numpy/pull/14710>`__: DEP: Finish the out kwarg deprecation for ufunc calls +* `#14711 <https://github.com/numpy/numpy/pull/14711>`__: DOC: Removing mentions of appveyor +* `#14714 <https://github.com/numpy/numpy/pull/14714>`__: BUG: Default start to 0 for timedelta arange +* `#14717 <https://github.com/numpy/numpy/pull/14717>`__: API: NaT (arg)min/max behavior +* `#14718 <https://github.com/numpy/numpy/pull/14718>`__: API: Forbid Q<->m safe casting +* `#14720 <https://github.com/numpy/numpy/pull/14720>`__: DEP: deprecate financial functions. +* `#14721 <https://github.com/numpy/numpy/pull/14721>`__: DOC: Move newsfragment to correct folder +* `#14723 <https://github.com/numpy/numpy/pull/14723>`__: DOC: cleaning up examples in maskedarray.generic +* `#14725 <https://github.com/numpy/numpy/pull/14725>`__: MAINT: umath: Change error message for unsupported bool subtraction. +* `#14730 <https://github.com/numpy/numpy/pull/14730>`__: ENH: Add complex number support for fromfile +* `#14732 <https://github.com/numpy/numpy/pull/14732>`__: TST: run refguide-check on rst files in doc/* +* `#14734 <https://github.com/numpy/numpy/pull/14734>`__: DOC: Edit NEP procedure for better discussion +* `#14736 <https://github.com/numpy/numpy/pull/14736>`__: DOC: Post 1.17.3 release update. +* `#14737 <https://github.com/numpy/numpy/pull/14737>`__: NEP: Accept NEP 29 as final +* `#14738 <https://github.com/numpy/numpy/pull/14738>`__: BUG: Don't narrow intp to int when producing error messages +* `#14742 <https://github.com/numpy/numpy/pull/14742>`__: DOC: lib: Fix deprecation markup in financial function docstrings. +* `#14743 <https://github.com/numpy/numpy/pull/14743>`__: DOC: Change from HTTP to HTTPS +* `#14745 <https://github.com/numpy/numpy/pull/14745>`__: BUG: clear only attribute errors in get_attr_string.h::maybe_get_attr +* `#14762 <https://github.com/numpy/numpy/pull/14762>`__: MAINT: doc: Remove doc/newdtype_example/ +* `#14763 <https://github.com/numpy/numpy/pull/14763>`__: Reword cautionary note about dtype.descr +* `#14769 <https://github.com/numpy/numpy/pull/14769>`__: BUG: fix integer size confusion in handling array's ndmin argument +* `#14771 <https://github.com/numpy/numpy/pull/14771>`__: TST, BUILD: add a gcc 4.8 run on ubuntu 18.04 +* `#14775 <https://github.com/numpy/numpy/pull/14775>`__: Update CLASSIFIERS with python 3.8 support +* `#14777 <https://github.com/numpy/numpy/pull/14777>`__: BUG: random: biased samples from integers() with 8 or 16 bit... +* `#14782 <https://github.com/numpy/numpy/pull/14782>`__: DOC: Add release note about changed random variate stream from... +* `#14786 <https://github.com/numpy/numpy/pull/14786>`__: DOC: Make changes to NEP procedure +* `#14790 <https://github.com/numpy/numpy/pull/14790>`__: DOC: random: Remove redundant 'See Also' entry in 'uniform' docstring. +* `#14791 <https://github.com/numpy/numpy/pull/14791>`__: MAINT: Minor typo fix +* `#14792 <https://github.com/numpy/numpy/pull/14792>`__: MAINT: Bump pytest from 5.2.1 to 5.2.2 +* `#14793 <https://github.com/numpy/numpy/pull/14793>`__: DOC: Adjust NEP-31 to new template. +* `#14794 <https://github.com/numpy/numpy/pull/14794>`__: DEP: issue deprecation warning when creating ragged array (NEP... +* `#14798 <https://github.com/numpy/numpy/pull/14798>`__: NEP: move 'NEP 29 random' from Accepted to Final +* `#14799 <https://github.com/numpy/numpy/pull/14799>`__: DOC: Add take_along_axis to the see also section in argmin, argmax... +* `#14800 <https://github.com/numpy/numpy/pull/14800>`__: ENH: change object-array comparisons to prefer OO->O unfuncs +* `#14805 <https://github.com/numpy/numpy/pull/14805>`__: TST: Don't construct Fraction instances from numpy scalars +* `#14814 <https://github.com/numpy/numpy/pull/14814>`__: Rename helper functions to not use the word rank +* `#14820 <https://github.com/numpy/numpy/pull/14820>`__: MAINT: Use templating to merge float loops +* `#14826 <https://github.com/numpy/numpy/pull/14826>`__: BUILD: ignore more build.log warnings +* `#14827 <https://github.com/numpy/numpy/pull/14827>`__: BLD: Prevent -flto from optimising long double representation... +* `#14829 <https://github.com/numpy/numpy/pull/14829>`__: BUG: raise ValueError for empty arrays passed to _pyarray_correlate +* `#14830 <https://github.com/numpy/numpy/pull/14830>`__: MAINT: move buffer.h -> npy_buffer.h to avoid conflicts +* `#14836 <https://github.com/numpy/numpy/pull/14836>`__: MAINT: Bump cython from 0.29.13 to 0.29.14 +* `#14841 <https://github.com/numpy/numpy/pull/14841>`__: ENH: add isinf, isnan, fmin, fmax loops for datetime64, timedelta64 +* `#14842 <https://github.com/numpy/numpy/pull/14842>`__: BLD: add 'apt update' to shippable +* `#14845 <https://github.com/numpy/numpy/pull/14845>`__: MAINT: revert gh-14800, which gave precedence to OO->O over OO->? +* `#14874 <https://github.com/numpy/numpy/pull/14874>`__: REL: Update master after 1.17.4 release. +* `#14878 <https://github.com/numpy/numpy/pull/14878>`__: BUILD: remove SSE2 flag from numpy.random builds +* `#14879 <https://github.com/numpy/numpy/pull/14879>`__: DOC: Update NEP29 with Python3.8 informations. +* `#14881 <https://github.com/numpy/numpy/pull/14881>`__: BUG: Remove builtins from __all__ +* `#14898 <https://github.com/numpy/numpy/pull/14898>`__: MAINT: Delete and ignore generated files +* `#14899 <https://github.com/numpy/numpy/pull/14899>`__: Update FUNDING.yml +* `#14901 <https://github.com/numpy/numpy/pull/14901>`__: MAINT: Remove uses of scalar aliases +* `#14903 <https://github.com/numpy/numpy/pull/14903>`__: NEP: move nep 34 to accepted +* `#14907 <https://github.com/numpy/numpy/pull/14907>`__: TST: Add s390x to the TravisCI test matrix. +* `#14912 <https://github.com/numpy/numpy/pull/14912>`__: DOC: Note FFT type promotion +* `#14914 <https://github.com/numpy/numpy/pull/14914>`__: TST: Test with Python3.8 on Windows. +* `#14915 <https://github.com/numpy/numpy/pull/14915>`__: TST: Update travis.yml +* `#14921 <https://github.com/numpy/numpy/pull/14921>`__: TST: add no_tracing decorator to refcount-sensitive codepath... +* `#14926 <https://github.com/numpy/numpy/pull/14926>`__: MAINT: Bump pytest from 5.2.2 to 5.2.4 +* `#14929 <https://github.com/numpy/numpy/pull/14929>`__: BUG: Fix step returned by linspace when num=1 and endpoint=False +* `#14932 <https://github.com/numpy/numpy/pull/14932>`__: DOC: Compare 'tolist' function to 'list' in example +* `#14935 <https://github.com/numpy/numpy/pull/14935>`__: DOC: Clarify return type for default_rng +* `#14944 <https://github.com/numpy/numpy/pull/14944>`__: MAINT: move numpy/random/examples -> numpy/random/_examples +* `#14947 <https://github.com/numpy/numpy/pull/14947>`__: DOC: testing: Note handling of scalars in assert_array_equal... +* `#14948 <https://github.com/numpy/numpy/pull/14948>`__: DOC, API: add random.__init__.pxd and document random.* functions +* `#14951 <https://github.com/numpy/numpy/pull/14951>`__: DOC: Clean up examples of low-level random access +* `#14954 <https://github.com/numpy/numpy/pull/14954>`__: TST. API: test using distributions.h via cffi +* `#14962 <https://github.com/numpy/numpy/pull/14962>`__: TST: skip if cython is not available +* `#14967 <https://github.com/numpy/numpy/pull/14967>`__: MAINT: Cleaned up mintypecode for Py3 +* `#14973 <https://github.com/numpy/numpy/pull/14973>`__: DOC: fix docstring of np.linalg.norm +* `#14974 <https://github.com/numpy/numpy/pull/14974>`__: MAINT: Added Python3.8 branch to dll lib discovery on Windows +* `#14976 <https://github.com/numpy/numpy/pull/14976>`__: DEV: update asv.conf.json +* `#14978 <https://github.com/numpy/numpy/pull/14978>`__: MAINT: Bump pytest from 5.2.4 to 5.3.0 +* `#14982 <https://github.com/numpy/numpy/pull/14982>`__: MAINT: Fix typos +* `#14983 <https://github.com/numpy/numpy/pull/14983>`__: REV: "ENH: Improved performance of PyArray_FromAny for sequences... +* `#14994 <https://github.com/numpy/numpy/pull/14994>`__: BUG: warn when saving dtype with metadata +* `#14996 <https://github.com/numpy/numpy/pull/14996>`__: DEP: Deprecate the axis argument to masked_rows and masked_cols +* `#15004 <https://github.com/numpy/numpy/pull/15004>`__: MAINT: Fix long name of PCG64 +* `#15007 <https://github.com/numpy/numpy/pull/15007>`__: DOC, API: improve the C-API/Cython documentation and interfaces... +* `#15009 <https://github.com/numpy/numpy/pull/15009>`__: DOC: Fix typo in numpy.loadtxt and numpy.genfromtxt documentation +* `#15012 <https://github.com/numpy/numpy/pull/15012>`__: ENH: allow using symbol-suffixed 64-bit BLAS/LAPACK for numpy.dot... +* `#15014 <https://github.com/numpy/numpy/pull/15014>`__: DOC: add a more useful comment to compat.py3k.py +* `#15019 <https://github.com/numpy/numpy/pull/15019>`__: DOC: lib: Use a clearer example of ddof in the notes of the cov... +* `#15021 <https://github.com/numpy/numpy/pull/15021>`__: TST: machinery for tests requiring large memory + lapack64 smoketest +* `#15023 <https://github.com/numpy/numpy/pull/15023>`__: MAINT: Only copy input array in _replace_nan() if there are nans... +* `#15025 <https://github.com/numpy/numpy/pull/15025>`__: MAINT: Bump pytest from 5.3.0 to 5.3.1 +* `#15027 <https://github.com/numpy/numpy/pull/15027>`__: REV: "ENH: Improved performance of PyArray_FromAny for sequences... +* `#15031 <https://github.com/numpy/numpy/pull/15031>`__: REL: Prepare for 1.18 branch +* `#15032 <https://github.com/numpy/numpy/pull/15032>`__: MAINT: Cleaned up mintypecode for Py3 (pt. 2) +* `#15036 <https://github.com/numpy/numpy/pull/15036>`__: BUG: Fix refcounting in ufunc object loops +* `#15039 <https://github.com/numpy/numpy/pull/15039>`__: BUG: Exceptions tracebacks are dropped +* `#15053 <https://github.com/numpy/numpy/pull/15053>`__: REV: Revert "Merge pull request #14794 from mattip/nep-0034-impl" +* `#15058 <https://github.com/numpy/numpy/pull/15058>`__: API, DOC: change names to multivariate_hypergeometric, improve docs +* `#15059 <https://github.com/numpy/numpy/pull/15059>`__: REL: Prepare for NumPy 1.18.0 release. +* `#15109 <https://github.com/numpy/numpy/pull/15109>`__: TST: Check requires_memory immediately before the test +* `#15111 <https://github.com/numpy/numpy/pull/15111>`__: ENH: Add support to sort timedelta64 `NaT` to end of the array +* `#15112 <https://github.com/numpy/numpy/pull/15112>`__: MAINT: follow-up cleanup for blas64 PR +* `#15113 <https://github.com/numpy/numpy/pull/15113>`__: ENH: f2py: add --f2cmap option for specifying the name of .f2py_f2cmap +* `#15114 <https://github.com/numpy/numpy/pull/15114>`__: ENH: add support for ILP64 OpenBLAS (without symbol suffix) +* `#15146 <https://github.com/numpy/numpy/pull/15146>`__: REL: Prepare for 1.18.0 release. diff --git a/doc/example.py b/doc/example.py index 560775038..8a5f9948f 100644 --- a/doc/example.py +++ b/doc/example.py @@ -112,9 +112,9 @@ def foo(var1, var2, long_var_name='hi'): use the function. >>> a = [1, 2, 3] - >>> print [x + 3 for x in a] + >>> print([x + 3 for x in a]) [4, 5, 6] - >>> print "a\n\nb" + >>> print("a\n\nb") a b diff --git a/doc/neps/index.rst.tmpl b/doc/neps/index.rst.tmpl index 0ad8e0f80..4c5b7766f 100644 --- a/doc/neps/index.rst.tmpl +++ b/doc/neps/index.rst.tmpl @@ -23,7 +23,7 @@ Meta-NEPs (NEPs about NEPs or Processes) .. toctree:: :maxdepth: 1 -{% for nep, tags in neps.items() if tags['Type'] == 'Process' %} +{% for nep, tags in neps.items() if tags['Status'] == 'Active' %} {{ tags['Title'] }} <{{ tags['Filename'] }}> {% endfor %} diff --git a/doc/neps/nep-0000.rst b/doc/neps/nep-0000.rst index 89ba177cb..0a2dbdefb 100644 --- a/doc/neps/nep-0000.rst +++ b/doc/neps/nep-0000.rst @@ -75,9 +75,11 @@ request`_ to the ``doc/neps`` directory with the name ``nep-<n>.rst`` where ``<n>`` is an appropriately assigned four-digit number (e.g., ``nep-0000.rst``). The draft must use the :doc:`nep-template` file. -Once the PR is in place, the NEP should be announced on the mailing -list for discussion (comments on the PR itself should be restricted to -minor editorial and technical fixes). +Once the PR for the NEP is in place, a post should be made to the +mailing list containing the sections upto "Backward compatibility", +with the purpose of limiting discussion there to usage and impact. +Discussion on the pull request will have a broader scope, also including +details of implementation. At the earliest convenience, the PR should be merged (regardless of whether it is accepted during discussion). Additional PRs may be made @@ -138,7 +140,7 @@ accepted that a competing proposal is a better alternative. When a NEP is ``Accepted``, ``Rejected``, or ``Withdrawn``, the NEP should be updated accordingly. In addition to updating the status field, at the very least the ``Resolution`` header should be added with a link to the relevant -post in the mailing list archives. +thread in the mailing list archives. NEPs can also be ``Superseded`` by a different NEP, rendering the original obsolete. The ``Replaced-By`` and ``Replaces`` headers diff --git a/doc/neps/nep-0019-rng-policy.rst b/doc/neps/nep-0019-rng-policy.rst index aa5fdc653..4f766fa2d 100644 --- a/doc/neps/nep-0019-rng-policy.rst +++ b/doc/neps/nep-0019-rng-policy.rst @@ -3,11 +3,11 @@ NEP 19 — Random Number Generator Policy ======================================= :Author: Robert Kern <robert.kern@gmail.com> -:Status: Accepted +:Status: Final :Type: Standards Track :Created: 2018-05-24 :Updated: 2019-05-21 -:Resolution: https://mail.python.org/pipermail/numpy-discussion/2018-June/078126.html +:Resolution: https://mail.python.org/pipermail/numpy-discussion/2018-July/078380.html Abstract -------- diff --git a/doc/neps/nep-0021-advanced-indexing.rst b/doc/neps/nep-0021-advanced-indexing.rst index 5acabbf16..dab9ab022 100644 --- a/doc/neps/nep-0021-advanced-indexing.rst +++ b/doc/neps/nep-0021-advanced-indexing.rst @@ -630,7 +630,7 @@ At this point we have left the straight forward world of ``oindex`` but can do random picking of any element from the array. Note that in the last example a method such as mentioned in the ``Related Questions`` section could be more straight forward. But this approach is even more flexible, since ``rows`` -does not have to be a simple ``arange``, but could be ``intersting_times``:: +does not have to be a simple ``arange``, but could be ``interesting_times``:: >>> interesting_times = np.array([0, 4, 8, 9, 10]) >>> correct_sensors_at_it = correct_sensors[interesting_times, :] diff --git a/doc/neps/nep-0024-missing-data-2.rst b/doc/neps/nep-0024-missing-data-2.rst index c8b19561f..f4414e0a0 100644 --- a/doc/neps/nep-0024-missing-data-2.rst +++ b/doc/neps/nep-0024-missing-data-2.rst @@ -28,7 +28,7 @@ Detailed description Rationale ^^^^^^^^^ -The purpose of this aNEP is to define two interfaces -- one for handling +The purpose of this NEP is to define two interfaces -- one for handling 'missing values', and one for handling 'masked arrays'. An ordinary value is something like an integer or a floating point number. A diff --git a/doc/neps/nep-0028-website-redesign.rst b/doc/neps/nep-0028-website-redesign.rst new file mode 100644 index 000000000..b418ca831 --- /dev/null +++ b/doc/neps/nep-0028-website-redesign.rst @@ -0,0 +1,334 @@ +=================================== +NEP 28 — numpy.org website redesign +=================================== + +:Author: Ralf Gommers <ralf.gommers@gmail.com> +:Author: Joe LaChance <joe@boldmetrics.com> +:Author: Shekhar Rajak <shekharrajak.1994@gmail.com> +:Status: Accepted +:Type: Informational +:Created: 2019-07-16 +:Resolution: https://mail.python.org/pipermail/numpy-discussion/2019-August/079889.html + + +Abstract +-------- + +NumPy is the fundamental library for numerical and scientific computing with +Python. It is used by millions and has a large team of maintainers and +contributors. Despite that, its `numpy.org <http://numpy.org>`_ website has +never received the attention it needed and deserved. We hope and intend to +change that soon. This document describes ideas and requirements for how to +design a replacement for the current website, to better serve the needs of +our diverse community. + +At a high level, what we're aiming for is: + +- a modern, clean look +- an easy to deploy static site +- a structure that's easy to navigate +- content that addresses all types of stakeholders +- Possible multilingual translations / i18n + +This website serves a couple of roles: + +- it's the entry point to the project for new users +- it should link to the documentation (which is hosted separately, now on + http://docs.scipy.org/ and in the near future on http://numpy.org/doc). +- it should address various aspects of the project (e.g. what NumPy is and + why you'd want to use it, community, project organization, funding, + relationship with NumFOCUS and possibly other organizations) +- it should link out to other places, so every type of stakeholder + (beginning and advanced user, educators, packagers, funders, etc.) + can find their way + + +Motivation and Scope +-------------------- + +The current numpy.org website has almost no content and its design is poor. +This affects many users, who come there looking for information. It also +affects many other aspects of the NumPy project, from finding new contributors +to fundraising. + +The scope of the proposed redesign is the top-level numpy.org site, which +now contains only a couple of pages and may contain on the order of ten +pages after the redesign. Changing the documentation (user guide, reference +guide, and some other pages in the NumPy Manual) is out of scope for +this proposal. + + +Detailed description +-------------------- + +User Experience +~~~~~~~~~~~~~~~ + +Besides the NumPy logo, there is little that can or needs to be kept from the +current website. We will rely to a large extent on ideas and proposals by the +designer(s) of the new website. + +As reference points we can use the `Jupyter website <https://jupyter.org/>`_, +which is probably the best designed site in our ecosystem, and the +`QuantEcon <https://quantecon.org>`_ and `Julia <https://julialang.org>`_ +sites which are well-designed too. + +The Website +~~~~~~~~~~~ + +A static site is a must. There are many high-quality static site generators. +The current website uses Sphinx, however that is not the best choice - it's +hard to theme and results in sites that are too text-heavy due to Sphinx' +primary aim being documentation. + +The following should be considered when choosing a static site generator: + +1. *How widely used is it?* This is important when looking for help maintaining + or improving the site. More popular frameworks are usually also better + maintained, so less chance of bugs or obsolescence. +2. *Ease of deployment.* Most generators meet this criterion, however things + like built-in support for GitHub Pages helps. +3. *Preferences of who implements the new site.* Everyone has their own + preferences. And it's a significant amount of work to build a new site. + So we should take the opinion of those doing the work into account. + +Traffic +``````` + +The current site receives on the order of 500,000 unique visitors per month. +With a redesigned site and relevant content, there is potential for visitor +counts to reach 5-6 million -- a similar level as +`scipy.org <http://scipy.org>`_ or `matplotlib.org <http://matplotlib.org>`_ -- +or more. + +Possible options for static site generators +``````````````````````````````````````````` + +1. *Jekyll.* This is a well maintained option with 855 Github contributors, + with contributions within the last month. Jekyll is written in Ruby, and + has a simple CLI interface. Jekyll also has a large directory of + `themes <https://jekyllthemes.io>`__, although a majority cost money. + There are several themes (`serif <https://jekyllthemes.io/theme/serif>`_, + `uBuild <https://jekyllthemes.io/theme/ubuild-jekyll-theme>`_, + `Just The Docs <https://jekyllthemes.io/theme/just-the-docs>`_) that are + appropriate and free. Most themes are likely responsive for mobile, and + that should be a requirement. Jekyll uses a combination of liquid templating + and YAML to render HTML, and content is written in Markdown. i18n + functionality is not native to Jekyll, but can be added easily. + One nice benefit of Jekyll is that it can be run automatically by GitHub + Pages, so deployment via a CI system doesn't need to be implemented. +2. *Hugo.* This is another well maintained option with 554 contributors, with + contributions within the last month. Hugo is written in Go, and similar to + Jekyll, has a simple to use CLI interface to generate static sites. Again, + similar to Jekyll, Hugo has a large directory of + `themes <https://themes.gohugo.io>`_. These themes appear to be free, + unlike some of Jekyll's themes. + (`Sample landing page theme <https://themes.gohugo.io/hugo-hero-theme>`_, + `docs theme <https://themes.gohugo.io/hugo-whisper-theme>`_). Hugo uses Jade + as its templating language, and content is also written in Markdown. i18n + functionality is native to Hugo. +3. *Docusaurus.* Docusaurus is a responsive static site generator made by Facebook. + Unlike the previous options, Docusaurus doesn't come with themes, and thus we + would not want to use this for our landing page. This is an excellent docs + option written in React. Docusaurus natively has support for i18n (via + Crowdin_, document versioning, and document search. + +Both Jekyll and Hugo are excellent options that should be supported into the +future and are good choices for NumPy. Docusaurus has several bonus features +such as versioning and search that Jekyll and Hugo don't have, but is likely +a poor candidate for a landing page - it could be a good option for a +high-level docs site later on though. + +Deployment +~~~~~~~~~~ + +There is no need for running a server, and doing so is in our experience a +significant drain on the time of maintainers. + +1. *Netlify.* Using netlify is free until 100GB of bandwidth is used. Additional + bandwidth costs $20/100GB. They support a global CDN system, which will keep + load times quick for users in other regions. Netlify also has Github integration, + which will allow for easy deployment. When a pull request is merged, Netlify + will automatically deploy the changes. DNS is simple, and HTTPS is also supported. +2. *Github Pages.* Github Pages also has a 100GB bandwidth limit, and is unclear if + additional bandwidth can be purchased. It is also unclear where sites are deployed, + and should be assumed sites aren't deployed globally. Github Pages has an easy to + use CI & DNS, similar to to Netlify. HTTPS is supported. +3. *Cloudflare.* An excellent option, additional CI is likely needed for the same + ease of deployment. + +All of the above options are appropriate for the NumPy site based on current +traffic. Updating to a new deployment strategy, if needed, is a minor amount of +work compared to developing the website itself. If a provider such as +Cloudflare is chosen, additional CI may be required, such as CircleCI, to +have a similar deployment to GitHub Pages or Netlify. + +Analytics +~~~~~~~~~ + +It's benefical to maintainers to know how many visitors are coming to +numpy.org. Google Analytics offers visitor counts and locations. This will +help to support and deploy more strategically, and help maintainers +understand where traffic is coming from. + +Google Analytics is free. A script, provided by Google, must be added to the home page. + +Website Structure +~~~~~~~~~~~~~~~~~ + +We aim to keep the first version of the new website small in terms of amount +of content. New pages can be added later on, it's more important right now to +get the site design right and get some essential information up. Note that in +the second half of 2019 we expect to get 1 or 2 tech writers involved in the +project via Google Season of Docs. They will likely help improve the content +and organization of that content. + +We propose the following structure: + +0. Front page: essentials of what NumPy is (compare e.g. jupyter.org), one or + a couple key user stories (compare e.g. julialang.org) +1. Install +2. Documentation +3. Array computing +4. Community +5. Learning +6. About Us +7. Contribute +8. Donate + +There may be a few other pages, e.g. a page on performance, that are linked +from one of the main pages. + +Stakeholder Content +~~~~~~~~~~~~~~~~~~~ + +This should have as little content as possible *within the site*. Somewhere +on the site we should link out to content that's specific to: + +- beginning users (quickstart, tutorial) +- advanced users +- educators +- packagers +- package authors that depend on NumPy +- funders (governance, roadmap) + +Translation (multilingual / i18n) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +NumPy has users all over the world. Most of those users are not native +English speakers, and many don't speak English well or at all. Therefore +having content in multiple languages is potentially addressing a large unmet +need. It would likely also help make the NumPy project more diverse and +welcoming. + +On the other hand, there are good reasons why few projects have a +multi-lingual site. It's potentially a lot of extra work. Extra work for +maintainers is costly - they're already struggling to keep up with the work +load. Therefore we have to very carefully consider whether a multi-lingual +site is feasible and weight costs and benefits. + +We start with an assertion: maintaining translations of all documentation, or +even the whole user guide, as part of the NumPy project is not feasible. One +simply has to look at the volume of our documentation and the frequency with +which we change it to realize that that's the case. Perhaps it will be +feasible though to translate just the top-level pages of the website. Those +do not change very often, and it will be a limited amount of content (order +of magnitude 5-10 pages of text). + +We propose the following requirements for adding a language: + +- The language must have a dedicated maintainer +- There must be a way to validate content changes (e.g. a second + maintainer/reviewer, or high quality language support in a freely + available machine translation tool) +- The language must have a reasonable size target audience (to be + assessed by the NumPy maintainers) + +Furthermore we propose a policy for when to remove support for a language again +(preferably by hiding it rather than deleting content). This may be done when +the language no longer has a maintainer, and coverage of translations falls +below an acceptable threshold (say 80%). + +Benefits of having translations include: + +- Better serve many existing and potential users +- Potentially attract a culturally and geographically more diverse set of contributors + +The tradeoffs are: + +- Cost of maintaining a more complex code base +- Cost of making decisions about whether or not to add a new language +- Higher cost to making content changes, creates work for language maintainers +- Any content change should be rolled out with enough delay to have translations in place + +Can we define a small enough set of pages and content that it makes sense to do this? +Probably yes. + +Is there an easy to use tool to maintain translations and add them to the website? +To be discussed - it needs investigating, and may depend on the choice of static site +generator. One potential option is Crowdin_, which is free for open source projects. + + +Style and graphic design +~~~~~~~~~~~~~~~~~~~~~~~~ + +Beyond the "a modern, clean look" goal we choose to not specify too much. A +designer may have much better ideas than the authors of this proposal, hence we +will work with the designer(s) during the implementation phase. + +The NumPy logo could use a touch-up. The logo widely recognized and its colors and +design are good, however the look-and-feel is perhaps a little dated. + + +Other aspects +~~~~~~~~~~~~~ + +A search box would be nice to have. The Sphinx documentation already has a +search box, however a search box on the main site which provides search results +for the docs, the website, and perhaps other domains that are relevant for +NumPy would make sense. + + +Backward compatibility +---------------------- + +Given a static site generator is chosen, we will migrate away from Sphinx for +numpy.org (the website, *not including the docs*). The current deployment can +be preserved until a future deprecation date is decided (potentially based on +the comfort level of our new site). + +All site generators listed above have visibility into the HTML and Javascript +that is generated, and can continue to be maintained in the event a given +project ceases to be maintained. + + +Alternatives +------------ + +Alternatives we considered for the overall design of the website: + +1. *Update current site.* A new Sphinx theme could be chosen. This would likely + take the least amount of resources initially, however, Sphinx does not have + the features we are looking for moving forward such as i18n, responsive design, + and a clean, modern look. + Note that updating the docs Sphinx theme is likely still a good idea - it's + orthogonal to this NEP though. +2. *Create custom site.* This would take the most amount of resources, and is + likely to have additional benefit in comparison to a static site generator. + All features would be able to be added at the cost of developer time. + + +Discussion +---------- + +Mailing list thread discussing this NEP: TODO + + +References and Footnotes +------------------------ +.. _Crowdin: https://crowdin.com/pricing#annual + +Copyright +--------- + +This document has been placed in the public domain. diff --git a/doc/neps/nep-0029-deprecation_policy.rst b/doc/neps/nep-0029-deprecation_policy.rst new file mode 100644 index 000000000..bdc0baa39 --- /dev/null +++ b/doc/neps/nep-0029-deprecation_policy.rst @@ -0,0 +1,305 @@ +================================================================================== +NEP 29 — Recommend Python and Numpy version support as a community policy standard +================================================================================== + + +:Author: Thomas A Caswell <tcaswell@gmail.com>, Andreas Mueller, Brian Granger, Madicken Munk, Ralf Gommers, Matt Haberland <mhaberla@calpoly.edu>, Matthias Bussonnier <bussonniermatthias@gmail.com>, Stefan van der Walt <stefanv@berkeley.edu> +:Status: Final +:Type: Informational +:Created: 2019-07-13 +:Resolution: https://mail.python.org/pipermail/numpy-discussion/2019-October/080128.html + + +Abstract +-------- + +This NEP recommends that all projects across the Scientific +Python ecosystem adopt a common "time window-based" policy for +support of Python and NumPy versions. Standardizing a recommendation +for project support of minimum Python and NumPy versions will improve +downstream project planning. + +This is an unusual NEP in that it offers recommendations for +community-wide policy and not for changes to NumPy itself. Since a +common place for SPEEPs (Scientific Python Ecosystem Enhancement +Proposals) does not exist and given NumPy's central role in the +ecosystem, a NEP provides a visible place to document the proposed +policy. + +This NEP is being put forward by maintainers of Matplotlib, scikit-learn, +IPython, Jupyter, yt, SciPy, NumPy, and scikit-image. + + + +Detailed description +-------------------- + +For the purposes of this NEP we assume semantic versioning and define: + +*major version* + A release that changes the first number (e.g. X.0.0) + +*minor version* + A release that changes the second number (e.g 1.Y.0) + +*patch version* + A release that changes the third number (e.g. 1.1.Z) + + +When a project releases a new major or minor version, we recommend that +they support at least all minor versions of Python +introduced and released in the prior 42 months *from the +anticipated release date* with a minimum of 2 minor versions of +Python, and all minor versions of NumPy released in the prior 24 +months *from the anticipated release date* with a minimum of 3 +minor versions of NumPy. + + +Consider the following timeline:: + + Jan 16 Jan 17 Jan 18 Jan 19 Jan 20 + | | | | | + +++++|+++++++++++|+++++++++++|+++++++++++|+++++++++++|++++++++++++ + | | | | + py 3.5.0 py 3.6.0 py 3.7.0 py 3.8.0 + |-----------------------------------------> Feb19 + |-----------------------------------------> Dec19 + |-----------------------------------------> Nov20 + +It shows the 42 month support windows for Python. A project with a +major or minor version release in February 2019 should support Python 3.5 and newer, +a project with a major or minor version released in December 2019 should +support Python 3.6 and newer, and a project with a major or minor version +release in November 2020 should support Python 3.7 and newer. + +The current Python release cadence is 18 months so a 42 month window +ensures that there will always be at least two minor versions of Python +in the window. The window is extended 6 months beyond the anticipated two-release +interval for Python to provides resilience against small fluctuations / +delays in its release schedule. + +Because Python minor version support is based only on historical +release dates, a 42 month time window, and a planned project release +date, one can predict with high confidence when a project will be able +to drop any given minor version of Python. This, in turn, could save +months of unnecessary maintenance burden. + +If a project releases immediately after a minor version of Python +drops out of the support window, there will inevitably be some +mismatch in supported versions—but this situation should only last +until other projects in the ecosystem make releases. + +Otherwise, once a project does a minor or major release, it is +guaranteed that there will be a stable release of all other projects +that, at the source level, support the same set of Python versions +supported by the new release. + +If there is a Python 4 or a NumPy 2 this policy will have to be +reviewed in light of the community's and projects' best interests. + + +Support Table +~~~~~~~~~~~~~ + +============ ====== ===== +Date Python NumPy +------------ ------ ----- +Jan 07, 2020 3.6+ 1.15+ +Jun 23, 2020 3.7+ 1.15+ +Jul 23, 2020 3.7+ 1.16+ +Jan 13, 2021 3.7+ 1.17+ +Jul 26, 2021 3.7+ 1.18+ +Dec 26, 2021 3.8+ 1.18+ +Apr 14, 2023 3.9+ 1.18+ +============ ====== ===== + + +Drop Schedule +~~~~~~~~~~~~~ + +:: + + On next release, drop support for Python 3.5 (initially released on Sep 13, 2015) + On Jan 07, 2020 drop support for Numpy 1.14 (initially released on Jan 06, 2018) + On Jun 23, 2020 drop support for Python 3.6 (initially released on Dec 23, 2016) + On Jul 23, 2020 drop support for Numpy 1.15 (initially released on Jul 23, 2018) + On Jan 13, 2021 drop support for Numpy 1.16 (initially released on Jan 13, 2019) + On Jul 26, 2021 drop support for Numpy 1.17 (initially released on Jul 26, 2019) + On Dec 26, 2021 drop support for Python 3.7 (initially released on Jun 27, 2018) + On Apr 14, 2023 drop support for Python 3.8 (initially released on Oct 14, 2019) + + +Implementation +-------------- + +We suggest that all projects adopt the following language into their +development guidelines: + + This project supports: + + - All minor versions of Python released 42 months prior to the + project, and at minimum the two latest minor versions. + - All minor versions of ``numpy`` released in the 24 months prior + to the project, and at minimum the last three minor versions. + + In ``setup.py``, the ``python_requires`` variable should be set to + the minimum supported version of Python. All supported minor + versions of Python should be in the test matrix and have binary + artifacts built for the release. + + Minimum Python and NumPy version support should be adjusted upward + on every major and minor release, but never on a patch release. + + +Backward compatibility +---------------------- + +No backward compatibility issues. + +Alternatives +------------ + +Ad-Hoc version support +~~~~~~~~~~~~~~~~~~~~~~ + +A project could, on every release, evaluate whether to increase +the minimum version of Python supported. +As a major downside, an ad-hoc approach makes it hard for downstream users to predict what +the future minimum versions will be. As there is no objective threshold +to when the minimum version should be dropped, it is easy for these +version support discussions to devolve into `bike shedding <https://en.wikipedia.org/wiki/Wikipedia:Avoid_Parkinson%27s_bicycle-shed_effect>`_ and acrimony. + + +All CPython supported versions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The CPython supported versions of Python are listed in the Python +Developers Guide and the Python PEPs. Supporting these is a very clear +and conservative approach. However, it means that there exists a four +year lag between when a new features is introduced into the language +and when a project is able to use it. Additionally, for projects with +compiled extensions this requires building many binary artifacts for +each release. + +For the case of NumPy, many projects carry workarounds to bugs that +are fixed in subsequent versions of NumPy. Being proactive about +increasing the minimum version of NumPy allows downstream +packages to carry fewer version-specific patches. + + + +Default version on Linux distribution +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The policy could be to support the version of Python that ships by +default in the latest Ubuntu LTS or CentOS/RHEL release. However, we +would still have to standardize across the community which +distribution to follow. + +By following the versions supported by major Linux distributions, we +are giving up technical control of our projects to external +organizations that may have different motivations and concerns than we +do. + + +N minor versions of Python +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Given the current release cadence of the Python, the proposed time (42 +months) is roughly equivalent to "the last two" Python minor versions. +However, if Python changes their release cadence substantially, any +rule based solely on the number of minor releases may need to be +changed to remain sensible. + +A more fundamental problem with a policy based on number of Python +releases is that it is hard to predict when support for a given minor +version of Python will be dropped as that requires correctly +predicting the release schedule of Python for the next 3-4 years. A +time-based rule, in contrast, only depends on past events +and the length of the support window. + + +Time window from the X.Y.1 Python release +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This is equivalent to a few month longer support window from the X.Y.0 +release. This is because X.Y.1 bug-fix release is typically a few +months after the X.Y.0 release, thus a N month window from X.Y.1 is +roughly equivalent to a N+3 month from X.Y.0. + +The X.Y.0 release is naturally a special release. If we were to +anchor the window on X.Y.1 we would then have the discussion of why +not X.Y.M? + + +Discussion +---------- + + +References and Footnotes +------------------------ + +Code to generate support and drop schedule tables :: + + from datetime import datetime, timedelta + + data = """Jan 15, 2017: Numpy 1.12 + Sep 13, 2015: Python 3.5 + Dec 23, 2016: Python 3.6 + Jun 27, 2018: Python 3.7 + Jun 07, 2017: Numpy 1.13 + Jan 06, 2018: Numpy 1.14 + Jul 23, 2018: Numpy 1.15 + Jan 13, 2019: Numpy 1.16 + Jul 26, 2019: Numpy 1.17 + Oct 14, 2019: Python 3.8 + """ + + releases = [] + + plus42 = timedelta(days=int(365*3.5 + 1)) + plus24 = timedelta(days=int(365*2 + 1)) + + for line in data.splitlines(): + date, project_version = line.split(':') + project, version = project_version.strip().split(' ') + release = datetime.strptime(date, '%b %d, %Y') + if project.lower() == 'numpy': + drop = release + plus24 + else: + drop = release + plus42 + releases.append((drop, project, version, release)) + + releases = sorted(releases, key=lambda x: x[0]) + + minpy = '3.9+' + minnum = '1.18+' + + toprint_drop_dates = [''] + toprint_support_table = [] + for d, p, v, r in releases[::-1]: + df = d.strftime('%b %d, %Y') + toprint_drop_dates.append( + f'On {df} drop support for {p} {v} ' + f'(initially released on {r.strftime("%b %d, %Y")})') + toprint_support_table.append(f'{df} {minpy:<6} {minnum:<5}') + if p.lower() == 'numpy': + minnum = v+'+' + else: + minpy = v+'+' + + for e in toprint_drop_dates[::-1]: + print(e) + + print('============ ====== =====') + print('Date Python NumPy') + print('------------ ------ -----') + for e in toprint_support_table[::-1]: + print(e) + print('============ ====== =====') + + +Copyright +--------- + +This document has been placed in the public domain. diff --git a/doc/neps/nep-0030-duck-array-protocol.rst b/doc/neps/nep-0030-duck-array-protocol.rst new file mode 100644 index 000000000..f98449ed6 --- /dev/null +++ b/doc/neps/nep-0030-duck-array-protocol.rst @@ -0,0 +1,183 @@ +====================================================== +NEP 30 — Duck Typing for NumPy Arrays - Implementation +====================================================== + +:Author: Peter Andreas Entschev <pentschev@nvidia.com> +:Author: Stephan Hoyer <shoyer@google.com> +:Status: Draft +:Type: Standards Track +:Created: 2019-07-31 +:Updated: 2019-07-31 +:Resolution: + +Abstract +-------- + +We propose the ``__duckarray__`` protocol, following the high-level overview +described in NEP 22, allowing downstream libraries to return arrays of their +defined types, in contrast to ``np.asarray``, that coerces those ``array_like`` +objects to NumPy arrays. + +Detailed description +-------------------- + +NumPy's API, including array definitions, is implemented and mimicked in +countless other projects. By definition, many of those arrays are fairly +similar in how they operate to the NumPy standard. The introduction of +``__array_function__`` allowed dispathing of functions implemented by several +of these projects directly via NumPy's API. This introduces a new requirement, +returning the NumPy-like array itself, rather than forcing a coercion into a +pure NumPy array. + +For the purpose above, NEP 22 introduced the concept of duck typing to NumPy +arrays. The suggested solution described in the NEP allows libraries to avoid +coercion of a NumPy-like array to a pure NumPy array where necessary, while +still allowing that NumPy-like array libraries that do not wish to implement +the protocol to coerce arrays to a pure Numpy array via ``np.asarray``. + +Usage Guidance +~~~~~~~~~~~~~~ + +Code that uses np.duckarray is meant for supporting other ndarray-like objects +that "follow the NumPy API". That is an ill-defined concept at the moment -- +every known library implements the NumPy API only partly, and many deviate +intentionally in at least some minor ways. This cannot be easily remedied, so +for users of ``__duckarray__`` we recommend the following strategy: check if the +NumPy functionality used by the code that follows your use of ``__duckarray__`` +is present in Dask, CuPy and Sparse. If so, it's reasonable to expect any duck +array to work here. If not, we suggest you indicate in your docstring what kinds +of duck arrays are accepted, or what properties they need to have. + +To exemplify the usage of duck arrays, suppose one wants to take the ``mean()`` +of an array-like object ``arr``. Using NumPy to achieve that, one could write +``np.asarray(arr).mean()`` to achieve the intended result. However, libraries +may expect ``arr`` to be a NumPy-like array, and at the same time, the array may +or may not be an object compliant to the NumPy API (either in full or partially) +such as a CuPy, Sparse or a Dask array. In the case where ``arr`` is already an +object compliant to the NumPy API, we would simply return it (and prevent it +from being coerced into a pure NumPy array), otherwise, it would then be coerced +into a NumPy array. + +Implementation +-------------- + +The implementation idea is fairly straightforward, requiring a new function +``duckarray`` to be introduced in NumPy, and a new method ``__duckarray__`` in +NumPy-like array classes. The new ``__duckarray__`` method shall return the +downstream array-like object itself, such as the ``self`` object. If appropriate, +an ``__array__`` method may be implemented that returns a NumPy array or possibly +raise a ``TypeError`` with a helpful message. + +The new NumPy ``duckarray`` function can be implemented as follows: + +.. code:: python + + def duckarray(array_like): + if hasattr(array_like, '__duckarray__'): + return array_like.__duckarray__() + return np.asarray(array_like) + +Example for a project implementing NumPy-like arrays +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now consider a library that implements a NumPy-compatible array class called +``NumPyLikeArray``, this class shall implement the methods described above, and +a complete implementation would look like the following: + +.. code:: python + + class NumPyLikeArray: + def __duckarray__(self): + return self + + def __array__(self): + return TypeError("NumPyLikeArray can not be converted to a numpy array. " + "You may want to use np.duckarray.") + +The implementation above exemplifies the simplest case, but the overall idea +is that libraries will implement a ``__duckarray__`` method that returns the +original object, and an ``__array__`` method that either creates and returns an +appropriate NumPy array, or raises a``TypeError`` to prevent unintentional use +as an object in a NumPy array (if ``np.asarray`` is called on an arbitrary +object that does not implement ``__array__``, it will create a NumPy array +scalar). + +In case of existing libraries that don't already implement ``__array__`` but +would like to use duck array typing, it is advised that they introduce +both ``__array__`` and``__duckarray__`` methods. + +Usage +----- + +An example of how the ``__duckarray__`` protocol could be used to write a +``stack`` function based on ``concatenate``, and its produced outcome, can be +seen below. The example here was chosen not only to demonstrate the usage of +the ``duckarray`` function, but also to demonstrate its dependency on the NumPy +API, demonstrated by checks on the array's ``shape`` attribute. Note that the +example is merely a simplified version of NumPy's actually implementation of +``stack`` working on the first axis, and it is assumed that Dask has implemented +the ``__duckarray__`` method. + +.. code:: python + + def duckarray_stack(arrays): + arrays = [np.duckarray(arr) for arr in arrays] + + shapes = {arr.shape for arr in arrays} + if len(shapes) != 1: + raise ValueError('all input arrays must have the same shape') + + expanded_arrays = [arr[np.newaxis, ...] for arr in arrays] + return np.concatenate(expanded_arrays, axis=0) + + dask_arr = dask.array.arange(10) + np_arr = np.arange(10) + np_like = list(range(10)) + + duckarray_stack((dask_arr, dask_arr)) # Returns dask.array + duckarray_stack((dask_arr, np_arr)) # Returns dask.array + duckarray_stack((dask_arr, np_like)) # Returns dask.array + +In contrast, using only ``np.asarray`` (at the time of writing of this NEP, this +is the usual method employed by library developers to ensure arrays are +NumPy-like) has a different outcome: + +.. code:: python + + def asarray_stack(arrays): + arrays = [np.asanyarray(arr) for arr in arrays] + + # The remaining implementation is the same as that of + # ``duckarray_stack`` above + + asarray_stack((dask_arr, dask_arr)) # Returns np.ndarray + asarray_stack((dask_arr, np_arr)) # Returns np.ndarray + asarray_stack((dask_arr, np_like)) # Returns np.ndarray + +Backward compatibility +---------------------- + +This proposal does not raise any backward compatibility issues within NumPy, +given that it only introduces a new function. However, downstream libraries +that opt to introduce the ``__duckarray__`` protocol may choose to remove the +ability of coercing arrays back to a NumPy array via ``np.array`` or +``np.asarray`` functions, preventing unintended effects of coercion of such +arrays back to a pure NumPy array (as some libraries already do, such as CuPy +and Sparse), but still leaving libraries not implementing the protocol with the +choice of utilizing ``np.duckarray`` to promote ``array_like`` objects to pure +NumPy arrays. + +Previous proposals and discussion +--------------------------------- + +The duck typing protocol proposed here was described in a high level in +`NEP 22 <https://numpy.org/neps/nep-0022-ndarray-duck-typing-overview.html>`_. + +Additionally, longer discussions about the protocol and related proposals +took place in +`numpy/numpy #13831 <https://github.com/numpy/numpy/issues/13831>`_ + +Copyright +--------- + +This document has been placed in the public domain. diff --git a/doc/neps/nep-0031-uarray.rst b/doc/neps/nep-0031-uarray.rst new file mode 100644 index 000000000..737d2b456 --- /dev/null +++ b/doc/neps/nep-0031-uarray.rst @@ -0,0 +1,657 @@ +============================================================ +NEP 31 — Context-local and global overrides of the NumPy API +============================================================ + +:Author: Hameer Abbasi <habbasi@quansight.com> +:Author: Ralf Gommers <rgommers@quansight.com> +:Author: Peter Bell <pbell@quansight.com> +:Status: Draft +:Type: Standards Track +:Created: 2019-08-22 + + +Abstract +-------- + +This NEP proposes to make all of NumPy's public API overridable via an +extensible backend mechanism. + +Acceptance of this NEP means NumPy would provide global and context-local +overrides in a separate namespace, as well as a dispatch mechanism similar +to NEP-18 [2]_. First experiences with ``__array_function__`` show that it +is necessary to be able to override NumPy functions that *do not take an +array-like argument*, and hence aren't overridable via +``__array_function__``. The most pressing need is array creation and coercion +functions, such as ``numpy.zeros`` or ``numpy.asarray``; see e.g. NEP-30 [9]_. + +This NEP proposes to allow, in an opt-in fashion, overriding any part of the +NumPy API. It is intended as a comprehensive resolution to NEP-22 [3]_, and +obviates the need to add an ever-growing list of new protocols for each new +type of function or object that needs to become overridable. + +Motivation and Scope +-------------------- + +The primary end-goal of this NEP is to make the following possible: + +.. code:: python + + # On the library side + import numpy.overridable as unp + + def library_function(array): + array = unp.asarray(array) + # Code using unumpy as usual + return array + + # On the user side: + import numpy.overridable as unp + import uarray as ua + import dask.array as da + + ua.register_backend(da) # Can be done within Dask itself + + library_function(dask_array) # works and returns dask_array + + with unp.set_backend(da): + library_function([1, 2, 3, 4]) # actually returns a Dask array. + +Here, ``backend`` can be any compatible object defined either by NumPy or an +external library, such as Dask or CuPy. Ideally, it should be the module +``dask.array`` or ``cupy`` itself. + +These kinds of overrides are useful for both the end-user as well as library +authors. End-users may have written or wish to write code that they then later +speed up or move to a different implementation, say PyData/Sparse. They can do +this simply by setting a backend. Library authors may also wish to write code +that is portable across array implementations, for example ``sklearn`` may wish +to write code for a machine learning algorithm that is portable across array +implementations while also using array creation functions. + +This NEP takes a holistic approach: It assumes that there are parts of +the API that need to be overridable, and that these will grow over time. It +provides a general framework and a mechanism to avoid a design of a new +protocol each time this is required. This was the goal of ``uarray``: to +allow for overrides in an API without needing the design of a new protocol. + +This NEP proposes the following: That ``unumpy`` [8]_ becomes the +recommended override mechanism for the parts of the NumPy API not yet covered +by ``__array_function__`` or ``__array_ufunc__``, and that ``uarray`` is +vendored into a new namespace within NumPy to give users and downstream +dependencies access to these overrides. This vendoring mechanism is similar +to what SciPy decided to do for making ``scipy.fft`` overridable (see [10]_). + +The motivation behind ``uarray`` is manyfold: First, there have been several +attempts to allow dispatch of parts of the NumPy API, including (most +prominently), the ``__array_ufunc__`` protocol in NEP-13 [4]_, and the +``__array_function__`` protocol in NEP-18 [2]_, but this has shown the need +for further protocols to be developed, including a protocol for coercion (see +[5]_, [9]_). The reasons these overrides are needed have been extensively +discussed in the references, and this NEP will not attempt to go into the +details of why these are needed; but in short: It is necessary for library +authors to be able to coerce arbitrary objects into arrays of their own types, +such as CuPy needing to coerce to a CuPy array, for example, instead of +a NumPy array. In simpler words, one needs things like ``np.asarray(...)`` or +an alternative to "just work" and return duck-arrays. + +Usage and Impact +---------------- + +This NEP allows for global and context-local overrides, as well as +automatic overrides a-la ``__array_function__``. + +Here are some use-cases this NEP would enable, besides the +first one stated in the motivation section: + +The first is allowing alternate dtypes to return their +respective arrays. + +.. code:: python + + # Returns an XND array + x = unp.ones((5, 5), dtype=xnd_dtype) # Or torch dtype + +The second is allowing overrides for parts of the API. +This is to allow alternate and/or optimised implementations +for ``np.linalg``, BLAS, and ``np.random``. + +.. code:: python + + import numpy as np + import pyfftw # Or mkl_fft + + # Makes pyfftw the default for FFT + np.set_global_backend(pyfftw) + + # Uses pyfftw without monkeypatching + np.fft.fft(numpy_array) + + with np.set_backend(pyfftw) # Or mkl_fft, or numpy + # Uses the backend you specified + np.fft.fft(numpy_array) + +This will allow an official way for overrides to work with NumPy without +monkeypatching or distributing a modified version of NumPy. + +Here are a few other use-cases, implied but not already +stated: + +.. code:: python + + data = da.from_zarr('myfile.zarr') + # result should still be dask, all things being equal + result = library_function(data) + result.to_zarr('output.zarr') + +This second one would work if ``magic_library`` was built +on top of ``unumpy``. + +.. code:: python + + from dask import array as da + from magic_library import pytorch_predict + + data = da.from_zarr('myfile.zarr') + # normally here one would use e.g. data.map_overlap + result = pytorch_predict(data) + result.to_zarr('output.zarr') + +There are some backends which may depend on other backends, for example xarray +depending on `numpy.fft`, and transforming a time axis into a frequency axis, +or Dask/xarray holding an array other than a NumPy array inside it. This would +be handled in the following manner inside code:: + + with ua.set_backend(cupy), ua.set_backend(dask.array): + # Code that has distributed GPU arrays here + +Backward compatibility +---------------------- + +There are no backward incompatible changes proposed in this NEP. + +Detailed description +-------------------- + +Proposals +~~~~~~~~~ + +The only change this NEP proposes at its acceptance, is to make ``unumpy`` the +officially recommended way to override NumPy, along with making some submodules +overridable by default via ``uarray``. ``unumpy`` will remain a separate +repository/package (which we propose to vendor to avoid a hard dependency, and +use the separate ``unumpy`` package only if it is installed, rather than depend +on for the time being). In concrete terms, ``numpy.overridable`` becomes an +alias for ``unumpy``, if available with a fallback to the a vendored version if +not. ``uarray`` and ``unumpy`` and will be developed primarily with the input +of duck-array authors and secondarily, custom dtype authors, via the usual +GitHub workflow. There are a few reasons for this: + +* Faster iteration in the case of bugs or issues. +* Faster design changes, in the case of needed functionality. +* ``unumpy`` will work with older versions of NumPy as well. +* The user and library author opt-in to the override process, + rather than breakages happening when it is least expected. + In simple terms, bugs in ``unumpy`` mean that ``numpy`` remains + unaffected. +* For ``numpy.fft``, ``numpy.linalg`` and ``numpy.random``, the functions in + the main namespace will mirror those in the ``numpy.overridable`` namespace. + The reason for this is that there may exist functions in the in these + submodules that need backends, even for ``numpy.ndarray`` inputs. + +Advantanges of ``unumpy`` over other solutions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``unumpy`` offers a number of advantanges over the approach of defining a new +protocol for every problem encountered: Whenever there is something requiring +an override, ``unumpy`` will be able to offer a unified API with very minor +changes. For example: + +* ``ufunc`` objects can be overridden via their ``__call__``, ``reduce`` and + other methods. +* Other functions can be overridden in a similar fashion. +* ``np.asduckarray`` goes away, and becomes ``np.overridable.asarray`` with a + backend set. +* The same holds for array creation functions such as ``np.zeros``, + ``np.empty`` and so on. + +This also holds for the future: Making something overridable would require only +minor changes to ``unumpy``. + +Another promise ``unumpy`` holds is one of default implementations. Default +implementations can be provided for any multimethod, in terms of others. This +allows one to override a large part of the NumPy API by defining only a small +part of it. This is to ease the creation of new duck-arrays, by providing +default implementations of many functions that can be easily expressed in +terms of others, as well as a repository of utility functions that help in the +implementation of duck-arrays that most duck-arrays would require. This would +allow us to avoid designing entire protocols, e.g., a protocol for stacking +and concatenating would be replaced by simply implementing ``stack`` and/or +``concatenate`` and then providing default implementations for everything else +in that class. The same applies for transposing, and many other functions for +which protocols haven't been proposed, such as ``isin`` in terms of ``in1d``, +``setdiff1d`` in terms of ``unique``, and so on. + +It also allows one to override functions in a manner which +``__array_function__`` simply cannot, such as overriding ``np.einsum`` with the +version from the ``opt_einsum`` package, or Intel MKL overriding FFT, BLAS +or ``ufunc`` objects. They would define a backend with the appropriate +multimethods, and the user would select them via a ``with`` statement, or +registering them as a backend. + +The last benefit is a clear way to coerce to a given backend (via the +``coerce`` keyword in ``ua.set_backend``), and a protocol +for coercing not only arrays, but also ``dtype`` objects and ``ufunc`` objects +with similar ones from other libraries. This is due to the existence of actual, +third party dtype packages, and their desire to blend into the NumPy ecosystem +(see [6]_). This is a separate issue compared to the C-level dtype redesign +proposed in [7]_, it's about allowing third-party dtype implementations to +work with NumPy, much like third-party array implementations. These can provide +features such as, for example, units, jagged arrays or other such features that +are outside the scope of NumPy. + +Mixing NumPy and ``unumpy`` in the same file +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Normally, one would only want to import only one of ``unumpy`` or ``numpy``, +you would import it as ``np`` for familiarity. However, there may be situations +where one wishes to mix NumPy and the overrides, and there are a few ways to do +this, depending on the user's style:: + + from numpy import overridable as unp + import numpy as np + +or:: + + import numpy as np + + # Use unumpy via np.overridable + +Duck-array coercion +~~~~~~~~~~~~~~~~~~~ + +There are inherent problems about returning objects that are not NumPy arrays +from ``numpy.array`` or ``numpy.asarray``, particularly in the context of C/C++ +or Cython code that may get an object with a different memory layout than the +one it expects. However, we believe this problem may apply not only to these +two functions but all functions that return NumPy arrays. For this reason, +overrides are opt-in for the user, by using the submodule ``numpy.overridable`` +rather than ``numpy``. NumPy will continue to work unaffected by anything in +``numpy.overridable``. + +If the user wishes to obtain a NumPy array, there are two ways of doing it: + +1. Use ``numpy.asarray`` (the non-overridable version). +2. Use ``numpy.overridable.asarray`` with the NumPy backend set and coercion + enabled + +Aliases outside of the ``numpy.overridable`` namespace +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All functionality in ``numpy.random``, ``numpy.linalg`` and ``numpy.fft`` +will be aliased to their respective overridable versions inside +``numpy.overridable``. The reason for this is that there are alternative +implementations of RNGs (``mkl-random``), linear algebra routines (``eigen``, +``blis``) and FFT routines (``mkl-fft``, ``pyFFTW``) that need to operate on +``numpy.ndarray`` inputs, but still need the ability to switch behaviour. + +This is different from monkeypatching in a few different ways: + +* The caller-facing signature of the function is always the same, + so there is at least the loose sense of an API contract. Monkeypatching + does not provide this ability. +* There is the ability of locally switching the backend. +* It has been `suggested <http://numpy-discussion.10968.n7.nabble.com/NEP-31-Context-local-and-global-overrides-of-the-NumPy-API-tp47452p47472.html>`_ + that the reason that 1.17 hasn't landed in the Anaconda defaults channel is + due to the incompatibility between monkeypatching and ``__array_function__``, + as monkeypatching would bypass the protocol completely. +* Statements of the form ``from numpy import x; x`` and ``np.x`` would have + different results depending on whether the import was made before or + after monkeypatching happened. + +All this isn't possible at all with ``__array_function__`` or +``__array_ufunc__``. + +It has been formally realised (at least in part) that a backend system is +needed for this, in the `NumPy roadmap <https://numpy.org/neps/roadmap.html#other-functionality>`_. + +For ``numpy.random``, it's still necessary to make the C-API fit the one +proposed in `NEP-19 <https://numpy.org/neps/nep-0019-rng-policy.html>`_. +This is impossible for `mkl-random`, because then it would need to be +rewritten to fit that framework. The guarantees on stream +compatibility will be the same as before, but if there's a backend that affects +``numpy.random`` set, we make no guarantees about stream compatibility, and it +is up to the backend author to provide their own guarantees. + +Providing a way for implicit dispatch +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It has been suggested that the ability to dispatch methods which do not take +a dispatchable is needed, while guessing that backend from another dispatchable. + +As a concrete example, consider the following: + +.. code:: python + + with unumpy.determine_backend(array_like, np.ndarray): + unumpy.arange(len(array_like)) + +While this does not exist yet in ``uarray``, it is trivial to add it. The need for +this kind of code exists because one might want to have an alternative for the +proposed ``*_like`` functions, or the ``like=`` keyword argument. The need for these +exists because there are functions in the NumPy API that do not take a dispatchable +argument, but there is still the need to select a backend based on a different +dispatchable. + +The need for an opt-in module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The need for an opt-in module is realised because of a few reasons: + +* There are parts of the API (like `numpy.asarray`) that simply cannot be + overridden due to incompatibility concerns with C/Cython extensions, however, + one may want to coerce to a duck-array using ``asarray`` with a backend set. +* There are possible issues around an implicit option and monkeypatching, such + as those mentioned above. + +NEP 18 notes that this may require maintenance of two separate APIs. However, +this burden may be lessened by, for example, parametrizing all tests over +``numpy.overridable`` separately via a fixture. This also has the side-effect +of thoroughly testing it, unlike ``__array_function__``. We also feel that it +provides an oppurtunity to separate the NumPy API contract properly from the +implementation. + +Benefits to end-users and mixing backends +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Mixing backends is easy in ``uarray``, one only has to do: + +.. code:: python + + # Explicitly say which backends you want to mix + ua.register_backend(backend1) + ua.register_backend(backend2) + ua.register_backend(backend3) + + # Freely use code that mixes backends here. + +The benefits to end-users extend beyond just writing new code. Old code +(usually in the form of scripts) can be easily ported to different backends +by a simple import switch and a line adding the preferred backend. This way, +users may find it easier to port existing code to GPU or distributed computing. + +Related Work +------------ + +Other override mechanisms +~~~~~~~~~~~~~~~~~~~~~~~~~ + +* NEP-18, the ``__array_function__`` protocol. [2]_ +* NEP-13, the ``__array_ufunc__`` protocol. [3]_ +* NEP-30, the ``__duck_array__`` protocol. [9]_ + +Existing NumPy-like array implementations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Dask: https://dask.org/ +* CuPy: https://cupy.chainer.org/ +* PyData/Sparse: https://sparse.pydata.org/ +* Xnd: https://xnd.readthedocs.io/ +* Astropy's Quantity: https://docs.astropy.org/en/stable/units/ + +Existing and potential consumers of alternative arrays +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Dask: https://dask.org/ +* scikit-learn: https://scikit-learn.org/ +* xarray: https://xarray.pydata.org/ +* TensorLy: http://tensorly.org/ + +Existing alternate dtype implementations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``ndtypes``: https://ndtypes.readthedocs.io/en/latest/ +* Datashape: https://datashape.readthedocs.io +* Plum: https://plum-py.readthedocs.io/ + +Alternate implementations of parts of the NumPy API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``mkl_random``: https://github.com/IntelPython/mkl_random +* ``mkl_fft``: https://github.com/IntelPython/mkl_fft +* ``bottleneck``: https://github.com/pydata/bottleneck +* ``opt_einsum``: https://github.com/dgasmith/opt_einsum + +Implementation +-------------- + +The implementation of this NEP will require the following steps: + +* Implementation of ``uarray`` multimethods corresponding to the + NumPy API, including classes for overriding ``dtype``, ``ufunc`` + and ``array`` objects, in the ``unumpy`` repository, which are usually + very easy to create. +* Moving backends from ``unumpy`` into the respective array libraries. + +Maintenance can be eased by testing over ``{numpy, unumpy}`` via parameterized +tests. If a new argument is added to a method, the corresponding argument +extractor and replacer will need to be updated within ``unumpy``. + +A lot of argument extractors can be re-used from the existing implementation +of the ``__array_function__`` protocol, and the replacers can be usually +re-used across many methods. + +For the parts of the namespace which are going to be overridable by default, +the main method will need to be renamed and hidden behind a ``uarray`` multimethod. + +Default implementations are usually seen in the documentation using the words +"equivalent to", and thus, are easily available. + +``uarray`` Primer +~~~~~~~~~~~~~~~~~ + +**Note:** *This section will not attempt to go into too much detail about +uarray, that is the purpose of the uarray documentation.* [1]_ +*However, the NumPy community will have input into the design of +uarray, via the issue tracker.* + +``unumpy`` is the interface that defines a set of overridable functions +(multimethods) compatible with the numpy API. To do this, it uses the +``uarray`` library. ``uarray`` is a general purpose tool for creating +multimethods that dispatch to one of multiple different possible backend +implementations. In this sense, it is similar to the ``__array_function__`` +protocol but with the key difference that the backend is explicitly installed +by the end-user and not coupled into the array type. + +Decoupling the backend from the array type gives much more flexibility to +end-users and backend authors. For example, it is possible to: + +* override functions not taking arrays as arguments +* create backends out of source from the array type +* install multiple backends for the same array type + +This decoupling also means that ``uarray`` is not constrained to dispatching +over array-like types. The backend is free to inspect the entire set of +function arguments to determine if it can implement the function e.g. ``dtype`` +parameter dispatching. + +Defining backends +^^^^^^^^^^^^^^^^^ + +``uarray`` consists of two main protocols: ``__ua_convert__`` and +``__ua_function__``, called in that order, along with ``__ua_domain__``. +``__ua_convert__`` is for conversion and coercion. It has the signature +``(dispatchables, coerce)``, where ``dispatchables`` is an iterable of +``ua.Dispatchable`` objects and ``coerce`` is a boolean indicating whether or +not to force the conversion. ``ua.Dispatchable`` is a simple class consisting +of three simple values: ``type``, ``value``, and ``coercible``. +``__ua_convert__`` returns an iterable of the converted values, or +``NotImplemented`` in the case of failure. + +``__ua_function__`` has the signature ``(func, args, kwargs)`` and defines +the actual implementation of the function. It receives the function and its +arguments. Returning ``NotImplemented`` will cause a move to the default +implementation of the function if one exists, and failing that, the next +backend. + +Here is what will happen assuming a ``uarray`` multimethod is called: + +1. We canonicalise the arguments so any arguments without a default + are placed in ``*args`` and those with one are placed in ``**kwargs``. +2. We check the list of backends. + + a. If it is empty, we try the default implementation. + +3. We check if the backend's ``__ua_convert__`` method exists. If it exists: + + a. We pass it the output of the dispatcher, + which is an iterable of ``ua.Dispatchable`` objects. + b. We feed this output, along with the arguments, + to the argument replacer. ``NotImplemented`` means we move to 3 + with the next backend. + c. We store the replaced arguments as the new arguments. + +4. We feed the arguments into ``__ua_function__``, and return the output, and + exit if it isn't ``NotImplemented``. +5. If the default implementation exists, we try it with the current backend. +6. On failure, we move to 3 with the next backend. If there are no more + backends, we move to 7. +7. We raise a ``ua.BackendNotImplementedError``. + +Defining overridable multimethods +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To define an overridable function (a multimethod), one needs a few things: + +1. A dispatcher that returns an iterable of ``ua.Dispatchable`` objects. +2. A reverse dispatcher that replaces dispatchable values with the supplied + ones. +3. A domain. +4. Optionally, a default implementation, which can be provided in terms of + other multimethods. + +As an example, consider the following:: + + import uarray as ua + + def full_argreplacer(args, kwargs, dispatchables): + def full(shape, fill_value, dtype=None, order='C'): + return (shape, fill_value), dict( + dtype=dispatchables[0], + order=order + ) + + return full(*args, **kwargs) + + @ua.create_multimethod(full_argreplacer, domain="numpy") + def full(shape, fill_value, dtype=None, order='C'): + return (ua.Dispatchable(dtype, np.dtype),) + +A large set of examples can be found in the ``unumpy`` repository, [8]_. +This simple act of overriding callables allows us to override: + +* Methods +* Properties, via ``fget`` and ``fset`` +* Entire objects, via ``__get__``. + +Examples for NumPy +^^^^^^^^^^^^^^^^^^ + +A library that implements a NumPy-like API will use it in the following +manner (as an example):: + + import numpy.overridable as unp + _ua_implementations = {} + + __ua_domain__ = "numpy" + + def __ua_function__(func, args, kwargs): + fn = _ua_implementations.get(func, None) + return fn(*args, **kwargs) if fn is not None else NotImplemented + + def implements(ua_func): + def inner(func): + _ua_implementations[ua_func] = func + return func + + return inner + + @implements(unp.asarray) + def asarray(a, dtype=None, order=None): + # Code here + # Either this method or __ua_convert__ must + # return NotImplemented for unsupported types, + # Or they shouldn't be marked as dispatchable. + + # Provides a default implementation for ones and zeros. + @implements(unp.full) + def full(shape, fill_value, dtype=None, order='C'): + # Code here + +Alternatives +------------ + +The current alternative to this problem is a combination of NEP-18 [2]_, +NEP-13 [4]_ and NEP-30 [9]_ plus adding more protocols (not yet specified) +in addition to it. Even then, some parts of the NumPy API will remain +non-overridable, so it's a partial alternative. + +The main alternative to vendoring ``unumpy`` is to simply move it into NumPy +completely and not distribute it as a separate package. This would also achieve +the proposed goals, however we prefer to keep it a separate package for now, +for reasons already stated above. + +The third alternative is to move ``unumpy`` into the NumPy organisation and +develop it as a NumPy project. This will also achieve the said goals, and is +also a possibility that can be considered by this NEP. However, the act of +doing an extra ``pip install`` or ``conda install`` may discourage some users +from adopting this method. + +An alternative to requiring opt-in is mainly to *not* override ``np.asarray`` +and ``np.array``, and making the rest of the NumPy API surface overridable, +instead providing ``np.duckarray`` and ``np.asduckarray`` +as duck-array friendly alternatives that used the respective overrides. However, +this has the downside of adding a minor overhead to NumPy calls. + +Discussion +---------- + +* ``uarray`` blogpost: https://labs.quansight.org/blog/2019/07/uarray-update-api-changes-overhead-and-comparison-to-__array_function__/ +* The discussion section of NEP-18: https://numpy.org/neps/nep-0018-array-function-protocol.html#discussion +* NEP-22: https://numpy.org/neps/nep-0022-ndarray-duck-typing-overview.html +* Dask issue #4462: https://github.com/dask/dask/issues/4462 +* PR #13046: https://github.com/numpy/numpy/pull/13046 +* Dask issue #4883: https://github.com/dask/dask/issues/4883 +* Issue #13831: https://github.com/numpy/numpy/issues/13831 +* Discussion PR 1: https://github.com/hameerabbasi/numpy/pull/3 +* Discussion PR 2: https://github.com/hameerabbasi/numpy/pull/4 +* Discussion PR 3: https://github.com/numpy/numpy/pull/14389 + + +References and Footnotes +------------------------ + +.. [1] uarray, A general dispatch mechanism for Python: https://uarray.readthedocs.io + +.. [2] NEP 18 — A dispatch mechanism for NumPy’s high level array functions: https://numpy.org/neps/nep-0018-array-function-protocol.html + +.. [3] NEP 22 — Duck typing for NumPy arrays – high level overview: https://numpy.org/neps/nep-0022-ndarray-duck-typing-overview.html + +.. [4] NEP 13 — A Mechanism for Overriding Ufuncs: https://numpy.org/neps/nep-0013-ufunc-overrides.html + +.. [5] Reply to Adding to the non-dispatched implementation of NumPy methods: http://numpy-discussion.10968.n7.nabble.com/Adding-to-the-non-dispatched-implementation-of-NumPy-methods-tp46816p46874.html + +.. [6] Custom Dtype/Units discussion: http://numpy-discussion.10968.n7.nabble.com/Custom-Dtype-Units-discussion-td43262.html + +.. [7] The epic dtype cleanup plan: https://github.com/numpy/numpy/issues/2899 + +.. [8] unumpy: NumPy, but implementation-independent: https://unumpy.readthedocs.io + +.. [9] NEP 30 — Duck Typing for NumPy Arrays - Implementation: https://www.numpy.org/neps/nep-0030-duck-array-protocol.html + +.. [10] http://scipy.github.io/devdocs/fft.html#backend-control + + +Copyright +--------- + +This document has been placed in the public domain. diff --git a/doc/neps/nep-0032-remove-financial-functions.rst b/doc/neps/nep-0032-remove-financial-functions.rst new file mode 100644 index 000000000..a78b11fea --- /dev/null +++ b/doc/neps/nep-0032-remove-financial-functions.rst @@ -0,0 +1,214 @@ +================================================== +NEP 32 — Remove the financial functions from NumPy +================================================== + +:Author: Warren Weckesser <warren.weckesser@gmail.com> +:Status: Accepted +:Type: Standards Track +:Created: 2019-08-30 +:Resolution: https://mail.python.org/pipermail/numpy-discussion/2019-September/080074.html + + +Abstract +-------- + +We propose deprecating and ultimately removing the financial functions [1]_ +from NumPy. The functions will be moved to an independent repository, +and provided to the community as a separate package with the name +``numpy_financial``. + + +Motivation and scope +-------------------- + +The NumPy financial functions [1]_ are the 10 functions ``fv``, ``ipmt``, +``irr``, ``mirr``, ``nper``, ``npv``, ``pmt``, ``ppmt``, ``pv`` and ``rate``. +The functions provide elementary financial calculations such as future value, +net present value, etc. These functions were added to NumPy in 2008 [2]_. + +In May, 2009, a request by Joe Harrington to add a function called ``xirr`` to +the financial functions triggered a long thread about these functions [3]_. +One important point that came up in that thread is that a "real" financial +library must be able to handle real dates. The NumPy financial functions do +not work with actual dates or calendars. The preference for a more capable +library independent of NumPy was expressed several times in that thread. + +In June, 2009, D. L. Goldsmith expressed concerns about the correctness of the +implementations of some of the financial functions [4]_. It was suggested then +to move the financial functions out of NumPy to an independent package. + +In a GitHub issue in 2013 [5]_, Nathaniel Smith suggested moving the financial +functions from the top-level namespace to ``numpy.financial``. He also +suggested giving the functions better names. Responses at that time included +the suggestion to deprecate them and move them from NumPy to a separate +package. This issue is still open. + +Later in 2013 [6]_, it was suggested on the mailing list that these functions +be removed from NumPy. + +The arguments for the removal of these functions from NumPy: + +* They are too specialized for NumPy. +* They are not actually useful for "real world" financial calculations, because + they do not handle real dates and calendars. +* The definition of "correctness" for some of these functions seems to be a + matter of convention, and the current NumPy developers do not have the + background to judge their correctness. +* There has been little interest among past and present NumPy developers + in maintaining these functions. + +The main arguments for keeping the functions in NumPy are: + +* Removing these functions will be disruptive for some users. Current users + will have to add the new ``numpy_financial`` package to their dependencies, + and then modify their code to use the new package. +* The functions provided, while not "industrial strength", are apparently + similar to functions provided by spreadsheets and some calculators. Having + them available in NumPy makes it easier for some developers to migrate their + software to Python and NumPy. + +It is clear from comments in the mailing list discussions and in the GitHub +issues that many current NumPy developers believe the benefits of removing +the functions outweigh the costs. For example, from [5]_:: + + The financial functions should probably be part of a separate package + -- Charles Harris + + If there's a better package we can point people to we could just deprecate + them and then remove them entirely... I'd be fine with that too... + -- Nathaniel Smith + + +1 to deprecate them. If no other package exists, it can be created if + someone feels the need for that. + -- Ralf Gommers + + I feel pretty strongly that we should deprecate these. If nobody on numpy’s + core team is interested in maintaining them, then it is purely a drag on + development for NumPy. + -- Stephan Hoyer + +And from the 2013 mailing list discussion, about removing the functions from +NumPy:: + + I am +1 as well, I don't think they should have been included in the first + place. + -- David Cournapeau + +But not everyone was in favor of removal:: + + The fin routines are tiny and don't require much maintenance once + written. If we made an effort (putting up pages with examples of common + financial calculations and collecting those under a topical web page, + then linking to that page from various places and talking it up), I + would think they could attract users looking for a free way to play with + financial scenarios. [...] + So, I would say we keep them. If ours are not the best, we should bring + them up to snuff. + -- Joe Harrington + +For an idea of the maintenance burden of the financial functions, one can +look for all the GitHub issues [7]_ and pull requests [8]_ that have the tag +``component: numpy.lib.financial``. + +One method for measuring the effect of removing these functions is to find +all the packages on GitHub that use them. Such a search can be performed +with the ``python-api-inspect`` service [9]_. A search for all uses of the +NumPy financial functions finds just eight repositories. (See the comments +in [5]_ for the actual SQL query.) + + +Implementation +-------------- + +* Create a new Python package, ``numpy_financial``, to be maintained in the + top-level NumPy github organization. This repository will contain the + definitions and unit tests for the financial functions. The package will + be added to PyPI so it can be installed with ``pip``. +* Deprecate the financial functions in the ``numpy`` namespace, beginning in + NumPy version 1.18. Remove the financial functions from NumPy version 1.20. + + +Backward compatibility +---------------------- + +The removal of these functions breaks backward compatibility, as explained +earlier. The effects are mitigated by providing the ``numpy_financial`` +library. + + +Alternatives +------------ + +The following alternatives were mentioned in [5]_: + +* *Maintain the functions as they are (i.e. do nothing).* + A review of the history makes clear that this is not the preference of many + NumPy developers. A recurring comment is that the functions simply do not + belong in NumPy. When that sentiment is combined with the history of bug + reports and the ongoing questions about the correctness of the functions, the + conclusion is that the cleanest solution is deprecation and removal. +* *Move the functions from the ``numpy`` namespace to ``numpy.financial``.* + This was the initial suggestion in [5]_. Such a change does not address the + maintenance issues, and doesn't change the misfit that many developers see + between these functions and NumPy. It causes disruption for the current + users of these functions without addressing what many developers see as the + fundamental problem. + + +Discussion +---------- + +Links to past mailing list discussions, and to relevant GitHub issues and pull +requests, have already been given. The announcement of this NEP was made on +the NumPy-Discussion mailing list on 3 September 2019 [10]_, and on the +PyData mailing list on 8 September 2019 [11]_. The formal proposal to accept +the NEP was made on 19 September 2019 [12]_; a notification was also sent to +PyData (same thread as [11]_). There have been no substantive objections. + + +References and footnotes +------------------------ + +.. [1] Financial functions, + https://numpy.org/doc/1.17/reference/routines.financial.html + +.. [2] Numpy-discussion mailing list, "Simple financial functions for NumPy", + https://mail.python.org/pipermail/numpy-discussion/2008-April/032353.html + +.. [3] Numpy-discussion mailing list, "add xirr to numpy financial functions?", + https://mail.python.org/pipermail/numpy-discussion/2009-May/042645.html + +.. [4] Numpy-discussion mailing list, "Definitions of pv, fv, nper, pmt, and rate", + https://mail.python.org/pipermail/numpy-discussion/2009-June/043188.html + +.. [5] Get financial functions out of main namespace, + https://github.com/numpy/numpy/issues/2880 + +.. [6] Numpy-discussion mailing list, "Deprecation of financial routines", + https://mail.python.org/pipermail/numpy-discussion/2013-August/067409.html + +.. [7] ``component: numpy.lib.financial`` issues, + https://github.com/numpy/numpy/issues?utf8=%E2%9C%93&q=is%3Aissue+label%3A%22component%3A+numpy.lib.financial%22+ + +.. [8] ``component: numpy.lib.financial`` pull requests, + https://github.com/numpy/numpy/pulls?utf8=%E2%9C%93&q=is%3Apr+label%3A%22component%3A+numpy.lib.financial%22+ + +.. [9] Quansight-Labs/python-api-inspect, + https://github.com/Quansight-Labs/python-api-inspect/ + +.. [10] Numpy-discussion mailing list, "NEP 32: Remove the financial functions + from NumPy" + https://mail.python.org/pipermail/numpy-discussion/2019-September/079965.html + +.. [11] PyData mailing list (pydata@googlegroups.com), "NumPy proposal to + remove the financial functions. + https://mail.google.com/mail/u/0/h/1w0mjgixc4rpe/?&th=16d5c38be45f77c4&q=nep+32&v=c&s=q + +.. [12] Numpy-discussion mailing list, "Proposal to accept NEP 32: Remove the + financial functions from NumPy" + https://mail.python.org/pipermail/numpy-discussion/2019-September/080074.html + +Copyright +--------- + +This document has been placed in the public domain. diff --git a/doc/neps/nep-0034.rst b/doc/neps/nep-0034.rst new file mode 100644 index 000000000..4863bad86 --- /dev/null +++ b/doc/neps/nep-0034.rst @@ -0,0 +1,141 @@ +=========================================================== +NEP 34 — Disallow inferring ``dtype=object`` from sequences +=========================================================== + +:Author: Matti Picus +:Status: Accepted +:Type: Standards Track +:Created: 2019-10-10 +:Resolution: https://mail.python.org/pipermail/numpy-discussion/2019-October/080200.html + +Abstract +-------- + +When users create arrays with sequences-of-sequences, they sometimes err in +matching the lengths of the nested sequences_, commonly called "ragged +arrays". Here we will refer to them as ragged nested sequences. Creating such +arrays via ``np.array([<ragged_nested_sequence>])`` with no ``dtype`` keyword +argument will today default to an ``object``-dtype array. Change the behaviour to +raise a ``ValueError`` instead. + +Motivation and Scope +-------------------- + +Users who specify lists-of-lists when creating a `numpy.ndarray` via +``np.array`` may mistakenly pass in lists of different lengths. Currently we +accept this input and automatically create an array with ``dtype=object``. This +can be confusing, since it is rarely what is desired. Changing the automatic +dtype detection to never return ``object`` for ragged nested sequences (defined as a +recursive sequence of sequences, where not all the sequences on the same +level have the same length) will force users who actually wish to create +``object`` arrays to specify that explicitly. Note that ``lists``, ``tuples``, +and ``nd.ndarrays`` are all sequences [0]_. See for instance `issue 5303`_. + +Usage and Impact +---------------- + +After this change, array creation with ragged nested sequences must explicitly +define a dtype: + + >>> np.array([[1, 2], [1]]) + ValueError: cannot guess the desired dtype from the input + + >>> np.array([[1, 2], [1]], dtype=object) + # succeeds, with no change from current behaviour + +The deprecation will affect any call that internally calls ``np.asarray``. For +instance, the ``assert_equal`` family of functions calls ``np.asarray``, so +users will have to change code like:: + + np.assert_equal(a, [[1, 2], 3]) + +to:: + + np.assert_equal(a, np.array([[1, 2], 3], dtype=object)) + +Detailed description +-------------------- + +To explicitly set the shape of the object array, since it is sometimes hard to +determine what shape is desired, one could use: + + >>> arr = np.empty(correct_shape, dtype=object) + >>> arr[...] = values + +We will also reject mixed sequences of non-sequence and sequence, for instance +all of these will be rejected: + + >>> arr = np.array([np.arange(10), [10]]) + >>> arr = np.array([[range(3), range(3), range(3)], [range(3), 0, 0]]) + +Related Work +------------ + +`PR 14341`_ tried to raise an error when ragged nested sequences were specified +with a numeric dtype ``np.array, [[1], [2, 3]], dtype=int)`` but failed due to +false-positives, for instance ``np.array([1, np.array([5])], dtype=int)``. + +.. _`PR 14341`: https://github.com/numpy/numpy/pull/14341 + +Implementation +-------------- + +The code to be changed is inside ``PyArray_GetArrayParamsFromObject`` and the +internal ``discover_dimentions`` function. See `PR 14794`_. + +Backward compatibility +---------------------- + +Anyone depending on creating object arrays from ragged nested sequences will +need to modify their code. There will be a deprecation period during which the +current behaviour will emit a ``DeprecationWarning``. + +Alternatives +------------ + +- We could continue with the current situation. + +- It was also suggested to add a kwarg ``depth`` to array creation, or perhaps + to add another array creation API function ``ragged_array_object``. The goal + was to eliminate the ambiguity in creating an object array from ``array([[1, + 2], [1]], dtype=object)``: should the returned array have a shape of + ``(1,)``, or ``(2,)``? This NEP does not deal with that issue, and only + deprecates the use of ``array`` with no ``dtype=object`` for ragged nested + sequences. Users of ragged nested sequences may face another deprecation + cycle in the future. Rationale: we expect that there are very few users who + intend to use ragged arrays like that, this was never intended as a use case + of NumPy arrays. Users are likely better off with `another library`_ or just + using list of lists. + +- It was also suggested to deprecate all automatic creation of ``object``-dtype + arrays, which would require adding an explicit ``dtype=object`` for something + like ``np.array([Decimal(10), Decimal(10)])``. This too is out of scope for + the current NEP. Rationale: it's harder to asses the impact of this larger + change, we're not sure how many users this may impact. + +Discussion +---------- + +Comments to `issue 5303`_ indicate this is unintended behaviour as far back as +2014. Suggestions to change it have been made in the ensuing years, but none +have stuck. The WIP implementation in `PR 14794`_ seems to point to the +viability of this approach. + +References and Footnotes +------------------------ + +.. _`issue 5303`: https://github.com/numpy/numpy/issues/5303 +.. _sequences: https://docs.python.org/3.7/glossary.html#term-sequence +.. _`PR 14794`: https://github.com/numpy/numpy/pull/14794 +.. _`another library`: https://github.com/scikit-hep/awkward-array + +.. [0] ``np.ndarrays`` are not recursed into, rather their shape is used + directly. This will not emit warnings:: + + ragged = np.array([[1], [1, 2, 3]], dtype=object) + np.array([ragged, ragged]) # no dtype needed + +Copyright +--------- + +This document has been placed in the public domain. diff --git a/doc/neps/nep-template.rst b/doc/neps/nep-template.rst index e869ebae3..42f717c7a 100644 --- a/doc/neps/nep-template.rst +++ b/doc/neps/nep-template.rst @@ -1,6 +1,6 @@ -============================= -NEP Template and Instructions -============================= +================================= +NEP X — Template and Instructions +================================= :Author: <list of authors' real names and optionally, email addresses> :Status: <Draft | Active | Accepted | Deferred | Rejected | Withdrawn | Final | Superseded> @@ -8,19 +8,57 @@ NEP Template and Instructions :Created: <date created on, in yyyy-mm-dd format> :Resolution: <url> (required for Accepted | Rejected | Withdrawn) + Abstract -------- The abstract should be a short description of what the NEP will achieve. +Note that the — in the title is an elongated dash, not -. + +Motivation and Scope +-------------------- + +This section describes the need for the proposed change. It should describe +the existing problem, who it affects, what it is trying to solve, and why. +This section should explicitly address the scope of and key requirements for +the proposed change. + +Usage and Impact +---------------- + +This section describes how users of NumPy will use features described in this +NEP. It should be comprised mainly of code examples that wouldn't be possible +without acceptance and implementation of this NEP, as well as the impact the +proposed changes would have on the ecosystem. This section should be written +from the perspective of the users of NumPy, and the benefits it will provide +them; and as such, it should include implementation details only if +necessary to explain the functionality. + +Backward compatibility +---------------------- + +This section describes the ways in which the NEP breaks backward compatibility. + +The mailing list post will contain the NEP up to and including this section. +Its purpose is to provide a high-level summary to users who are not interested +in detailed technical discussion, but may have opinions around, e.g., usage and +impact. Detailed description -------------------- -This section describes the need for the NEP. It should describe the existing -problem that it is trying to solve and why this NEP makes the situation better. -It should include examples of how the new functionality would be used and -perhaps some use cases. +This section should provide a detailed description of the proposed change. +It should include examples of how the new functionality would be used, +intended use-cases and pseudo-code illustrating its use. + + +Related Work +------------ + +This section should list relevant and/or similar technologies, possibly in other +libraries. It does not need to be comprehensive, just list the major examples of +prior and relevant art. Implementation @@ -28,20 +66,14 @@ Implementation This section lists the major steps required to implement the NEP. Where possible, it should be noted where one step is dependent on another, and which -steps may be optionally omitted. Where it makes sense, each step should -include a link related pull requests as the implementation progresses. +steps may be optionally omitted. Where it makes sense, each step should +include a link to related pull requests as the implementation progresses. Any pull requests or development branches containing work on this NEP should be linked to from here. (A NEP does not need to be implemented in a single pull request if it makes sense to implement it in discrete phases). -Backward compatibility ----------------------- - -This section describes the ways in which the NEP breaks backward compatibility. - - Alternatives ------------ diff --git a/doc/neps/roadmap.rst b/doc/neps/roadmap.rst index 2ec0b7520..c5abc5f25 100644 --- a/doc/neps/roadmap.rst +++ b/doc/neps/roadmap.rst @@ -42,7 +42,7 @@ improve the dtype system. - One of these should probably be the default for text data. The current behavior on Python 3 is neither efficient nor user friendly. -- `np.int` should not be platform dependent +- ``np.dtype(int)`` should not be platform dependent - Better coercion for string + number Performance diff --git a/doc/newdtype_example/example.py b/doc/newdtype_example/example.py deleted file mode 100644 index 6be9caa75..000000000 --- a/doc/newdtype_example/example.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import division, absolute_import, print_function - -import floatint.floatint as ff -import numpy as np - -# Setting using array is hard because -# The parser doesn't stop at tuples always -# So, the setitem code will be called with scalars on the -# wrong shaped array. -# But we can get a view as an ndarray of the given type: -g = np.array([1, 2, 3, 4, 5, 6, 7, 8]).view(ff.floatint_type) - -# Now, the elements will be the scalar type associated -# with the ndarray. -print(g[0]) -print(type(g[1])) - -# Now, you need to register ufuncs and more arrfuncs to do useful things... diff --git a/doc/newdtype_example/floatint.c b/doc/newdtype_example/floatint.c deleted file mode 100644 index 0cc198388..000000000 --- a/doc/newdtype_example/floatint.c +++ /dev/null @@ -1,152 +0,0 @@ - -#include "Python.h" -#include "structmember.h" /* for offset of macro if needed */ -#include "numpy/arrayobject.h" - - -/* Use a Python float as the canonical type being added -*/ - -typedef struct _floatint { - PyObject_HEAD - npy_int32 first; - npy_int32 last; -} PyFloatIntObject; - -static PyTypeObject PyFloatInt_Type = { - PyObject_HEAD_INIT(NULL) - 0, /*ob_size*/ - "floatint.floatint", /*tp_name*/ - sizeof(PyFloatIntObject), /*tp_basicsize*/ -}; - -static PyArray_ArrFuncs _PyFloatInt_Funcs; - -#define _ALIGN(type) offsetof(struct {char c; type v;},v) - -/* The scalar-type */ - -static PyArray_Descr _PyFloatInt_Dtype = { - PyObject_HEAD_INIT(NULL) - &PyFloatInt_Type, - 'f', - '0', - '=', - 0, - 0, - sizeof(double), - _ALIGN(double), - NULL, - NULL, - NULL, - &_PyFloatInt_Funcs -}; - -static void -twoint_copyswap(void *dst, void *src, int swap, void *arr) -{ - if (src != NULL) { - memcpy(dst, src, sizeof(double)); - } - - if (swap) { - register char *a, *b, c; - a = (char *)dst; - b = a + 7; - c = *a; *a++ = *b; *b-- = c; - c = *a; *a++ = *b; *b-- = c; - c = *a; *a++ = *b; *b-- = c; - c = *a; *a++ = *b; *b = c; - } -} - -static PyObject * -twoint_getitem(char *ip, PyArrayObject *ap) { - npy_int32 a[2]; - - if ((ap==NULL) || PyArray_ISBEHAVED_RO(ap)) { - a[0] = *((npy_int32 *)ip); - a[1] = *((npy_int32 *)ip + 1); - } - else { - ap->descr->f->copyswap(a, ip, !PyArray_ISNOTSWAPPED(ap), ap); - } - return Py_BuildValue("(ii)", a[0], a[1]); -} - -static int -twoint_setitem(PyObject *op, char *ov, PyArrayObject *ap) { - npy_int32 a[2]; - - if (!PyTuple_Check(op)) { - PyErr_SetString(PyExc_TypeError, "must be a tuple"); - return -1; - } - if (!PyArg_ParseTuple(op, "ii", a, a+1)) return -1; - - if (ap == NULL || PyArray_ISBEHAVED(ap)) { - memcpy(ov, a, sizeof(double)); - } - else { - ap->descr->f->copyswap(ov, a, !PyArray_ISNOTSWAPPED(ap), ap); - } - return 0; -} - -static PyArray_Descr * _register_dtype(void) -{ - int userval; - PyArray_InitArrFuncs(&_PyFloatInt_Funcs); - /* Add copyswap, - nonzero, getitem, setitem*/ - _PyFloatInt_Funcs.copyswap = twoint_copyswap; - _PyFloatInt_Funcs.getitem = (PyArray_GetItemFunc *)twoint_getitem; - _PyFloatInt_Funcs.setitem = (PyArray_SetItemFunc *)twoint_setitem; - _PyFloatInt_Dtype.ob_type = &PyArrayDescr_Type; - - userval = PyArray_RegisterDataType(&_PyFloatInt_Dtype); - return PyArray_DescrFromType(userval); -} - - -/* Initialization function for the module (*must* be called init<name>) */ - -PyMODINIT_FUNC initfloatint(void) { - PyObject *m, *d; - PyArray_Descr *dtype; - - /* Create the module and add the functions */ - m = Py_InitModule("floatint", NULL); - - /* Import the array objects */ - import_array(); - - - /* Initialize the new float type */ - - /* Add some symbolic constants to the module */ - d = PyModule_GetDict(m); - - if (PyType_Ready(&PyFloat_Type) < 0) return; - PyFloatInt_Type.tp_base = &PyFloat_Type; - /* This is only needed because we are sub-typing the - Float type and must pre-set some function pointers - to get PyType_Ready to fill in the rest. - */ - PyFloatInt_Type.tp_alloc = PyType_GenericAlloc; - PyFloatInt_Type.tp_new = PyFloat_Type.tp_new; - PyFloatInt_Type.tp_dealloc = PyFloat_Type.tp_dealloc; - PyFloatInt_Type.tp_free = PyObject_Del; - if (PyType_Ready(&PyFloatInt_Type) < 0) return; - /* End specific code */ - - - dtype = _register_dtype(); - Py_XINCREF(dtype); - if (dtype != NULL) { - PyDict_SetItemString(d, "floatint_type", (PyObject *)dtype); - } - Py_INCREF(&PyFloatInt_Type); - PyDict_SetItemString(d, "floatint", (PyObject *)&PyFloatInt_Type); - return; -} diff --git a/doc/newdtype_example/floatint/__init__.py b/doc/newdtype_example/floatint/__init__.py deleted file mode 100644 index 1d0f69b67..000000000 --- a/doc/newdtype_example/floatint/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import division, absolute_import, print_function diff --git a/doc/newdtype_example/setup.py b/doc/newdtype_example/setup.py deleted file mode 100644 index d7ab040a1..000000000 --- a/doc/newdtype_example/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import division, print_function - -from numpy.distutils.core import setup - -def configuration(parent_package = '', top_path=None): - from numpy.distutils.misc_util import Configuration - config = Configuration('floatint', parent_package, top_path) - - config.add_extension('floatint', - sources = ['floatint.c']) - return config - -setup(configuration=configuration) diff --git a/doc/records.rst.txt b/doc/records.rst.txt index a608880d7..3c0d55216 100644 --- a/doc/records.rst.txt +++ b/doc/records.rst.txt @@ -50,7 +50,7 @@ New possibilities for the "data-type" **Dictionary (keys "names", "titles", and "formats")** - This will be converted to a ``PyArray_VOID`` type with corresponding + This will be converted to a ``NPY_VOID`` type with corresponding fields parameter (the formats list will be converted to actual ``PyArray_Descr *`` objects). @@ -58,10 +58,10 @@ New possibilities for the "data-type" **Objects (anything with an .itemsize and .fields attribute)** If its an instance of (a sub-class of) void type, then a new ``PyArray_Descr*`` structure is created corresponding to its - typeobject (and ``PyArray_VOID``) typenumber. If the type is + typeobject (and ``NPY_VOID``) typenumber. If the type is registered, then the registered type-number is used. - Otherwise a new ``PyArray_VOID PyArray_Descr*`` structure is created + Otherwise a new ``NPY_VOID PyArray_Descr*`` structure is created and filled ->elsize and ->fields filled in appropriately. The itemsize attribute must return a number > 0. The fields diff --git a/doc/release/1.16.6-notes.rst b/doc/release/1.16.6-notes.rst deleted file mode 100644 index 0aeba3cd3..000000000 --- a/doc/release/1.16.6-notes.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. currentmodule:: numpy - -========================== -NumPy 1.16.6 Release Notes -========================== - -The NumPy 1.16.6 release fixes bugs reported against the 1.16.5 release, and -also backports several enhancements from master that seem appropriate for a -release series that is the last to support Python 2.7. The wheels on PyPI are -linked with OpenBLAS v0.3.7, which should fix errors on Skylake series -cpus. - -Downstream developers building this release should use Cython >= 0.29.2 and, if -using OpenBLAS, OpenBLAS >= v0.3.7. The supported Python versions are 2.7 and -3.5-3.7. - -Highlights -========== - -- The ``np.testing.utils`` functions have been updated from 1.19.0-dev0. - This improves the function documentation and error messages as well - extending the ``assert_array_compare`` function to additional types. - - -New functions -============= - -Allow matmul (`@` operator) to work with object arrays. -------------------------------------------------------- -This is an enhancement that was added in NumPy 1.17 and seems reasonable to -include in the LTS 1.16 release series. - - -Compatibility notes -=================== - -Fix regression in matmul (`@` operator) for boolean types ---------------------------------------------------------- -Booleans were being treated as integers rather than booleans, -which was a regression from previous behavior. - - -Improvements -============ - -Array comparison assertions include maximum differences -------------------------------------------------------- -Error messages from array comparison tests such as ``testing.assert_allclose`` -now include "max absolute difference" and "max relative difference," in -addition to the previous "mismatch" percentage. This information makes it -easier to update absolute and relative error tolerances. - -Contributors -============ - -A total of 10 people contributed to this release. - -* CakeWithSteak -* Charles Harris -* Chris Burr -* Eric Wieser -* Fernando Saravia -* Lars Grueter -* Matti Picus -* Maxwell Aladago -* Qiming Sun -* Warren Weckesser - -Pull requests merged -==================== - -A total of 14 pull requests were merged for this release. - -* `#14211 <https://github.com/numpy/numpy/pull/14211>`__: BUG: Fix uint-overflow if padding with linear_ramp and negative... -* `#14275 <https://github.com/numpy/numpy/pull/14275>`__: BUG: fixing to allow unpickling of PY3 pickles from PY2 -* `#14340 <https://github.com/numpy/numpy/pull/14340>`__: BUG: Fix misuse of .names and .fields in various places (backport... -* `#14423 <https://github.com/numpy/numpy/pull/14423>`__: BUG: test, fix regression in converting to ctypes. -* `#14434 <https://github.com/numpy/numpy/pull/14434>`__: BUG: Fixed maximum relative error reporting in assert_allclose -* `#14509 <https://github.com/numpy/numpy/pull/14509>`__: BUG: Fix regression in boolean matmul. -* `#14686 <https://github.com/numpy/numpy/pull/14686>`__: BUG: properly define PyArray_DescrCheck -* `#14853 <https://github.com/numpy/numpy/pull/14853>`__: BLD: add 'apt update' to shippable -* `#14854 <https://github.com/numpy/numpy/pull/14854>`__: BUG: Fix _ctypes class circular reference. (#13808) -* `#14856 <https://github.com/numpy/numpy/pull/14856>`__: BUG: Fix `np.einsum` errors on Power9 Linux and z/Linux -* `#14863 <https://github.com/numpy/numpy/pull/14863>`__: BLD: Prevent -flto from optimising long double representation... -* `#14864 <https://github.com/numpy/numpy/pull/14864>`__: BUG: lib: Fix histogram problem with signed integer arrays. -* `#15172 <https://github.com/numpy/numpy/pull/15172>`__: ENH: Backport improvements to testing functions. -* `#15191 <https://github.com/numpy/numpy/pull/15191>`__: REL: Prepare for 1.16.6 release. diff --git a/doc/release/1.17.5-notes.rst b/doc/release/1.17.5-notes.rst deleted file mode 100644 index 7a09b7fab..000000000 --- a/doc/release/1.17.5-notes.rst +++ /dev/null @@ -1,45 +0,0 @@ -.. currentmodule:: numpy - -========================== -NumPy 1.17.5 Release Notes -========================== - -This release contains fixes for bugs reported against NumPy 1.17.4 along with -some build improvements. The Python versions supported in this release -are 3.5-3.8. - -Downstream developers should use Cython >= 0.29.13 for Python 3.8 support and -OpenBLAS >= 3.7 to avoid errors on the Skylake architecture. - -It is recommended that developers interested in the new random bit generators -upgrade to the NumPy 1.18.x series, as it has updated documentation and -many small improvements. - - -Contributors -============ - -A total of 6 people contributed to this release. People with a "+" by their -names contributed a patch for the first time. - -* Charles Harris -* Eric Wieser -* Ilhan Polat -* Matti Picus -* Michael Hudson-Doyle -* Ralf Gommers - - -Pull requests merged -==================== - -A total of 8 pull requests were merged for this release. - -* `#14593 <https://github.com/numpy/numpy/pull/14593>`__: MAINT: backport Cython API cleanup to 1.17.x, remove docs -* `#14937 <https://github.com/numpy/numpy/pull/14937>`__: BUG: fix integer size confusion in handling array's ndmin argument -* `#14939 <https://github.com/numpy/numpy/pull/14939>`__: BUILD: remove SSE2 flag from numpy.random builds -* `#14993 <https://github.com/numpy/numpy/pull/14993>`__: MAINT: Added Python3.8 branch to dll lib discovery -* `#15038 <https://github.com/numpy/numpy/pull/15038>`__: BUG: Fix refcounting in ufunc object loops -* `#15067 <https://github.com/numpy/numpy/pull/15067>`__: BUG: Exceptions tracebacks are dropped -* `#15175 <https://github.com/numpy/numpy/pull/15175>`__: ENH: Backport improvements to testing functions. -* `#15213 <https://github.com/numpy/numpy/pull/15213>`__: REL: Prepare for the NumPy 1.17.5 release. diff --git a/doc/release/time_based_proposal.rst b/doc/release/time_based_proposal.rst deleted file mode 100644 index 2eb13562d..000000000 --- a/doc/release/time_based_proposal.rst +++ /dev/null @@ -1,129 +0,0 @@ -.. vim:syntax=rst - -Introduction -============ - -This document proposes some enhancements for numpy and scipy releases. -Successive numpy and scipy releases are too far apart from a time point of -view - some people who are in the numpy release team feel that it cannot -improve without a bit more formal release process. The main proposal is to -follow a time-based release, with expected dates for code freeze, beta and rc. -The goal is two folds: make release more predictable, and move the code forward. - -Rationale -========= - -Right now, the release process of numpy is relatively organic. When some -features are there, we may decide to make a new release. Because there is not -fixed schedule, people don't really know when new features and bug fixes will -go into a release. More significantly, having an expected release schedule -helps to *coordinate* efforts: at the beginning of a cycle, everybody can jump -in and put new code, even break things if needed. But after some point, only -bug fixes are accepted: this makes beta and RC releases much easier; calming -things down toward the release date helps focusing on bugs and regressions - -Proposal -======== - -Time schedule -------------- - -The proposed schedule is to release numpy every 9 weeks - the exact period can -be tweaked if it ends up not working as expected. There will be several stages -for the cycle: - - * Development: anything can happen (by anything, we mean as currently - done). The focus is on new features, refactoring, etc... - - * Beta: no new features. No bug fixing which requires heavy changes. - regression fixes which appear on supported platforms and were not - caught earlier. - - * Polish/RC: only docstring changes and blocker regressions are allowed. - -The schedule would be as follows: - - +------+-----------------+-----------------+------------------+ - | Week | 1.3.0 | 1.4.0 | Release time | - +======+=================+=================+==================+ - | 1 | Development | | | - +------+-----------------+-----------------+------------------+ - | 2 | Development | | | - +------+-----------------+-----------------+------------------+ - | 3 | Development | | | - +------+-----------------+-----------------+------------------+ - | 4 | Development | | | - +------+-----------------+-----------------+------------------+ - | 5 | Development | | | - +------+-----------------+-----------------+------------------+ - | 6 | Development | | | - +------+-----------------+-----------------+------------------+ - | 7 | Beta | | | - +------+-----------------+-----------------+------------------+ - | 8 | Beta | | | - +------+-----------------+-----------------+------------------+ - | 9 | Beta | | 1.3.0 released | - +------+-----------------+-----------------+------------------+ - | 10 | Polish | Development | | - +------+-----------------+-----------------+------------------+ - | 11 | Polish | Development | | - +------+-----------------+-----------------+------------------+ - | 12 | Polish | Development | | - +------+-----------------+-----------------+------------------+ - | 13 | Polish | Development | | - +------+-----------------+-----------------+------------------+ - | 14 | | Development | | - +------+-----------------+-----------------+------------------+ - | 15 | | Development | | - +------+-----------------+-----------------+------------------+ - | 16 | | Beta | | - +------+-----------------+-----------------+------------------+ - | 17 | | Beta | | - +------+-----------------+-----------------+------------------+ - | 18 | | Beta | 1.4.0 released | - +------+-----------------+-----------------+------------------+ - -Each stage can be defined as follows: - - +------------------+-------------+----------------+----------------+ - | | Development | Beta | Polish | - +==================+=============+================+================+ - | Python Frozen | | slushy | Y | - +------------------+-------------+----------------+----------------+ - | Docstring Frozen | | slushy | thicker slush | - +------------------+-------------+----------------+----------------+ - | C code Frozen | | thicker slush | thicker slush | - +------------------+-------------+----------------+----------------+ - -Terminology: - - * slushy: you can change it if you beg the release team and it's really - important and you coordinate with docs/translations; no "big" - changes. - - * thicker slush: you can change it if it's an open bug marked - showstopper for the Polish release, you beg the release team, the - change is very very small yet very very important, and you feel - extremely guilty about your transgressions. - -The different frozen states are intended to be gradients. The exact meaning is -decided by the release manager: he has the last word on what's go in, what -doesn't. The proposed schedule means that there would be at most 12 weeks -between putting code into the source code repository and being released. - -Release team ------------- - -For every release, there would be at least one release manager. We propose to -rotate the release manager: rotation means it is not always the same person -doing the dirty job, and it should also keep the release manager honest. - -References -========== - - * Proposed schedule for Gnome from Havoc Pennington (one of the core - GTK and Gnome manager): - https://mail.gnome.org/archives/gnome-hackers/2002-June/msg00041.html - The proposed schedule is heavily based on this email - - * https://wiki.gnome.org/ReleasePlanning/Freezes diff --git a/doc/release/upcoming_changes/14498.changes.rst b/doc/release/upcoming_changes/14498.changes.rst deleted file mode 100644 index fd784e289..000000000 --- a/doc/release/upcoming_changes/14498.changes.rst +++ /dev/null @@ -1,7 +0,0 @@ -Remove ``numpy.random.entropy`` without a deprecation ------------------------------------------------------ - -``numpy.random.entropy`` was added to the `numpy.random` namespace in 1.17.0. -It was meant to be a private c-extension module, but was exposed as public. -It has been replaced by `numpy.random.SeedSequence` so the module was -completely removed. diff --git a/doc/release/upcoming_changes/14501.improvement.rst b/doc/release/upcoming_changes/14501.improvement.rst deleted file mode 100644 index f397ecccf..000000000 --- a/doc/release/upcoming_changes/14501.improvement.rst +++ /dev/null @@ -1,6 +0,0 @@ -`numpy.random.randint` produced incorrect value when the range was ``2**32`` ----------------------------------------------------------------------------- -The implementation introduced in 1.17.0 had an incorrect check when -determining whether to use the 32-bit path or the full 64-bit -path that incorrectly redirected random integer generation with a high - low -range of ``2**32`` to the 64-bit generator. diff --git a/doc/release/upcoming_changes/README.rst b/doc/release/upcoming_changes/README.rst new file mode 100644 index 000000000..7f6476bda --- /dev/null +++ b/doc/release/upcoming_changes/README.rst @@ -0,0 +1,55 @@ +:orphan: + +Changelog +========= + +This directory contains "news fragments" which are short files that contain a +small **ReST**-formatted text that will be added to the next what's new page. + +Make sure to use full sentences with correct case and punctuation, and please +try to use Sphinx intersphinx using backticks. The fragment should have a +header line and an underline using ``------`` + +Each file should be named like ``<PULL REQUEST>.<TYPE>.rst``, where +``<PULL REQUEST>`` is a pull request number, and ``<TYPE>`` is one of: + +* ``new_function``: New user facing functions. +* ``deprecation``: Changes existing code to emit a DeprecationWarning. +* ``future``: Changes existing code to emit a FutureWarning. +* ``expired``: Removal of a deprecated part of the API. +* ``compatibility``: A change which requires users to change code and is not + backwards compatible. (Not to be used for removal of deprecated features.) +* ``c_api``: Changes in the Numpy C-API exported functions +* ``new_feature``: New user facing features like ``kwargs``. +* ``improvement``: Performance and edge-case changes +* ``change``: Other changes +* ``highlight``: Adds a highlight bullet point to use as a possibly highlight + of the release. + +Most categories should be formatted as paragraphs with a heading. +So for example: ``123.new_feature.rst`` would have the content:: + + ``my_new_feature`` option for `my_favorite_function` + ---------------------------------------------------- + The ``my_new_feature`` option is now available for `my_favorite_function`. + To use it, write ``np.my_favorite_function(..., my_new_feature=True)``. + +``highlight`` is usually formatted as bulled points making the fragment +``* This is a highlight``. + +Note the use of single-backticks to get an internal link (assuming +``my_favorite_function`` is exported from the ``numpy`` namespace), +and double-backticks for code. + +If you are unsure what pull request type to use, don't hesitate to ask in your +PR. + +You can install ``towncrier`` and run ``towncrier --draft --version 1.18`` +if you want to get a preview of how your change will look in the final release +notes. + +.. note:: + + This README was adapted from the pytest changelog readme under the terms of + the MIT licence. + diff --git a/doc/release/upcoming_changes/template.rst b/doc/release/upcoming_changes/template.rst new file mode 100644 index 000000000..9c8a3b5fc --- /dev/null +++ b/doc/release/upcoming_changes/template.rst @@ -0,0 +1,38 @@ +{% set title = "NumPy {} Release Notes".format(versiondata.version) %} +{{ "=" * title|length }} +{{ title }} +{{ "=" * title|length }} + +{% for section, _ in sections.items() %} +{% set underline = underlines[0] %}{% if section %}{{ section }} +{{ underline * section|length }}{% set underline = underlines[1] %} + +{% endif %} +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section] %} + +{{ definitions[category]['name'] }} +{{ underline * definitions[category]['name']|length }} + +{% if definitions[category]['showcontent'] %} +{% for text, values in sections[section][category].items() %} +{{ text }} +{{ get_indent(text) }}({{values|join(', ') }}) + +{% endfor %} +{% else %} +- {{ sections[section][category]['']|join(', ') }} + +{% endif %} +{% if sections[section][category]|length == 0 %} +No significant changes. + +{% else %} +{% endif %} +{% endfor %} +{% else %} +No significant changes. + + +{% endif %} +{% endfor %} diff --git a/doc/source/_static/numpy_logo.png b/doc/source/_static/numpy_logo.png Binary files differnew file mode 100644 index 000000000..af8cbe323 --- /dev/null +++ b/doc/source/_static/numpy_logo.png diff --git a/doc/source/_templates/autosummary/base.rst b/doc/source/_templates/autosummary/base.rst new file mode 100644 index 000000000..0331154a7 --- /dev/null +++ b/doc/source/_templates/autosummary/base.rst @@ -0,0 +1,14 @@ +{% if objtype == 'property' %} +:orphan: +{% endif %} + +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +{% if objtype == 'property' %} +property +{% endif %} + +.. auto{{ objtype }}:: {{ objname }} + diff --git a/doc/source/_templates/indexsidebar.html b/doc/source/_templates/indexsidebar.html index 51e7c4308..4707fc0e8 100644 --- a/doc/source/_templates/indexsidebar.html +++ b/doc/source/_templates/indexsidebar.html @@ -1,4 +1,5 @@ <h3>Resources</h3> <ul> + <li><a href="https://numpy.org/">NumPy.org website</a></li> <li><a href="https://scipy.org/">Scipy.org website</a></li> </ul> diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html index 77da54a00..beaa297db 100644 --- a/doc/source/_templates/layout.html +++ b/doc/source/_templates/layout.html @@ -1,5 +1,15 @@ {% extends "!layout.html" %} +{%- block header %} +<div class="container"> + <div class="top-scipy-org-logo-header" style="background-color: #a2bae8;"> + <a href="{{ pathto('index') }}"> + <img border=0 alt="NumPy" src="{{ pathto('_static/numpy_logo.png', 1) }}"></a> + </div> + </div> +</div> + +{% endblock %} {% block rootrellink %} {% if pagename != 'index' %} <li class="active"><a href="{{ pathto('index') }}">{{ shorttitle|e }}</a></li> diff --git a/doc/source/conf.py b/doc/source/conf.py index fa0c0e7e4..83cecc917 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -3,12 +3,8 @@ from __future__ import division, absolute_import, print_function import sys, os, re -# Check Sphinx version -import sphinx -if sphinx.__version__ < "1.2.1": - raise RuntimeError("Sphinx 1.2.1 or newer required") - -needs_sphinx = '1.0' +# Minimum version, enforced by sphinx +needs_sphinx = '2.2.0' # ----------------------------------------------------------------------------- # General configuration @@ -31,13 +27,10 @@ extensions = [ 'matplotlib.sphinxext.plot_directive', 'IPython.sphinxext.ipython_console_highlighting', 'IPython.sphinxext.ipython_directive', + 'sphinx.ext.imgmath', ] -if sphinx.__version__ >= "1.4": - extensions.append('sphinx.ext.imgmath') - imgmath_image_format = 'svg' -else: - extensions.append('sphinx.ext.pngmath') +imgmath_image_format = 'svg' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -45,6 +38,8 @@ templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' +master_doc = 'contents' + # General substitutions. project = 'NumPy' copyright = '2008-2019, The SciPy community' @@ -93,6 +88,7 @@ pygments_style = 'sphinx' def setup(app): # add a config value for `ifconfig` directives app.add_config_value('python_version_major', str(sys.version_info.major), 'env') + app.add_lexer('NumPyC', NumPyLexer(stripnl=False)) # ----------------------------------------------------------------------------- # HTML output @@ -121,7 +117,9 @@ else: "edit_link": False, "sidebar": "left", "scipy_org_logo": False, - "rootlinks": [] + "rootlinks": [("https://numpy.org/", "NumPy.org"), + ("https://numpy.org/doc", "Docs"), + ] } html_sidebars = {'index': ['indexsidebar.html', 'searchbox.html']} @@ -175,6 +173,10 @@ latex_documents = [ # not chapters. #latex_use_parts = False +latex_elements = { + 'fontenc': r'\usepackage[LGR,T1]{fontenc}' +} + # Additional stuff for the LaTeX preamble. latex_preamble = r''' \usepackage{amsmath} @@ -366,18 +368,15 @@ def linkcode_resolve(domain, info): from pygments.lexers import CLexer from pygments import token -from sphinx.highlighting import lexers import copy class NumPyLexer(CLexer): name = 'NUMPYLEXER' - tokens = copy.deepcopy(lexers['c'].tokens) + tokens = copy.deepcopy(CLexer.tokens) # Extend the regex for valid identifiers with @ for k, val in tokens.items(): for i, v in enumerate(val): if isinstance(v, tuple): if isinstance(v[0], str): val[i] = (v[0].replace('a-zA-Z', 'a-zA-Z@'),) + v[1:] - -lexers['NumPyC'] = NumPyLexer(stripnl=False) diff --git a/doc/source/dev/development_environment.rst b/doc/source/dev/development_environment.rst index bc491b711..c20b6fe40 100644 --- a/doc/source/dev/development_environment.rst +++ b/doc/source/dev/development_environment.rst @@ -3,20 +3,26 @@ Setting up and using your development environment ================================================= +.. _recommended-development-setup: + Recommended development setup ----------------------------- Since NumPy contains parts written in C and Cython that need to be compiled before use, make sure you have the necessary compilers and Python development headers installed - see :ref:`building-from-source`. Building -NumPy as of version ``1.17`` requires a C99 compliant compiler. For -some older compilers this may require ``export CFLAGS='-std=c99'``. +NumPy as of version ``1.17`` requires a C99 compliant compiler. Having compiled code also means that importing NumPy from the development sources needs some additional steps, which are explained below. For the rest of this chapter we assume that you have set up your git repo as described in :ref:`using-git`. +.. _testing-builds: + +Testing builds +-------------- + To build the development version of NumPy and run tests, spawn interactive shells with the Python import paths properly set up etc., do one of:: @@ -45,10 +51,24 @@ When using pytest as a target (the default), you can $ python runtests.py -v -t numpy/core/tests/test_multiarray.py -- -k "MatMul and not vector" +.. note:: + + Remember that all tests of NumPy should pass before committing your changes. + Using ``runtests.py`` is the recommended approach to running tests. There are also a number of alternatives to it, for example in-place build or installing to a virtualenv. See the FAQ below for details. +.. note:: + + Some of the tests in the test suite require a large amount of + memory, and are skipped if your system does not have enough. + + To override the automatic detection of available memory, set the + environment variable ``NPY_AVAILABLE_MEM``, for example + ``NPY_AVAILABLE_MEM=32GB``, or using pytest ``--available-memory=32GB`` + target option. + Building in-place ----------------- @@ -85,19 +105,31 @@ installs a ``.egg-link`` file into your site-packages as well as adjusts the Other build options ------------------- +Build options can be discovered by running any of:: + + $ python setup.py --help + $ python setup.py --help-commands + It's possible to do a parallel build with ``numpy.distutils`` with the ``-j`` option; see :ref:`parallel-builds` for more details. -In order to install the development version of NumPy in ``site-packages``, use -``python setup.py install --user``. - A similar approach to in-place builds and use of ``PYTHONPATH`` but outside the source tree is to use:: - $ python setup.py install --prefix /some/owned/folder + $ pip install . --prefix /some/owned/folder $ export PYTHONPATH=/some/owned/folder/lib/python3.4/site-packages +NumPy uses a series of tests to probe the compiler and libc libraries for +funtions. The results are stored in ``_numpyconfig.h`` and ``config.h`` files +using ``HAVE_XXX`` definitions. These tests are run during the ``build_src`` +phase of the ``_multiarray_umath`` module in the ``generate_config_h`` and +``generate_numpyconfig_h`` functions. Since the output of these calls includes +many compiler warnings and errors, by default it is run quietly. If you wish +to see this output, you can run the ``build_src`` stage verbosely:: + + $ python build build_src -v + Using virtualenvs ----------------- @@ -123,7 +155,7 @@ Running tests Besides using ``runtests.py``, there are various ways to run the tests. Inside the interpreter, tests can be run like this:: - >>> np.test() + >>> np.test() # doctest: +SKIPBLOCK >>> np.test('full') # Also run tests marked as slow >>> np.test('full', verbose=2) # Additionally print test name/file diff --git a/doc/source/dev/development_workflow.rst b/doc/source/dev/development_workflow.rst index 200d95b92..900431374 100644 --- a/doc/source/dev/development_workflow.rst +++ b/doc/source/dev/development_workflow.rst @@ -28,7 +28,7 @@ In short: - *Core developers* If you want to push changes without further review, see the notes :ref:`below <pushing-to-main>`. - + This way of working helps to keep work well organized and the history as clear as possible. @@ -69,7 +69,7 @@ Overview git status # Optional git diff # Optional git add modified_file - git commit + git commit # push the branch to your own Github repo git push origin my-new-feature @@ -112,38 +112,38 @@ In more detail properly formatted and sufficiently detailed commit message. After saving your message and closing the editor, your commit will be saved. For trivial commits, a short commit message can be passed in through the command line - using the ``-m`` flag. For example, ``git commit -am "ENH: Some message"``. - + using the ``-m`` flag. For example, ``git commit -am "ENH: Some message"``. + In some cases, you will see this form of the commit command: ``git commit -a``. The extra ``-a`` flag automatically commits all modified files and removes all deleted files. This can save you some typing of numerous ``git add`` commands; however, it can add unwanted changes to a commit if you're not careful. For more information, see `why the -a flag?`_ - and the - helpful use-case description in the `tangled working copy problem`_. + helpful use-case description in the `tangled working copy problem`_. #. Push the changes to your forked repo on github_:: git push origin my-new-feature For more information, see `git push`_. - + .. note:: - + Assuming you have followed the instructions in these pages, git will create a default link to your github_ repo called ``origin``. In git >= 1.7 you can ensure that the link to origin is permanently set by using the ``--set-upstream`` option:: - + git push --set-upstream origin my-new-feature - + From now on git_ will know that ``my-new-feature`` is related to the ``my-new-feature`` branch in your own github_ repo. Subsequent push calls are then simplified to the following:: git push - + You have to use ``--set-upstream`` for each new branch that you create. - + It may be the case that while you were working on your edits, new commits have been added to ``upstream`` that affect your work. In this case, follow the @@ -194,12 +194,18 @@ Asking for your changes to be merged with the main repo ======================================================= When you feel your work is finished, you can create a pull request (PR). Github -has a nice help page that outlines the process for `filing pull requests`_. +has a nice help page that outlines the process for `filing pull requests`_. If your changes involve modifications to the API or addition/modification of a -function, you should initiate a code review. This involves sending an email to -the `NumPy mailing list`_ with a link to your PR along with a description of -and a motivation for your changes. +function, you should + +- send an email to the `NumPy mailing list`_ with a link to your PR along with + a description of and a motivation for your changes. This may generate + changes and feedback. It might be prudent to start with this step if your + change may be controversial. +- add a release note to the ``doc/release/upcoming_changes/`` directory, + following the instructions and format in the + ``doc/release/upcoming_changes/README.rst`` file. .. _rebasing-on-master: @@ -500,7 +506,7 @@ them to ``upstream`` as follows: git push upstream my-feature-branch:master -.. note:: +.. note:: It's usually a good idea to use the ``-n`` flag to ``git push`` to check first that you're about to push the changes you want to the place you diff --git a/doc/source/dev/gitwash/development_setup.rst b/doc/source/dev/gitwash/development_setup.rst index 1ebd4b486..9027dda64 100644 --- a/doc/source/dev/gitwash/development_setup.rst +++ b/doc/source/dev/gitwash/development_setup.rst @@ -25,6 +25,8 @@ to the instructions at http://help.github.com/forking/ - please see that page for more detail. We're repeating some of it here just to give the specifics for the NumPy_ project, and to suggest some default names. +.. _set-up-and-configure-a-github-account: + Set up and configure a github_ account ====================================== diff --git a/doc/source/dev/governance/people.rst b/doc/source/dev/governance/people.rst index 40347f9bf..10af7f221 100644 --- a/doc/source/dev/governance/people.rst +++ b/doc/source/dev/governance/people.rst @@ -48,7 +48,7 @@ NumFOCUS Subcommittee * Jaime Fernández del RÃo -* Nathaniel Smith +* Sebastian Berg * External member: Thomas Caswell diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index f0b81ba5d..306c15069 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -2,6 +2,33 @@ Contributing to NumPy ##################### +Not a coder? Not a problem! NumPy is multi-faceted, and we can use a lot of help. +These are all activities we'd like to get help with (they're all important, so +we list them in alphabetical order): + +- Code maintenance and development +- Community coordination +- DevOps +- Developing educational content & narrative documentation +- Writing technical documentation +- Fundraising +- Project management +- Marketing +- Translating content +- Website design and development + +The rest of this document discusses working on the NumPy code base and documentation. +We're in the process of updating our descriptions of other activities and roles. +If you are interested in these other activities, please contact us! +You can do this via +the `numpy-discussion mailing list <https://scipy.org/scipylib/mailing-lists.html>`__, +or on GitHub (open an issue or comment on a relevant issue). These are our preferred +communication channels (open source is open by nature!), however if you prefer +to discuss in private first, please reach out to our community coordinators +at `numpy-team@googlegroups.com` or `numpy-team.slack.com` (send an email to +`numpy-team@googlegroups.com` for an invite the first time). + + Development process - summary ============================= @@ -44,11 +71,11 @@ Here's the short summary, complete TOC links are below: git checkout -b linspace-speedups * Commit locally as you progress (``git add`` and ``git commit``) - Use a `properly formatted <writing-the-commit-message>` commit message, + Use a :ref:`properly formatted<writing-the-commit-message>` commit message, write tests that fail before your change and pass afterward, run all the - `tests locally <development-environment>`. Be sure to document any + :ref:`tests locally<development-environment>`. Be sure to document any changed behavior in docstrings, keeping to the NumPy docstring - `standard <howto-document>`. + :ref:`standard<howto-document>`. 3. To submit your contribution: @@ -57,8 +84,8 @@ Here's the short summary, complete TOC links are below: git push origin linspace-speedups * Enter your GitHub username and password (repeat contributors or advanced - users can remove this step by connecting to GitHub with `SSH <set-up-and- - configure-a-github-account>`. + users can remove this step by connecting to GitHub with + :ref:`SSH<set-up-and-configure-a-github-account>` . * Go to GitHub. The new branch will show up with a green Pull Request button. Make sure the title and message are clear, concise, and self- @@ -92,8 +119,9 @@ Here's the short summary, complete TOC links are below: coding style of your branch. The CI tests must pass before your PR can be merged. If CI fails, you can find out why by clicking on the "failed" icon (red cross) and inspecting the build and test log. To avoid overuse - and waste of this resource, `test your work <recommended-development- - setup>` locally before committing. + and waste of this resource, + :ref:`test your work<recommended-development-setup>` locally before + committing. * A PR must be **approved** by at least one core team member before merging. Approval means the core team member has carefully reviewed the changes, @@ -103,8 +131,11 @@ Here's the short summary, complete TOC links are below: Beyond changes to a functions docstring and possible description in the general documentation, if your change introduces any user-facing - modifications, update the current release notes under - ``doc/release/X.XX-notes.rst`` + modifications they may need to be mentioned in the release notes. + To add your change to the release notes, you need to create a short file + with a summary and place it in ``doc/release/upcoming_changes``. + The file ``doc/release/upcoming_changes/README.rst`` details the format and + filename conventions. If your change introduces a deprecation, make sure to discuss this first on GitHub or the mailing list first. If agreement on the deprecation is @@ -131,13 +162,14 @@ Divergence between ``upstream/master`` and your feature branch If GitHub indicates that the branch of your Pull Request can no longer be merged automatically, you have to incorporate changes that have been made since you started into your branch. Our recommended way to do this is to -`rebase on master <rebasing-on-master>`. +:ref:`rebase on master<rebasing-on-master>`. Guidelines ---------- * All code should have tests (see `test coverage`_ below for more details). -* All code should be `documented <docstring-standard>`. +* All code should be `documented <https://numpydoc.readthedocs.io/ + en/latest/format.html#docstring-standard>`_. * No changes are ever committed without review and approval by a core team member.Please ask politely on the PR or on the `mailing list`_ if you get no response to your pull request within a week. @@ -156,14 +188,14 @@ Stylistic Guidelines import numpy as np -* For C code, see the `numpy-c-style-guide` +* For C code, see the :ref:`numpy-c-style-guide<style_guide>` Test coverage ------------- Pull requests (PRs) that modify code should either have new tests, or modify existing -tests to fail before the PR and pass afterwards. You should `run the tests +tests to fail before the PR and pass afterwards. You should :ref:`run the tests <development-environment>` before pushing a PR. Tests for a module should ideally cover all code in that module, @@ -175,7 +207,7 @@ and then run:: $ python runtests.py --coverage -This will create a report in `build/coverage`, which can be viewed with:: +This will create a report in ``build/coverage``, which can be viewed with:: $ firefox build/coverage/index.html @@ -197,7 +229,7 @@ Requirements ~~~~~~~~~~~~ `Sphinx <http://www.sphinx-doc.org/en/stable/>`__ is needed to build -the documentation. Matplotlib and SciPy are also required. +the documentation. Matplotlib, SciPy, and IPython are also required. Fixing Warnings ~~~~~~~~~~~~~~~ @@ -226,6 +258,7 @@ The rest of the story releasing governance/index -NumPy-specific workflow is in `numpy-development-workflow`. +NumPy-specific workflow is in :ref:`numpy-development-workflow +<development-workflow>`. .. _`mailing list`: https://mail.python.org/mailman/listinfo/numpy-devel diff --git a/doc/source/docs/howto_build_docs.rst b/doc/source/docs/howto_build_docs.rst index 98d1b88ba..6deacda5c 100644 --- a/doc/source/docs/howto_build_docs.rst +++ b/doc/source/docs/howto_build_docs.rst @@ -5,7 +5,7 @@ Building the NumPy API and reference docs ========================================= We currently use Sphinx_ for generating the API and reference -documentation for NumPy. You will need Sphinx 1.8.3 or newer. +documentation for NumPy. You will need Sphinx 1.8.3 <= 1.8.5. If you only want to get the documentation, note that pre-built versions can be found at @@ -30,11 +30,9 @@ In addition, building the documentation requires the Sphinx extension `plot_directive`, which is shipped with Matplotlib_. This Sphinx extension can be installed by installing Matplotlib. You will also need python3.6. -Since large parts of the main documentation are stored in -docstrings, you will need to first build NumPy, and install it so -that the correct version is imported by - - >>> import numpy +Since large parts of the main documentation are obtained from numpy via +``import numpy`` and examining the docstrings, you will need to first build +NumPy, and install it so that the correct version is imported. Note that you can eg. install NumPy to a temporary location and set the PYTHONPATH environment variable appropriately. @@ -46,8 +44,11 @@ generate the docs, so write:: make html in the ``doc/`` directory. If all goes well, this will generate a -``build/html`` subdirectory containing the built documentation. Note -that building the documentation on Windows is currently not actively +``build/html`` subdirectory containing the built documentation. If you get +a message about ``installed numpy != current repo git version``, you must +either override the check by setting ``GITVER`` or re-install NumPy. + +Note that building the documentation on Windows is currently not actively supported, though it should be possible. (See Sphinx_ documentation for more information.) diff --git a/doc/source/f2py/advanced.rst b/doc/source/f2py/advanced.rst index c9f3862e6..375922033 100644 --- a/doc/source/f2py/advanced.rst +++ b/doc/source/f2py/advanced.rst @@ -43,3 +43,55 @@ In Python: .. include:: var_session.dat :literal: + + +Dealing with KIND specifiers +============================ + +Currently, F2PY can handle only ``<type spec>(kind=<kindselector>)`` +declarations where ``<kindselector>`` is a numeric integer (e.g. 1, 2, +4,...), but not a function call ``KIND(..)`` or any other +expression. F2PY needs to know what would be the corresponding C type +and a general solution for that would be too complicated to implement. + +However, F2PY provides a hook to overcome this difficulty, namely, +users can define their own <Fortran type> to <C type> maps. For +example, if Fortran 90 code contains:: + + REAL(kind=KIND(0.0D0)) ... + +then create a mapping file containing a Python dictionary:: + + {'real': {'KIND(0.0D0)': 'double'}} + +for instance. + +Use the ``--f2cmap`` command-line option to pass the file name to F2PY. +By default, F2PY assumes file name is ``.f2py_f2cmap`` in the current +working directory. + +Or more generally, the f2cmap file must contain a dictionary +with items:: + + <Fortran typespec> : {<selector_expr>:<C type>} + +that defines mapping between Fortran type:: + + <Fortran typespec>([kind=]<selector_expr>) + +and the corresponding <C type>. <C type> can be one of the following:: + + char + signed_char + short + int + long_long + float + double + long_double + complex_float + complex_double + complex_long_double + string + +For more information, see F2Py source code ``numpy/f2py/capi_maps.py``. diff --git a/doc/source/f2py/distutils.rst b/doc/source/f2py/distutils.rst index fdcd38468..71f6eab5a 100644 --- a/doc/source/f2py/distutils.rst +++ b/doc/source/f2py/distutils.rst @@ -26,7 +26,7 @@ sources, call F2PY to construct extension modules, etc. :mod:`numpy.distutils` extends ``distutils`` with the following features: -* ``Extension`` class argument ``sources`` may contain Fortran source +* :class:`Extension` class argument ``sources`` may contain Fortran source files. In addition, the list ``sources`` may contain at most one F2PY signature file, and then the name of an Extension module must match with the ``<modulename>`` used in signature file. It is @@ -37,7 +37,7 @@ sources, call F2PY to construct extension modules, etc. to scan Fortran source files for routine signatures to construct the wrappers to Fortran codes. - Additional options to F2PY process can be given using ``Extension`` + Additional options to F2PY process can be given using :class:`Extension` class argument ``f2py_options``. * The following new ``distutils`` commands are defined: diff --git a/doc/source/reference/alignment.rst b/doc/source/reference/alignment.rst index ebc8f353c..5e4315b38 100644 --- a/doc/source/reference/alignment.rst +++ b/doc/source/reference/alignment.rst @@ -67,7 +67,7 @@ There are 4 relevant uses of the word ``align`` used in numpy: field alignment. In either case ``dtype.isalignedstruct`` is also set to True. * ``IsUintAligned`` is used to determine if an ndarray is "uint aligned" in - an analagous way to how ``IsAligned`` checks for true-alignment. + an analogous way to how ``IsAligned`` checks for true-alignment. Consequences of alignment ------------------------- diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 0fafc593e..9dcbb6267 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -51,7 +51,7 @@ NumPy provides several hooks that classes can customize: .. versionadded:: 1.13 Any class, ndarray subclass or not, can define this method or set it to - :obj:`None` in order to override the behavior of NumPy's ufuncs. This works + None in order to override the behavior of NumPy's ufuncs. This works quite similarly to Python's ``__mul__`` and other binary operation routines. - *ufunc* is the ufunc object that was called. @@ -82,7 +82,7 @@ NumPy provides several hooks that classes can customize: :func:`~numpy.matmul`, which currently is not a Ufunc, but could be relatively easily be rewritten as a (set of) generalized Ufuncs. The same may happen with functions such as :func:`~numpy.median`, - :func:`~numpy.min`, and :func:`~numpy.argsort`. + :func:`~numpy.amin`, and :func:`~numpy.argsort`. Like with some other special methods in python, such as ``__hash__`` and ``__iter__``, it is possible to indicate that your class does *not* @@ -94,13 +94,13 @@ NumPy provides several hooks that classes can customize: :class:`ndarray` handles binary operations like ``arr + obj`` and ``arr < obj`` when ``arr`` is an :class:`ndarray` and ``obj`` is an instance of a custom class. There are two possibilities. If - ``obj.__array_ufunc__`` is present and not :obj:`None`, then + ``obj.__array_ufunc__`` is present and not None, then ``ndarray.__add__`` and friends will delegate to the ufunc machinery, meaning that ``arr + obj`` becomes ``np.add(arr, obj)``, and then :func:`~numpy.add` invokes ``obj.__array_ufunc__``. This is useful if you want to define an object that acts like an array. - Alternatively, if ``obj.__array_ufunc__`` is set to :obj:`None`, then as a + Alternatively, if ``obj.__array_ufunc__`` is set to None, then as a special case, special methods like ``ndarray.__add__`` will notice this and *unconditionally* raise :exc:`TypeError`. This is useful if you want to create objects that interact with arrays via binary operations, but @@ -135,7 +135,7 @@ NumPy provides several hooks that classes can customize: place rather than separately by the ufunc machinery and by the binary operation rules (which gives preference to special methods of subclasses; the alternative way to enforce a one-place only hierarchy, - of setting :func:`__array_ufunc__` to :obj:`None`, would seem very + of setting :func:`__array_ufunc__` to None, would seem very unexpected and thus confusing, as then the subclass would not work at all with ufuncs). - :class:`ndarray` defines its own :func:`__array_ufunc__`, which, @@ -280,7 +280,7 @@ NumPy provides several hooks that classes can customize: .. py:method:: class.__array_prepare__(array, context=None) - At the beginning of every :ref:`ufunc <ufuncs.output-type>`, this + At the beginning of every :ref:`ufunc <ufuncs-output-type>`, this method is called on the input object with the highest array priority, or the output object if one was specified. The output array is passed in and whatever is returned is passed to the ufunc. @@ -295,7 +295,7 @@ NumPy provides several hooks that classes can customize: .. py:method:: class.__array_wrap__(array, context=None) - At the end of every :ref:`ufunc <ufuncs.output-type>`, this method + At the end of every :ref:`ufunc <ufuncs-output-type>`, this method is called on the input object with the highest array priority, or the output object if one was specified. The ufunc-computed array is passed in and whatever is returned is passed to the user. @@ -322,7 +322,7 @@ NumPy provides several hooks that classes can customize: If a class (ndarray subclass or not) having the :func:`__array__` method is used as the output object of an :ref:`ufunc - <ufuncs.output-type>`, results will be written to the object + <ufuncs-output-type>`, results will be written to the object returned by :func:`__array__`. Similar conversion is done on input arrays. @@ -570,7 +570,7 @@ object, then the Python code:: some code involving val ... -calls ``val = myiter.next()`` repeatedly until :exc:`StopIteration` is +calls ``val = next(myiter)`` repeatedly until :exc:`StopIteration` is raised by the iterator. There are several ways to iterate over an array that may be useful: default iteration, flat iteration, and :math:`N`-dimensional enumeration. diff --git a/doc/source/reference/arrays.datetime.rst b/doc/source/reference/arrays.datetime.rst index 387515f59..9c45e04c7 100644 --- a/doc/source/reference/arrays.datetime.rst +++ b/doc/source/reference/arrays.datetime.rst @@ -26,7 +26,9 @@ be either a :ref:`date unit <arrays.dtypes.dateunits>` or a :ref:`time unit <arrays.dtypes.timeunits>`. The date units are years ('Y'), months ('M'), weeks ('W'), and days ('D'), while the time units are hours ('h'), minutes ('m'), seconds ('s'), milliseconds ('ms'), and -some additional SI-prefix seconds-based units. +some additional SI-prefix seconds-based units. The datetime64 data type +also accepts the string "NAT", in any combination of lowercase/uppercase +letters, for a "Not A Time" value. .. admonition:: Example @@ -50,6 +52,11 @@ some additional SI-prefix seconds-based units. >>> np.datetime64('2005-02-25T03:30') numpy.datetime64('2005-02-25T03:30') + NAT (not a time): + + >>> numpy.datetime64('nat') + numpy.datetime64('NaT') + When creating an array of datetimes from a string, it is still possible to automatically select the unit from the inputs, by using the datetime type with generic units. @@ -100,7 +107,21 @@ Datetime and Timedelta Arithmetic NumPy allows the subtraction of two Datetime values, an operation which produces a number with a time unit. Because NumPy doesn't have a physical quantities system in its core, the timedelta64 data type was created -to complement datetime64. +to complement datetime64. The arguments for timedelta64 are a number, +to represent the number of units, and a date/time unit, such as +(D)ay, (M)onth, (Y)ear, (h)ours, (m)inutes, or (s)econds. The timedelta64 +data type also accepts the string "NAT" in place of the number for a "Not A Time" value. + +.. admonition:: Example + + >>> numpy.timedelta64(1, 'D') + numpy.timedelta64(1,'D') + + >>> numpy.timedelta64(4, 'h') + numpy.timedelta64(4,'h') + + >>> numpy.timedelta64('nAt') + numpy.timedelta64('NaT') Datetimes and Timedeltas work together to provide ways for simple datetime calculations. @@ -122,6 +143,12 @@ simple datetime calculations. >>> np.timedelta64(1,'W') % np.timedelta64(10,'D') numpy.timedelta64(7,'D') + >>> numpy.datetime64('nat') - numpy.datetime64('2009-01-01') + numpy.timedelta64('NaT','D') + + >>> numpy.datetime64('2009-01-01') + numpy.timedelta64('nat') + numpy.datetime64('NaT') + There are two Timedelta units ('Y', years and 'M', months) which are treated specially, because how much time they represent changes depending on when they are used. While a timedelta day unit is equivalent to @@ -341,7 +368,7 @@ times in UTC. By default, creating a datetime64 object from a string or printing it would convert from or to local time:: # old behavior - >>>> np.datetime64('2000-01-01T00:00:00') + >>> np.datetime64('2000-01-01T00:00:00') numpy.datetime64('2000-01-01T00:00:00-0800') # note the timezone offset -08:00 A consensus of datetime64 users agreed that this behavior is undesirable @@ -351,7 +378,7 @@ most use cases, a timezone naive datetime type is preferred, similar to the datetime64 no longer assumes that input is in local time, nor does it print local times:: - >>>> np.datetime64('2000-01-01T00:00:00') + >>> np.datetime64('2000-01-01T00:00:00') numpy.datetime64('2000-01-01T00:00:00') For backwards compatibility, datetime64 still parses timezone offsets, which @@ -367,131 +394,3 @@ with date units and datetimes with timeunits. With timezone naive datetimes, the rule for casting from dates to times is no longer ambiguous. .. _pandas: http://pandas.pydata.org - - -Differences Between 1.6 and 1.7 Datetimes -========================================= - -The NumPy 1.6 release includes a more primitive datetime data type -than 1.7. This section documents many of the changes that have taken -place. - -String Parsing -`````````````` - -The datetime string parser in NumPy 1.6 is very liberal in what it accepts, -and silently allows invalid input without raising errors. The parser in -NumPy 1.7 is quite strict about only accepting ISO 8601 dates, with a few -convenience extensions. 1.6 always creates microsecond (us) units by -default, whereas 1.7 detects a unit based on the format of the string. -Here is a comparison.:: - - # NumPy 1.6.1 - >>> np.datetime64('1979-03-22') - 1979-03-22 00:00:00 - # NumPy 1.7.0 - >>> np.datetime64('1979-03-22') - numpy.datetime64('1979-03-22') - - # NumPy 1.6.1, unit default microseconds - >>> np.datetime64('1979-03-22').dtype - dtype('datetime64[us]') - # NumPy 1.7.0, unit of days detected from string - >>> np.datetime64('1979-03-22').dtype - dtype('<M8[D]') - - # NumPy 1.6.1, ignores invalid part of string - >>> np.datetime64('1979-03-2corruptedstring') - 1979-03-02 00:00:00 - # NumPy 1.7.0, raises error for invalid input - >>> np.datetime64('1979-03-2corruptedstring') - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - ValueError: Error parsing datetime string "1979-03-2corruptedstring" at position 8 - - # NumPy 1.6.1, 'nat' produces today's date - >>> np.datetime64('nat') - 2012-04-30 00:00:00 - # NumPy 1.7.0, 'nat' produces not-a-time - >>> np.datetime64('nat') - numpy.datetime64('NaT') - - # NumPy 1.6.1, 'garbage' produces today's date - >>> np.datetime64('garbage') - 2012-04-30 00:00:00 - # NumPy 1.7.0, 'garbage' raises an exception - >>> np.datetime64('garbage') - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - ValueError: Error parsing datetime string "garbage" at position 0 - - # NumPy 1.6.1, can't specify unit in scalar constructor - >>> np.datetime64('1979-03-22T19:00', 'h') - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: function takes at most 1 argument (2 given) - # NumPy 1.7.0, unit in scalar constructor - >>> np.datetime64('1979-03-22T19:00', 'h') - numpy.datetime64('1979-03-22T19:00-0500','h') - - # NumPy 1.6.1, reads ISO 8601 strings w/o TZ as UTC - >>> np.array(['1979-03-22T19:00'], dtype='M8[h]') - array([1979-03-22 19:00:00], dtype=datetime64[h]) - # NumPy 1.7.0, reads ISO 8601 strings w/o TZ as local (ISO specifies this) - >>> np.array(['1979-03-22T19:00'], dtype='M8[h]') - array(['1979-03-22T19-0500'], dtype='datetime64[h]') - - # NumPy 1.6.1, doesn't parse all ISO 8601 strings correctly - >>> np.array(['1979-03-22T12'], dtype='M8[h]') - array([1979-03-22 00:00:00], dtype=datetime64[h]) - >>> np.array(['1979-03-22T12:00'], dtype='M8[h]') - array([1979-03-22 12:00:00], dtype=datetime64[h]) - # NumPy 1.7.0, handles this case correctly - >>> np.array(['1979-03-22T12'], dtype='M8[h]') - array(['1979-03-22T12-0500'], dtype='datetime64[h]') - >>> np.array(['1979-03-22T12:00'], dtype='M8[h]') - array(['1979-03-22T12-0500'], dtype='datetime64[h]') - -Unit Conversion -``````````````` - -The 1.6 implementation of datetime does not convert between units correctly.:: - - # NumPy 1.6.1, the representation value is untouched - >>> np.array(['1979-03-22'], dtype='M8[D]') - array([1979-03-22 00:00:00], dtype=datetime64[D]) - >>> np.array(['1979-03-22'], dtype='M8[D]').astype('M8[M]') - array([2250-08-01 00:00:00], dtype=datetime64[M]) - # NumPy 1.7.0, the representation is scaled accordingly - >>> np.array(['1979-03-22'], dtype='M8[D]') - array(['1979-03-22'], dtype='datetime64[D]') - >>> np.array(['1979-03-22'], dtype='M8[D]').astype('M8[M]') - array(['1979-03'], dtype='datetime64[M]') - -Datetime Arithmetic -``````````````````` - -The 1.6 implementation of datetime only works correctly for a small subset of -arithmetic operations. Here we show some simple cases.:: - - # NumPy 1.6.1, produces invalid results if units are incompatible - >>> a = np.array(['1979-03-22T12'], dtype='M8[h]') - >>> b = np.array([3*60], dtype='m8[m]') - >>> a + b - array([1970-01-01 00:00:00.080988], dtype=datetime64[us]) - # NumPy 1.7.0, promotes to higher-resolution unit - >>> a = np.array(['1979-03-22T12'], dtype='M8[h]') - >>> b = np.array([3*60], dtype='m8[m]') - >>> a + b - array(['1979-03-22T15:00-0500'], dtype='datetime64[m]') - - # NumPy 1.6.1, arithmetic works if everything is microseconds - >>> a = np.array(['1979-03-22T12:00'], dtype='M8[us]') - >>> b = np.array([3*60*60*1000000], dtype='m8[us]') - >>> a + b - array([1979-03-22 15:00:00], dtype=datetime64[us]) - # NumPy 1.7.0 - >>> a = np.array(['1979-03-22T12:00'], dtype='M8[us]') - >>> b = np.array([3*60*60*1000000], dtype='m8[us]') - >>> a + b - array(['1979-03-22T15:00:00.000000-0500'], dtype='datetime64[us]') diff --git a/doc/source/reference/arrays.dtypes.rst b/doc/source/reference/arrays.dtypes.rst index ab743a8ee..231707b11 100644 --- a/doc/source/reference/arrays.dtypes.rst +++ b/doc/source/reference/arrays.dtypes.rst @@ -128,7 +128,7 @@ What can be converted to a data-type object is described below: Used as-is. -:const:`None` +None .. index:: triple: dtype; construction; from None @@ -392,7 +392,7 @@ Type strings their values must each be lists of the same length as the *names* and *formats* lists. The *offsets* value is a list of byte offsets (limited to `ctypes.c_int`) for each field, while the *titles* value is a - list of titles for each field (:const:`None` can be used if no title is + list of titles for each field (None can be used if no title is desired for that field). The *titles* can be any :class:`string` or :class:`unicode` object and will add another entry to the fields dictionary keyed by the title and referencing the same diff --git a/doc/source/reference/arrays.indexing.rst b/doc/source/reference/arrays.indexing.rst index 0c0c8dff6..8ec8d8330 100644 --- a/doc/source/reference/arrays.indexing.rst +++ b/doc/source/reference/arrays.indexing.rst @@ -3,6 +3,10 @@ Indexing ======== +.. seealso:: + + :ref:`Indexing basics <basics.indexing>` + .. sectionauthor:: adapted from "Guide to NumPy" by Travis E. Oliphant .. currentmodule:: numpy diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index f361ccb06..f36a083aa 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -138,18 +138,18 @@ This approach to the interface consists of the object having an This attribute can also be an object exposing the :c:func:`buffer interface <PyObject_AsCharBuffer>` which will be used to share the data. If this key is not present (or - returns :class:`None`), then memory sharing will be done + returns None), then memory sharing will be done through the buffer interface of the object itself. In this case, the offset key can be used to indicate the start of the buffer. A reference to the object exposing the array interface must be stored by the new object if the memory area is to be secured. - **Default**: :const:`None` + **Default**: None **strides** (optional) - Either :const:`None` to indicate a C-style contiguous array or + Either None to indicate a C-style contiguous array or a Tuple of strides which provides the number of bytes needed to jump to the next array element in the corresponding dimension. Each entry must be an integer (a Python @@ -157,29 +157,29 @@ This approach to the interface consists of the object having an be larger than can be represented by a C "int" or "long"; the calling code should handle this appropriately, either by raising an error, or by using :c:type:`Py_LONG_LONG` in C. The - default is :const:`None` which implies a C-style contiguous + default is None which implies a C-style contiguous memory buffer. In this model, the last dimension of the array varies the fastest. For example, the default strides tuple for an object whose array entries are 8 bytes long and whose shape is (10,20,30) would be (4800, 240, 8) - **Default**: :const:`None` (C-style contiguous) + **Default**: None (C-style contiguous) **mask** (optional) - :const:`None` or an object exposing the array interface. All + None or an object exposing the array interface. All elements of the mask array should be interpreted only as true or not true indicating which elements of this array are valid. The shape of this object should be `"broadcastable" <arrays.broadcasting.broadcastable>` to the shape of the original array. - **Default**: :const:`None` (All array values are valid) + **Default**: None (All array values are valid) **offset** (optional) An integer offset into the array data region. This can only be - used when data is :const:`None` or returns a :class:`buffer` + used when data is None or returns a :class:`buffer` object. **Default**: 0. diff --git a/doc/source/reference/arrays.ndarray.rst b/doc/source/reference/arrays.ndarray.rst index 8f431bc9c..831d211bc 100644 --- a/doc/source/reference/arrays.ndarray.rst +++ b/doc/source/reference/arrays.ndarray.rst @@ -329,7 +329,7 @@ Item selection and manipulation ------------------------------- For array methods that take an *axis* keyword, it defaults to -:const:`None`. If axis is *None*, then the array is treated as a 1-D +*None*. If axis is *None*, then the array is treated as a 1-D array. Any other value for *axis* represents the dimension along which the operation should proceed. diff --git a/doc/source/reference/arrays.nditer.rst b/doc/source/reference/arrays.nditer.rst index fa8183f75..7dab09a71 100644 --- a/doc/source/reference/arrays.nditer.rst +++ b/doc/source/reference/arrays.nditer.rst @@ -115,13 +115,18 @@ context is exited. array([[ 0, 2, 4], [ 6, 8, 10]]) +If you are writing code that needs to support older versions of numpy, +note that prior to 1.15, :class:`nditer` was not a context manager and +did not have a `close` method. Instead it relied on the destructor to +initiate the writeback of the buffer. + Using an External Loop ---------------------- In all the examples so far, the elements of `a` are provided by the iterator one at a time, because all the looping logic is internal to the -iterator. While this is simple and convenient, it is not very efficient. A -better approach is to move the one-dimensional innermost loop into your +iterator. While this is simple and convenient, it is not very efficient. +A better approach is to move the one-dimensional innermost loop into your code, external to the iterator. This way, NumPy's vectorized operations can be used on larger chunks of the elements being visited. @@ -156,41 +161,29 @@ element in a computation. For example, you may want to visit the elements of an array in memory order, but use a C-order, Fortran-order, or multidimensional index to look up values in a different array. -The Python iterator protocol doesn't have a natural way to query these -additional values from the iterator, so we introduce an alternate syntax -for iterating with an :class:`nditer`. This syntax explicitly works -with the iterator object itself, so its properties are readily accessible -during iteration. With this looping construct, the current value is -accessible by indexing into the iterator, and the index being tracked -is the property `index` or `multi_index` depending on what was requested. - -The Python interactive interpreter unfortunately prints out the -values of expressions inside the while loop during each iteration of the -loop. We have modified the output in the examples using this looping -construct in order to be more readable. +The index is tracked by the iterator object itself, and accessible +through the `index` or `multi_index` properties, depending on what was +requested. The examples below show printouts demonstrating the +progression of the index: .. admonition:: Example >>> a = np.arange(6).reshape(2,3) >>> it = np.nditer(a, flags=['f_index']) - >>> while not it.finished: - ... print("%d <%d>" % (it[0], it.index), end=' ') - ... it.iternext() + >>> for x in it: + ... print("%d <%d>" % (x, it.index), end=' ') ... 0 <0> 1 <2> 2 <4> 3 <1> 4 <3> 5 <5> >>> it = np.nditer(a, flags=['multi_index']) - >>> while not it.finished: - ... print("%d <%s>" % (it[0], it.multi_index), end=' ') - ... it.iternext() + >>> for x in it: + ... print("%d <%s>" % (x, it.multi_index), end=' ') ... 0 <(0, 0)> 1 <(0, 1)> 2 <(0, 2)> 3 <(1, 0)> 4 <(1, 1)> 5 <(1, 2)> - >>> it = np.nditer(a, flags=['multi_index'], op_flags=['writeonly']) - >>> with it: - .... while not it.finished: - ... it[0] = it.multi_index[1] - it.multi_index[0] - ... it.iternext() + >>> with np.nditer(a, flags=['multi_index'], op_flags=['writeonly']) as it: + ... for x in it: + ... x[...] = it.multi_index[1] - it.multi_index[0] ... >>> a array([[ 0, 1, 2], @@ -199,7 +192,7 @@ construct in order to be more readable. Tracking an index or multi-index is incompatible with using an external loop, because it requires a different index value per element. If you try to combine these flags, the :class:`nditer` object will -raise an exception +raise an exception. .. admonition:: Example @@ -209,6 +202,42 @@ raise an exception File "<stdin>", line 1, in <module> ValueError: Iterator flag EXTERNAL_LOOP cannot be used if an index or multi-index is being tracked +Alternative Looping and Element Access +-------------------------------------- + +To make its properties more readily accessible during iteration, +:class:`nditer` has an alternative syntax for iterating, which works +explicitly with the iterator object itself. With this looping construct, +the current value is accessible by indexing into the iterator. Other +properties, such as tracked indices remain as before. The examples below +produce identical results to the ones in the previous section. + +.. admonition:: Example + + >>> a = np.arange(6).reshape(2,3) + >>> it = np.nditer(a, flags=['f_index']) + >>> while not it.finished: + ... print("%d <%d>" % (it[0], it.index), end=' ') + ... it.iternext() + ... + 0 <0> 1 <2> 2 <4> 3 <1> 4 <3> 5 <5> + + >>> it = np.nditer(a, flags=['multi_index']) + >>> while not it.finished: + ... print("%d <%s>" % (it[0], it.multi_index), end=' ') + ... it.iternext() + ... + 0 <(0, 0)> 1 <(0, 1)> 2 <(0, 2)> 3 <(1, 0)> 4 <(1, 1)> 5 <(1, 2)> + + >>> with np.nditer(a, flags=['multi_index'], op_flags=['writeonly']) as it: + ... while not it.finished: + ... it[0] = it.multi_index[1] - it.multi_index[0] + ... it.iternext() + ... + >>> a + array([[ 0, 1, 2], + [-1, 0, 1]]) + Buffering the Array Elements ---------------------------- diff --git a/doc/source/reference/c-api.array.rst b/doc/source/reference/c-api/array.rst index bd6062b16..0530a5747 100644 --- a/doc/source/reference/c-api.array.rst +++ b/doc/source/reference/c-api/array.rst @@ -20,27 +20,44 @@ Array API Array structure and data access ------------------------------- -These macros all access the :c:type:`PyArrayObject` structure members. The input -argument, arr, can be any :c:type:`PyObject *<PyObject>` that is directly interpretable -as a :c:type:`PyArrayObject *` (any instance of the :c:data:`PyArray_Type` and its -sub-types). +These macros access the :c:type:`PyArrayObject` structure members and are +defined in ``ndarraytypes.h``. The input argument, *arr*, can be any +:c:type:`PyObject *<PyObject>` that is directly interpretable as a +:c:type:`PyArrayObject *` (any instance of the :c:data:`PyArray_Type` +and itssub-types). .. c:function:: int PyArray_NDIM(PyArrayObject *arr) The number of dimensions in the array. -.. c:function:: npy_intp *PyArray_DIMS(PyArrayObject *arr) +.. c:function:: int PyArray_FLAGS(PyArrayObject* arr) - Returns a pointer to the dimensions/shape of the array. The - number of elements matches the number of dimensions - of the array. Can return ``NULL`` for 0-dimensional arrays. + Returns an integer representing the :ref:`array-flags<array-flags>`. -.. c:function:: npy_intp *PyArray_SHAPE(PyArrayObject *arr) +.. c:function:: int PyArray_TYPE(PyArrayObject* arr) + + Return the (builtin) typenumber for the elements of this array. + +.. c:function:: int PyArray_SETITEM( \ + PyArrayObject* arr, void* itemptr, PyObject* obj) + + Convert obj and place it in the ndarray, *arr*, at the place + pointed to by itemptr. Return -1 if an error occurs or 0 on + success. + +.. c:function:: void PyArray_ENABLEFLAGS(PyArrayObject* arr, int flags) .. versionadded:: 1.7 - A synonym for PyArray_DIMS, named to be consistent with the - 'shape' usage within Python. + Enables the specified array flags. This function does no validation, + and assumes that you know what you're doing. + +.. c:function:: void PyArray_CLEARFLAGS(PyArrayObject* arr, int flags) + + .. versionadded:: 1.7 + + Clears the specified array flags. This function does no validation, + and assumes that you know what you're doing. .. c:function:: void *PyArray_DATA(PyArrayObject *arr) @@ -53,6 +70,19 @@ sub-types). array then be sure you understand how to access the data in the array to avoid memory and/or alignment problems. +.. c:function:: npy_intp *PyArray_DIMS(PyArrayObject *arr) + + Returns a pointer to the dimensions/shape of the array. The + number of elements matches the number of dimensions + of the array. Can return ``NULL`` for 0-dimensional arrays. + +.. c:function:: npy_intp *PyArray_SHAPE(PyArrayObject *arr) + + .. versionadded:: 1.7 + + A synonym for :c:func:`PyArray_DIMS`, named to be consistent with the + `shape <numpy.ndarray.shape>` usage within Python. + .. c:function:: npy_intp *PyArray_STRIDES(PyArrayObject* arr) Returns a pointer to the strides of the array. The @@ -67,6 +97,27 @@ sub-types). Return the stride in the *n* :math:`^{\textrm{th}}` dimension. +.. c:function:: npy_intp PyArray_ITEMSIZE(PyArrayObject* arr) + + Return the itemsize for the elements of this array. + + Note that, in the old API that was deprecated in version 1.7, this function + had the return type ``int``. + +.. c:function:: npy_intp PyArray_SIZE(PyArrayObject* arr) + + Returns the total size (in number of elements) of the array. + +.. c:function:: npy_intp PyArray_Size(PyArrayObject* obj) + + Returns 0 if *obj* is not a sub-class of ndarray. Otherwise, + returns the total number of elements in the array. Safer version + of :c:func:`PyArray_SIZE` (*obj*). + +.. c:function:: npy_intp PyArray_NBYTES(PyArrayObject* arr) + + Returns the total number of bytes consumed by the array. + .. c:function:: PyObject *PyArray_BASE(PyArrayObject* arr) This returns the base object of the array. In most cases, this @@ -93,60 +144,12 @@ sub-types). A synonym for PyArray_DESCR, named to be consistent with the 'dtype' usage within Python. -.. c:function:: void PyArray_ENABLEFLAGS(PyArrayObject* arr, int flags) - - .. versionadded:: 1.7 - - Enables the specified array flags. This function does no validation, - and assumes that you know what you're doing. - -.. c:function:: void PyArray_CLEARFLAGS(PyArrayObject* arr, int flags) - - .. versionadded:: 1.7 - - Clears the specified array flags. This function does no validation, - and assumes that you know what you're doing. - -.. c:function:: int PyArray_FLAGS(PyArrayObject* arr) - -.. c:function:: npy_intp PyArray_ITEMSIZE(PyArrayObject* arr) - - Return the itemsize for the elements of this array. - - Note that, in the old API that was deprecated in version 1.7, this function - had the return type ``int``. - -.. c:function:: int PyArray_TYPE(PyArrayObject* arr) - - Return the (builtin) typenumber for the elements of this array. - .. c:function:: PyObject *PyArray_GETITEM(PyArrayObject* arr, void* itemptr) - Get a Python object of a builtin type from the ndarray, *arr*, + Get a Python object of a builtin type from the ndarray, *arr*, at the location pointed to by itemptr. Return ``NULL`` on failure. - - `numpy.ndarray.item` is identical to PyArray_GETITEM. - -.. c:function:: int PyArray_SETITEM( \ - PyArrayObject* arr, void* itemptr, PyObject* obj) - - Convert obj and place it in the ndarray, *arr*, at the place - pointed to by itemptr. Return -1 if an error occurs or 0 on - success. - -.. c:function:: npy_intp PyArray_SIZE(PyArrayObject* arr) - - Returns the total size (in number of elements) of the array. -.. c:function:: npy_intp PyArray_Size(PyArrayObject* obj) - - Returns 0 if *obj* is not a sub-class of ndarray. Otherwise, - returns the total number of elements in the array. Safer version - of :c:func:`PyArray_SIZE` (*obj*). - -.. c:function:: npy_intp PyArray_NBYTES(PyArrayObject* arr) - - Returns the total number of bytes consumed by the array. + `numpy.ndarray.item` is identical to PyArray_GETITEM. Data access @@ -223,7 +226,7 @@ From scratch If *data* is not ``NULL``, then it is assumed to point to the memory to be used for the array and the *flags* argument is used as the - new flags for the array (except the state of :c:data:`NPY_OWNDATA`, + new flags for the array (except the state of :c:data:`NPY_ARRAY_OWNDATA`, :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` and :c:data:`NPY_ARRAY_UPDATEIFCOPY` flags of the new array will be reset). @@ -913,122 +916,126 @@ enumerated array data type. For the array type checking macros the argument must be a :c:type:`PyObject *<PyObject>` that can be directly interpreted as a :c:type:`PyArrayObject *`. -.. c:function:: PyTypeNum_ISUNSIGNED(num) +.. c:function:: PyTypeNum_ISUNSIGNED(int num) -.. c:function:: PyDataType_ISUNSIGNED(descr) +.. c:function:: PyDataType_ISUNSIGNED(PyArray_Descr *descr) -.. c:function:: PyArray_ISUNSIGNED(obj) +.. c:function:: PyArray_ISUNSIGNED(PyArrayObject *obj) Type represents an unsigned integer. -.. c:function:: PyTypeNum_ISSIGNED(num) +.. c:function:: PyTypeNum_ISSIGNED(int num) -.. c:function:: PyDataType_ISSIGNED(descr) +.. c:function:: PyDataType_ISSIGNED(PyArray_Descr *descr) -.. c:function:: PyArray_ISSIGNED(obj) +.. c:function:: PyArray_ISSIGNED(PyArrayObject *obj) Type represents a signed integer. -.. c:function:: PyTypeNum_ISINTEGER(num) +.. c:function:: PyTypeNum_ISINTEGER(int num) -.. c:function:: PyDataType_ISINTEGER(descr) +.. c:function:: PyDataType_ISINTEGER(PyArray_Descr* descr) -.. c:function:: PyArray_ISINTEGER(obj) +.. c:function:: PyArray_ISINTEGER(PyArrayObject *obj) Type represents any integer. -.. c:function:: PyTypeNum_ISFLOAT(num) +.. c:function:: PyTypeNum_ISFLOAT(int num) -.. c:function:: PyDataType_ISFLOAT(descr) +.. c:function:: PyDataType_ISFLOAT(PyArray_Descr* descr) -.. c:function:: PyArray_ISFLOAT(obj) +.. c:function:: PyArray_ISFLOAT(PyArrayObject *obj) Type represents any floating point number. -.. c:function:: PyTypeNum_ISCOMPLEX(num) +.. c:function:: PyTypeNum_ISCOMPLEX(int num) -.. c:function:: PyDataType_ISCOMPLEX(descr) +.. c:function:: PyDataType_ISCOMPLEX(PyArray_Descr* descr) -.. c:function:: PyArray_ISCOMPLEX(obj) +.. c:function:: PyArray_ISCOMPLEX(PyArrayObject *obj) Type represents any complex floating point number. -.. c:function:: PyTypeNum_ISNUMBER(num) +.. c:function:: PyTypeNum_ISNUMBER(int num) -.. c:function:: PyDataType_ISNUMBER(descr) +.. c:function:: PyDataType_ISNUMBER(PyArray_Descr* descr) -.. c:function:: PyArray_ISNUMBER(obj) +.. c:function:: PyArray_ISNUMBER(PyArrayObject *obj) Type represents any integer, floating point, or complex floating point number. -.. c:function:: PyTypeNum_ISSTRING(num) +.. c:function:: PyTypeNum_ISSTRING(int num) -.. c:function:: PyDataType_ISSTRING(descr) +.. c:function:: PyDataType_ISSTRING(PyArray_Descr* descr) -.. c:function:: PyArray_ISSTRING(obj) +.. c:function:: PyArray_ISSTRING(PyArrayObject *obj) Type represents a string data type. -.. c:function:: PyTypeNum_ISPYTHON(num) +.. c:function:: PyTypeNum_ISPYTHON(int num) -.. c:function:: PyDataType_ISPYTHON(descr) +.. c:function:: PyDataType_ISPYTHON(PyArray_Descr* descr) -.. c:function:: PyArray_ISPYTHON(obj) +.. c:function:: PyArray_ISPYTHON(PyArrayObject *obj) Type represents an enumerated type corresponding to one of the standard Python scalar (bool, int, float, or complex). -.. c:function:: PyTypeNum_ISFLEXIBLE(num) +.. c:function:: PyTypeNum_ISFLEXIBLE(int num) -.. c:function:: PyDataType_ISFLEXIBLE(descr) +.. c:function:: PyDataType_ISFLEXIBLE(PyArray_Descr* descr) -.. c:function:: PyArray_ISFLEXIBLE(obj) +.. c:function:: PyArray_ISFLEXIBLE(PyArrayObject *obj) Type represents one of the flexible array types ( :c:data:`NPY_STRING`, :c:data:`NPY_UNICODE`, or :c:data:`NPY_VOID` ). -.. c:function:: PyDataType_ISUNSIZED(descr): +.. c:function:: PyDataType_ISUNSIZED(PyArray_Descr* descr): Type has no size information attached, and can be resized. Should only be called on flexible dtypes. Types that are attached to an array will always be sized, hence the array form of this macro not existing. -.. c:function:: PyTypeNum_ISUSERDEF(num) + .. versionchanged:: 1.18 -.. c:function:: PyDataType_ISUSERDEF(descr) + For structured datatypes with no fields this function now returns False. -.. c:function:: PyArray_ISUSERDEF(obj) +.. c:function:: PyTypeNum_ISUSERDEF(int num) + +.. c:function:: PyDataType_ISUSERDEF(PyArray_Descr* descr) + +.. c:function:: PyArray_ISUSERDEF(PyArrayObject *obj) Type represents a user-defined type. -.. c:function:: PyTypeNum_ISEXTENDED(num) +.. c:function:: PyTypeNum_ISEXTENDED(int num) -.. c:function:: PyDataType_ISEXTENDED(descr) +.. c:function:: PyDataType_ISEXTENDED(PyArray_Descr* descr) -.. c:function:: PyArray_ISEXTENDED(obj) +.. c:function:: PyArray_ISEXTENDED(PyArrayObject *obj) Type is either flexible or user-defined. -.. c:function:: PyTypeNum_ISOBJECT(num) +.. c:function:: PyTypeNum_ISOBJECT(int num) -.. c:function:: PyDataType_ISOBJECT(descr) +.. c:function:: PyDataType_ISOBJECT(PyArray_Descr* descr) -.. c:function:: PyArray_ISOBJECT(obj) +.. c:function:: PyArray_ISOBJECT(PyArrayObject *obj) Type represents object data type. -.. c:function:: PyTypeNum_ISBOOL(num) +.. c:function:: PyTypeNum_ISBOOL(int num) -.. c:function:: PyDataType_ISBOOL(descr) +.. c:function:: PyDataType_ISBOOL(PyArray_Descr* descr) -.. c:function:: PyArray_ISBOOL(obj) +.. c:function:: PyArray_ISBOOL(PyArrayObject *obj) Type represents Boolean data type. -.. c:function:: PyDataType_HASFIELDS(descr) +.. c:function:: PyDataType_HASFIELDS(PyArray_Descr* descr) -.. c:function:: PyArray_HASFIELDS(obj) +.. c:function:: PyArray_HASFIELDS(PyArrayObject *obj) Type has fields associated with it. @@ -1397,6 +1404,7 @@ Special functions for NPY_OBJECT Returns 0 for success, -1 for failure. +.. _array-flags: Array flags ----------- @@ -1574,9 +1582,9 @@ Flag checking ^^^^^^^^^^^^^ For all of these macros *arr* must be an instance of a (subclass of) -:c:data:`PyArray_Type`, but no checking is done. +:c:data:`PyArray_Type`. -.. c:function:: PyArray_CHKFLAGS(arr, flags) +.. c:function:: PyArray_CHKFLAGS(PyObject *arr, flags) The first parameter, arr, must be an ndarray or subclass. The parameter, *flags*, should be an integer consisting of bitwise @@ -1586,60 +1594,60 @@ For all of these macros *arr* must be an instance of a (subclass of) :c:data:`NPY_ARRAY_WRITEABLE`, :c:data:`NPY_ARRAY_WRITEBACKIFCOPY`, :c:data:`NPY_ARRAY_UPDATEIFCOPY`. -.. c:function:: PyArray_IS_C_CONTIGUOUS(arr) +.. c:function:: PyArray_IS_C_CONTIGUOUS(PyObject *arr) Evaluates true if *arr* is C-style contiguous. -.. c:function:: PyArray_IS_F_CONTIGUOUS(arr) +.. c:function:: PyArray_IS_F_CONTIGUOUS(PyObject *arr) Evaluates true if *arr* is Fortran-style contiguous. -.. c:function:: PyArray_ISFORTRAN(arr) +.. c:function:: PyArray_ISFORTRAN(PyObject *arr) Evaluates true if *arr* is Fortran-style contiguous and *not* C-style contiguous. :c:func:`PyArray_IS_F_CONTIGUOUS` is the correct way to test for Fortran-style contiguity. -.. c:function:: PyArray_ISWRITEABLE(arr) +.. c:function:: PyArray_ISWRITEABLE(PyObject *arr) Evaluates true if the data area of *arr* can be written to -.. c:function:: PyArray_ISALIGNED(arr) +.. c:function:: PyArray_ISALIGNED(PyObject *arr) Evaluates true if the data area of *arr* is properly aligned on the machine. -.. c:function:: PyArray_ISBEHAVED(arr) +.. c:function:: PyArray_ISBEHAVED(PyObject *arr) Evaluates true if the data area of *arr* is aligned and writeable and in machine byte-order according to its descriptor. -.. c:function:: PyArray_ISBEHAVED_RO(arr) +.. c:function:: PyArray_ISBEHAVED_RO(PyObject *arr) Evaluates true if the data area of *arr* is aligned and in machine byte-order. -.. c:function:: PyArray_ISCARRAY(arr) +.. c:function:: PyArray_ISCARRAY(PyObject *arr) Evaluates true if the data area of *arr* is C-style contiguous, and :c:func:`PyArray_ISBEHAVED` (*arr*) is true. -.. c:function:: PyArray_ISFARRAY(arr) +.. c:function:: PyArray_ISFARRAY(PyObject *arr) Evaluates true if the data area of *arr* is Fortran-style contiguous and :c:func:`PyArray_ISBEHAVED` (*arr*) is true. -.. c:function:: PyArray_ISCARRAY_RO(arr) +.. c:function:: PyArray_ISCARRAY_RO(PyObject *arr) Evaluates true if the data area of *arr* is C-style contiguous, aligned, and in machine byte-order. -.. c:function:: PyArray_ISFARRAY_RO(arr) +.. c:function:: PyArray_ISFARRAY_RO(PyObject *arr) Evaluates true if the data area of *arr* is Fortran-style contiguous, aligned, and in machine byte-order **.** -.. c:function:: PyArray_ISONESEGMENT(arr) +.. c:function:: PyArray_ISONESEGMENT(PyObject *arr) Evaluates true if the data area of *arr* consists of a single (C-style or Fortran-style) contiguous segment. @@ -2045,9 +2053,20 @@ Calculation .. tip:: Pass in :c:data:`NPY_MAXDIMS` for axis in order to achieve the same - effect that is obtained by passing in *axis* = :const:`None` in Python + effect that is obtained by passing in ``axis=None`` in Python (treating the array as a 1-d array). + +.. note:: + + The out argument specifies where to place the result. If out is + NULL, then the output array is created, otherwise the output is + placed in out which must be the correct size and type. A new + reference to the output array is always returned even when out + is not NULL. The caller of the routine has the responsibility + to ``Py_DECREF`` out if not NULL or a memory-leak will occur. + + .. c:function:: PyObject* PyArray_ArgMax( \ PyArrayObject* self, int axis, PyArrayObject* out) @@ -2060,18 +2079,6 @@ Calculation Equivalent to :meth:`ndarray.argmin<numpy.ndarray.argmin>` (*self*, *axis*). Return the index of the smallest element of *self* along *axis*. - - - -.. note:: - - The out argument specifies where to place the result. If out is - NULL, then the output array is created, otherwise the output is - placed in out which must be the correct size and type. A new - reference to the output array is always returned even when out - is not NULL. The caller of the routine has the responsibility - to ``DECREF`` out if not NULL or a memory-leak will occur. - .. c:function:: PyObject* PyArray_Max( \ PyArrayObject* self, int axis, PyArrayObject* out) @@ -2651,22 +2658,33 @@ cost of a slight overhead. The mode should be one of: - * NPY_NEIGHBORHOOD_ITER_ZERO_PADDING: zero padding. Outside bounds values - will be 0. - * NPY_NEIGHBORHOOD_ITER_ONE_PADDING: one padding, Outside bounds values - will be 1. - * NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING: constant padding. Outside bounds - values will be the same as the first item in fill_value. - * NPY_NEIGHBORHOOD_ITER_MIRROR_PADDING: mirror padding. Outside bounds - values will be as if the array items were mirrored. For example, for the - array [1, 2, 3, 4], x[-2] will be 2, x[-2] will be 1, x[4] will be 4, - x[5] will be 1, etc... - * NPY_NEIGHBORHOOD_ITER_CIRCULAR_PADDING: circular padding. Outside bounds - values will be as if the array was repeated. For example, for the - array [1, 2, 3, 4], x[-2] will be 3, x[-2] will be 4, x[4] will be 1, - x[5] will be 2, etc... - - If the mode is constant filling (NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING), + .. c:macro:: NPY_NEIGHBORHOOD_ITER_ZERO_PADDING + + Zero padding. Outside bounds values will be 0. + + .. c:macro:: NPY_NEIGHBORHOOD_ITER_ONE_PADDING + + One padding, Outside bounds values will be 1. + + .. c:macro:: NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING + + Constant padding. Outside bounds values will be the + same as the first item in fill_value. + + .. c:macro:: NPY_NEIGHBORHOOD_ITER_MIRROR_PADDING + + Mirror padding. Outside bounds values will be as if the + array items were mirrored. For example, for the array [1, 2, 3, 4], + x[-2] will be 2, x[-2] will be 1, x[4] will be 4, x[5] will be 1, + etc... + + .. c:macro:: NPY_NEIGHBORHOOD_ITER_CIRCULAR_PADDING + + Circular padding. Outside bounds values will be as if the array + was repeated. For example, for the array [1, 2, 3, 4], x[-2] will + be 3, x[-2] will be 4, x[4] will be 1, x[5] will be 2, etc... + + If the mode is constant filling (`NPY_NEIGHBORHOOD_ITER_CONSTANT_PADDING`), fill_value should point to an array object which holds the filling value (the first item will be the filling value if the array contains more than one item). For other cases, fill_value may be NULL. @@ -2788,10 +2806,7 @@ Array Scalars *arr* is not ``NULL`` and the first element is negative then :c:data:`NPY_INTNEG_SCALAR` is returned, otherwise :c:data:`NPY_INTPOS_SCALAR` is returned. The possible return values - are :c:data:`NPY_{kind}_SCALAR` where ``{kind}`` can be **INTPOS**, - **INTNEG**, **FLOAT**, **COMPLEX**, **BOOL**, or **OBJECT**. - :c:data:`NPY_NOSCALAR` is also an enumerated value - :c:type:`NPY_SCALARKIND` variables can take on. + are the enumerated values in :c:type:`NPY_SCALARKIND`. .. c:function:: int PyArray_CanCoerceScalar( \ char thistype, char neededtype, NPY_SCALARKIND scalar) @@ -3372,7 +3387,7 @@ Group 1 Useful to release the GIL only if *dtype* does not contain arbitrary Python objects which may need the Python interpreter - during execution of the loop. Equivalent to + during execution of the loop. .. c:function:: NPY_END_THREADS_DESCR(PyArray_Descr *dtype) @@ -3466,6 +3481,10 @@ Other constants The maximum number of dimensions allowed in arrays. +.. c:var:: NPY_MAXARGS + + The maximum number of array arguments that can be used in functions. + .. c:var:: NPY_VERSION The current version of the ndarray object (check to see if this @@ -3498,6 +3517,10 @@ Miscellaneous Macros Evaluates as True if arrays *a1* and *a2* have the same shape. +.. c:var:: a + +.. c:var:: b + .. c:macro:: PyArray_MAX(a,b) Returns the maximum of *a* and *b*. If (*a*) or (*b*) are @@ -3558,10 +3581,18 @@ Enumerated Types .. c:type:: NPY_SORTKIND - A special variable-type which can take on the values :c:data:`NPY_{KIND}` - where ``{KIND}`` is + A special variable-type which can take on different values to indicate + the sorting algorithm being used. + + .. c:var:: NPY_QUICKSORT - **QUICKSORT**, **HEAPSORT**, **MERGESORT**, **STABLESORT** + .. c:var:: NPY_HEAPSORT + + .. c:var:: NPY_MERGESORT + + .. c:var:: NPY_STABLESORT + + Used as an alias of :c:data:`NPY_MERGESORT` and vica versa. .. c:var:: NPY_NSORTS @@ -3575,11 +3606,21 @@ Enumerated Types A special variable type indicating the number of "kinds" of scalars distinguished in determining scalar-coercion rules. This - variable can take on the values :c:data:`NPY_{KIND}` where ``{KIND}`` can be + variable can take on the values: + + .. c:var:: NPY_NOSCALAR + + .. c:var:: NPY_BOOL_SCALAR + + .. c:var:: NPY_INTPOS_SCALAR + + .. c:var:: NPY_INTNEG_SCALAR + + .. c:var:: NPY_FLOAT_SCALAR + + .. c:var:: NPY_COMPLEX_SCALAR - **NOSCALAR**, **BOOL_SCALAR**, **INTPOS_SCALAR**, - **INTNEG_SCALAR**, **FLOAT_SCALAR**, **COMPLEX_SCALAR**, - **OBJECT_SCALAR** + .. c:var:: NPY_OBJECT_SCALAR .. c:var:: NPY_NSCALARKINDS diff --git a/doc/source/reference/c-api.config.rst b/doc/source/reference/c-api/config.rst index 05e6fe44d..05e6fe44d 100644 --- a/doc/source/reference/c-api.config.rst +++ b/doc/source/reference/c-api/config.rst diff --git a/doc/source/reference/c-api.coremath.rst b/doc/source/reference/c-api/coremath.rst index 7e00322f9..7e00322f9 100644 --- a/doc/source/reference/c-api.coremath.rst +++ b/doc/source/reference/c-api/coremath.rst diff --git a/doc/source/reference/c-api.deprecations.rst b/doc/source/reference/c-api/deprecations.rst index a382017a2..a382017a2 100644 --- a/doc/source/reference/c-api.deprecations.rst +++ b/doc/source/reference/c-api/deprecations.rst diff --git a/doc/source/reference/c-api.dtype.rst b/doc/source/reference/c-api/dtype.rst index 72e908861..72e908861 100644 --- a/doc/source/reference/c-api.dtype.rst +++ b/doc/source/reference/c-api/dtype.rst diff --git a/doc/source/reference/c-api.generalized-ufuncs.rst b/doc/source/reference/c-api/generalized-ufuncs.rst index b59f077ad..b59f077ad 100644 --- a/doc/source/reference/c-api.generalized-ufuncs.rst +++ b/doc/source/reference/c-api/generalized-ufuncs.rst diff --git a/doc/source/reference/c-api.rst b/doc/source/reference/c-api/index.rst index b8cbe97b2..56fe8e473 100644 --- a/doc/source/reference/c-api.rst +++ b/doc/source/reference/c-api/index.rst @@ -40,12 +40,12 @@ code. .. toctree:: :maxdepth: 2 - c-api.types-and-structures - c-api.config - c-api.dtype - c-api.array - c-api.iterator - c-api.ufunc - c-api.generalized-ufuncs - c-api.coremath - c-api.deprecations + types-and-structures + config + dtype + array + iterator + ufunc + generalized-ufuncs + coremath + deprecations diff --git a/doc/source/reference/c-api.iterator.rst b/doc/source/reference/c-api/iterator.rst index b77d029cc..b77d029cc 100644 --- a/doc/source/reference/c-api.iterator.rst +++ b/doc/source/reference/c-api/iterator.rst diff --git a/doc/source/reference/c-api.types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index a716b5a06..336dff211 100644 --- a/doc/source/reference/c-api.types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -1,3 +1,4 @@ + ***************************** Python Types and C-Structures ***************************** @@ -75,7 +76,8 @@ PyArray_Type and PyArrayObject these structure members should normally be accessed using the provided macros. If you need a shorter name, then you can make use of :c:type:`NPY_AO` (deprecated) which is defined to be equivalent to - :c:type:`PyArrayObject`. + :c:type:`PyArrayObject`. Direct access to the struct fields are + deprecated. Use the `PyArray_*(arr)` form instead. .. code-block:: c @@ -103,7 +105,8 @@ PyArray_Type and PyArrayObject .. c:member:: char *PyArrayObject.data - A pointer to the first element of the array. This pointer can + Accessible via :c:data:`PyArray_DATA`, this data member is a + pointer to the first element of the array. This pointer can (and normally should) be recast to the data type of the array. .. c:member:: int PyArrayObject.nd @@ -111,26 +114,29 @@ PyArray_Type and PyArrayObject An integer providing the number of dimensions for this array. When nd is 0, the array is sometimes called a rank-0 array. Such arrays have undefined dimensions and strides and - cannot be accessed. :c:data:`NPY_MAXDIMS` is the largest number of - dimensions for any array. + cannot be accessed. Macro :c:data:`PyArray_NDIM` defined in + ``ndarraytypes.h`` points to this data member. :c:data:`NPY_MAXDIMS` + is the largest number of dimensions for any array. .. c:member:: npy_intp PyArrayObject.dimensions An array of integers providing the shape in each dimension as long as nd :math:`\geq` 1. The integer is always large enough to hold a pointer on the platform, so the dimension size is - only limited by memory. + only limited by memory. :c:data:`PyArray_DIMS` is the macro + associated with this data member. .. c:member:: npy_intp *PyArrayObject.strides An array of integers providing for each dimension the number of bytes that must be skipped to get to the next element in that - dimension. + dimension. Associated with macro :c:data:`PyArray_STRIDES`. .. c:member:: PyObject *PyArrayObject.base - This member is used to hold a pointer to another Python object that - is related to this array. There are two use cases: + Pointed to by :c:data:`PyArray_BASE`, this member is used to hold a + pointer to another Python object that is related to this array. + There are two use cases: - If this array does not own its own memory, then base points to the Python object that owns it (perhaps another array object) @@ -149,11 +155,13 @@ PyArray_Type and PyArrayObject descriptor structure for each data type supported. This descriptor structure contains useful information about the type as well as a pointer to a table of function pointers to - implement specific functionality. + implement specific functionality. As the name suggests, it is + associated with the macro :c:data:`PyArray_DESCR`. .. c:member:: int PyArrayObject.flags - Flags indicating how the memory pointed to by data is to be + Pointed to by the macro :c:data:`PyArray_FLAGS`, this data member represents + the flags indicating how the memory pointed to by data is to be interpreted. Possible flags are :c:data:`NPY_ARRAY_C_CONTIGUOUS`, :c:data:`NPY_ARRAY_F_CONTIGUOUS`, :c:data:`NPY_ARRAY_OWNDATA`, :c:data:`NPY_ARRAY_ALIGNED`, :c:data:`NPY_ARRAY_WRITEABLE`, diff --git a/doc/source/reference/c-api.ufunc.rst b/doc/source/reference/c-api/ufunc.rst index 0499ccf5b..c9cc60141 100644 --- a/doc/source/reference/c-api.ufunc.rst +++ b/doc/source/reference/c-api/ufunc.rst @@ -21,7 +21,17 @@ Constants .. c:var:: PyUFunc_{VALUE} - ``{VALUE}`` can be **One** (1), **Zero** (0), or **None** (-1) + .. c:var:: PyUFunc_One + + .. c:var:: PyUFunc_Zero + + .. c:var:: PyUFunc_MinusOne + + .. c:var:: PyUFunc_ReorderableNone + + .. c:var:: PyUFunc_None + + .. c:var:: PyUFunc_IdentityValue Macros @@ -39,28 +49,6 @@ Macros Used in universal function code to re-acquire the Python GIL if it was released (because loop->obj was not true). -.. c:function:: UFUNC_CHECK_ERROR(loop) - - A macro used internally to check for errors and goto fail if - found. This macro requires a fail label in the current code - block. The *loop* variable must have at least members (obj, - errormask, and errorobj). If *loop* ->obj is nonzero, then - :c:func:`PyErr_Occurred` () is called (meaning the GIL must be held). If - *loop* ->obj is zero, then if *loop* ->errormask is nonzero, - :c:func:`PyUFunc_checkfperr` is called with arguments *loop* ->errormask - and *loop* ->errobj. If the result of this check of the IEEE - floating point registers is true then the code redirects to the - fail label which must be defined. - -.. c:function:: UFUNC_CHECK_STATUS(ret) - - Deprecated: use npy_clear_floatstatus from npy_math.h instead. - - A macro that expands to platform-dependent code. The *ret* - variable can be any integer. The :c:data:`UFUNC_FPE_{ERR}` bits are - set in *ret* according to the status of the corresponding error - flags of the floating point processor. - Functions --------- @@ -210,10 +198,10 @@ Functions to calling PyUFunc_FromFuncAndData. A copy of the string is made, so the passed in buffer can be freed. -.. c:function:: PyObject* PyUFunc_FromFuncAndDataAndSignatureAndIdentity( +.. c:function:: PyObject* PyUFunc_FromFuncAndDataAndSignatureAndIdentity( \ PyUFuncGenericFunction *func, void **data, char *types, int ntypes, \ - int nin, int nout, int identity, char *name, char *doc, int unused, char *signature, - PyObject *identity_value) + int nin, int nout, int identity, char *name, char *doc, int unused, \ + char *signature, PyObject *identity_value) This function is very similar to `PyUFunc_FromFuncAndDataAndSignature` above, but has an extra *identity_value* argument, to define an arbitrary identity diff --git a/doc/source/reference/distutils.rst b/doc/source/reference/distutils.rst index 46e5ec25e..a22db3e8e 100644 --- a/doc/source/reference/distutils.rst +++ b/doc/source/reference/distutils.rst @@ -22,38 +22,30 @@ information is available in the :ref:`distutils-user-guide`. Modules in :mod:`numpy.distutils` ================================= +.. toctree:: + :maxdepth: 2 -misc_util ---------- + distutils/misc_util -.. module:: numpy.distutils.misc_util + +.. currentmodule:: numpy.distutils .. autosummary:: :toctree: generated/ - get_numpy_include_dirs - dict_append - appendpath - allpath - dot_join - generate_config_py - get_cmd - terminal_has_colors - red_text - green_text - yellow_text - blue_text - cyan_text - cyg2win32 - all_strings - has_f_sources - has_cxx_sources - filter_sources - get_dependencies - is_local_src_dir - get_ext_source_files - get_script_files + ccompiler + cpuinfo.cpu + core.Extension + exec_command + log.set_verbosity + system_info.get_info + system_info.get_standard_file + + +Configuration class +=================== +.. currentmodule:: numpy.distutils.misc_util .. class:: Configuration(package_name=None, parent_name=None, top_path=None, package_path=None, **attrs) @@ -109,20 +101,6 @@ misc_util .. automethod:: get_info -Other modules -------------- - -.. currentmodule:: numpy.distutils - -.. autosummary:: - :toctree: generated/ - - system_info.get_info - system_info.get_standard_file - cpuinfo.cpu - log.set_verbosity - exec_command - Building Installable C libraries ================================ diff --git a/doc/source/reference/distutils/misc_util.rst b/doc/source/reference/distutils/misc_util.rst new file mode 100644 index 000000000..bbb83a5ab --- /dev/null +++ b/doc/source/reference/distutils/misc_util.rst @@ -0,0 +1,7 @@ +distutils.misc_util +=================== + +.. automodule:: numpy.distutils.misc_util + :members: + :undoc-members: + :exclude-members: Configuration diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst index 6accb8535..6742d605a 100644 --- a/doc/source/reference/index.rst +++ b/doc/source/reference/index.rst @@ -24,7 +24,7 @@ For learning how to use NumPy, see also :ref:`user`. routines distutils distutils_guide - c-api + c-api/index internals swig diff --git a/doc/source/reference/maskedarray.baseclass.rst b/doc/source/reference/maskedarray.baseclass.rst index 204ebfe08..5bbdd0299 100644 --- a/doc/source/reference/maskedarray.baseclass.rst +++ b/doc/source/reference/maskedarray.baseclass.rst @@ -160,9 +160,9 @@ replaced with ``n`` integers which will be interpreted as an n-tuple. Item selection and manipulation ------------------------------- -For array methods that take an *axis* keyword, it defaults to `None`. -If axis is *None*, then the array is treated as a 1-D array. -Any other value for *axis* represents the dimension along which +For array methods that take an ``axis`` keyword, it defaults to None. +If axis is None, then the array is treated as a 1-D array. +Any other value for ``axis`` represents the dimension along which the operation should proceed. .. autosummary:: diff --git a/doc/source/reference/maskedarray.generic.rst b/doc/source/reference/maskedarray.generic.rst index 7375d60fb..41c3ee564 100644 --- a/doc/source/reference/maskedarray.generic.rst +++ b/doc/source/reference/maskedarray.generic.rst @@ -74,7 +74,7 @@ To create an array with the second element invalid, we would do:: To create a masked array where all values close to 1.e20 are invalid, we would do:: - >>> z = masked_values([1.0, 1.e20, 3.0, 4.0], 1.e20) + >>> z = ma.masked_values([1.0, 1.e20, 3.0, 4.0], 1.e20) For a complete discussion of creation methods for masked arrays please see section :ref:`Constructing masked arrays <maskedarray.generic.constructing>`. @@ -110,15 +110,15 @@ There are several ways to construct a masked array. >>> x = np.array([1, 2, 3]) >>> x.view(ma.MaskedArray) - masked_array(data = [1 2 3], - mask = False, - fill_value = 999999) + masked_array(data=[1, 2, 3], + mask=False, + fill_value=999999) >>> x = np.array([(1, 1.), (2, 2.)], dtype=[('a',int), ('b', float)]) >>> x.view(ma.MaskedArray) - masked_array(data = [(1, 1.0) (2, 2.0)], - mask = [(False, False) (False, False)], - fill_value = (999999, 1e+20), - dtype = [('a', '<i4'), ('b', '<f8')]) + masked_array(data=[(1, 1.0), (2, 2.0)], + mask=[(False, False), (False, False)], + fill_value=(999999, 1.e+20), + dtype=[('a', '<i8'), ('b', '<f8')]) * Yet another possibility is to use any of the following functions: @@ -195,9 +195,9 @@ index. The inverse of the mask can be calculated with the >>> x = ma.array([[1, 2], [3, 4]], mask=[[0, 1], [1, 0]]) >>> x[~x.mask] - masked_array(data = [1 4], - mask = [False False], - fill_value = 999999) + masked_array(data=[1, 4], + mask=[False, False], + fill_value=999999) Another way to retrieve the valid data is to use the :meth:`compressed` method, which returns a one-dimensional :class:`~numpy.ndarray` (or one of its @@ -223,27 +223,26 @@ as invalid is to assign the special value :attr:`masked` to them:: >>> x = ma.array([1, 2, 3]) >>> x[0] = ma.masked >>> x - masked_array(data = [-- 2 3], - mask = [ True False False], - fill_value = 999999) + masked_array(data=[--, 2, 3], + mask=[ True, False, False], + fill_value=999999) >>> y = ma.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> y[(0, 1, 2), (1, 2, 0)] = ma.masked >>> y - masked_array(data = - [[1 -- 3] - [4 5 --] - [-- 8 9]], - mask = - [[False True False] - [False False True] - [ True False False]], - fill_value = 999999) + masked_array( + data=[[1, --, 3], + [4, 5, --], + [--, 8, 9]], + mask=[[False, True, False], + [False, False, True], + [ True, False, False]], + fill_value=999999) >>> z = ma.array([1, 2, 3, 4]) >>> z[:-2] = ma.masked >>> z - masked_array(data = [-- -- 3 4], - mask = [ True True False False], - fill_value = 999999) + masked_array(data=[--, --, 3, 4], + mask=[ True, True, False, False], + fill_value=999999) A second possibility is to modify the :attr:`~MaskedArray.mask` directly, @@ -263,9 +262,10 @@ mask:: >>> x = ma.array([1, 2, 3], mask=[0, 0, 1]) >>> x.mask = True >>> x - masked_array(data = [-- -- --], - mask = [ True True True], - fill_value = 999999) + masked_array(data=[--, --, --], + mask=[ True, True, True], + fill_value=999999, + dtype=int64) Finally, specific entries can be masked and/or unmasked by assigning to the mask a sequence of booleans:: @@ -273,9 +273,9 @@ mask a sequence of booleans:: >>> x = ma.array([1, 2, 3]) >>> x.mask = [0, 1, 0] >>> x - masked_array(data = [1 -- 3], - mask = [False True False], - fill_value = 999999) + masked_array(data=[1, --, 3], + mask=[False, True, False], + fill_value=999999) Unmasking an entry ~~~~~~~~~~~~~~~~~~ @@ -285,14 +285,14 @@ new valid values to them:: >>> x = ma.array([1, 2, 3], mask=[0, 0, 1]) >>> x - masked_array(data = [1 2 --], - mask = [False False True], - fill_value = 999999) + masked_array(data=[1, 2, --], + mask=[False, False, True], + fill_value=999999) >>> x[-1] = 5 >>> x - masked_array(data = [1 2 5], - mask = [False False False], - fill_value = 999999) + masked_array(data=[1, 2, 5], + mask=[False, False, False], + fill_value=999999) .. note:: Unmasking an entry by direct assignment will silently fail if the masked @@ -304,21 +304,27 @@ new valid values to them:: >>> x = ma.array([1, 2, 3], mask=[0, 0, 1], hard_mask=True) >>> x - masked_array(data = [1 2 --], - mask = [False False True], - fill_value = 999999) + masked_array(data=[1, 2, --], + mask=[False, False, True], + fill_value=999999) >>> x[-1] = 5 >>> x - masked_array(data = [1 2 --], - mask = [False False True], - fill_value = 999999) + masked_array(data=[1, 2, --], + mask=[False, False, True], + fill_value=999999) >>> x.soften_mask() + masked_array(data=[1, 2, --], + mask=[False, False, True], + fill_value=999999) >>> x[-1] = 5 >>> x - masked_array(data = [1 2 5], - mask = [False False False], - fill_value = 999999) + masked_array(data=[1, 2, 5], + mask=[False, False, False], + fill_value=999999) >>> x.harden_mask() + masked_array(data=[1, 2, 5], + mask=[False, False, False], + fill_value=999999) To unmask all masked entries of a masked array (provided the mask isn't a hard @@ -327,15 +333,14 @@ mask:: >>> x = ma.array([1, 2, 3], mask=[0, 0, 1]) >>> x - masked_array(data = [1 2 --], - mask = [False False True], - fill_value = 999999) + masked_array(data=[1, 2, --], + mask=[False, False, True], + fill_value=999999) >>> x.mask = ma.nomask >>> x - masked_array(data = [1 2 3], - mask = [False False False], - fill_value = 999999) - + masked_array(data=[1, 2, 3], + mask=[False, False, False], + fill_value=999999) Indexing and slicing @@ -353,9 +358,7 @@ the mask is ``True``):: >>> x[0] 1 >>> x[-1] - masked_array(data = --, - mask = True, - fill_value = 1e+20) + masked >>> x[-1] is ma.masked True @@ -370,10 +373,7 @@ is masked. >>> y[0] (1, 2) >>> y[-1] - masked_array(data = (3, --), - mask = (False, True), - fill_value = (999999, 999999), - dtype = [('a', '<i4'), ('b', '<i4')]) + (3, --) When accessing a slice, the output is a masked array whose @@ -385,20 +385,19 @@ required to ensure propagation of any modification of the mask to the original. >>> x = ma.array([1, 2, 3, 4, 5], mask=[0, 1, 0, 0, 1]) >>> mx = x[:3] >>> mx - masked_array(data = [1 -- 3], - mask = [False True False], - fill_value = 999999) + masked_array(data=[1, --, 3], + mask=[False, True, False], + fill_value=999999) >>> mx[1] = -1 >>> mx - masked_array(data = [1 -1 3], - mask = [False False False], - fill_value = 999999) + masked_array(data=[1, -1, 3], + mask=[False, False, False], + fill_value=999999) >>> x.mask - array([False, True, False, False, True]) + array([False, False, False, False, True]) >>> x.data array([ 1, -1, 3, 4, 5]) - Accessing a field of a masked array with structured datatype returns a :class:`MaskedArray`. @@ -421,9 +420,9 @@ ufuncs. Unary and binary functions that have a validity domain (such as constant whenever the input is masked or falls outside the validity domain:: >>> ma.log([-1, 0, 1, 2]) - masked_array(data = [-- -- 0.0 0.69314718056], - mask = [ True True False False], - fill_value = 1e+20) + masked_array(data=[--, --, 0.0, 0.6931471805599453], + mask=[ True, True, False, False], + fill_value=1e+20) Masked arrays also support standard numpy ufuncs. The output is then a masked array. The result of a unary ufunc is masked wherever the input is masked. The @@ -435,10 +434,9 @@ input fall outside the validity domain:: >>> x = ma.array([-1, 1, 0, 2, 3], mask=[0, 0, 0, 0, 1]) >>> np.log(x) - masked_array(data = [-- -- 0.0 0.69314718056 --], - mask = [ True True False False True], - fill_value = 1e+20) - + masked_array(data=[--, 0.0, --, 0.6931471805599453, --], + mask=[ True, False, True, False, True], + fill_value=1e+20) Examples @@ -454,11 +452,11 @@ of anomalies (deviations from the average):: >>> import numpy.ma as ma >>> x = [0.,1.,-9999.,3.,4.] >>> mx = ma.masked_values (x, -9999.) - >>> print mx.mean() + >>> print(mx.mean()) 2.0 - >>> print mx - mx.mean() + >>> print(mx - mx.mean()) [-2.0 -1.0 -- 1.0 2.0] - >>> print mx.anom() + >>> print(mx.anom()) [-2.0 -1.0 -- 1.0 2.0] @@ -468,7 +466,7 @@ Filling in the missing data Suppose now that we wish to print that same data, but with the missing values replaced by the average value. - >>> print mx.filled(mx.mean()) + >>> print(mx.filled(mx.mean())) [ 0. 1. 2. 3. 4.] @@ -478,10 +476,10 @@ Numerical operations Numerical operations can be easily performed without worrying about missing values, dividing by zero, square roots of negative numbers, etc.:: - >>> import numpy as np, numpy.ma as ma + >>> import numpy.ma as ma >>> x = ma.array([1., -1., 3., 4., 5., 6.], mask=[0,0,0,0,1,0]) >>> y = ma.array([1., 2., 0., 4., 5., 6.], mask=[0,0,0,0,0,1]) - >>> print np.sqrt(x/y) + >>> print(ma.sqrt(x/y)) [1.0 -- -- 1.0 -- --] Four values of the output are invalid: the first one comes from taking the @@ -492,8 +490,10 @@ the last two where the inputs were masked. Ignoring extreme values ----------------------- -Let's consider an array ``d`` of random floats between 0 and 1. We wish to +Let's consider an array ``d`` of floats between 0 and 1. We wish to compute the average of the values of ``d`` while ignoring any data outside -the range ``[0.1, 0.9]``:: +the range ``[0.2, 0.9]``:: - >>> print ma.masked_outside(d, 0.1, 0.9).mean() + >>> d = np.linspace(0, 1, 20) + >>> print(d.mean() - ma.masked_outside(d, 0.2, 0.9).mean()) + -0.05263157894736836 diff --git a/doc/source/reference/random/bit_generators/bitgenerators.rst b/doc/source/reference/random/bit_generators/bitgenerators.rst deleted file mode 100644 index 1474f7dac..000000000 --- a/doc/source/reference/random/bit_generators/bitgenerators.rst +++ /dev/null @@ -1,11 +0,0 @@ -:orphan: - -BitGenerator ------------- - -.. currentmodule:: numpy.random.bit_generator - -.. autosummary:: - :toctree: generated/ - - BitGenerator diff --git a/doc/source/reference/random/bit_generators/index.rst b/doc/source/reference/random/bit_generators/index.rst index 35d9e5d09..315657172 100644 --- a/doc/source/reference/random/bit_generators/index.rst +++ b/doc/source/reference/random/bit_generators/index.rst @@ -1,5 +1,3 @@ -.. _bit_generator: - .. currentmodule:: numpy.random Bit Generators @@ -21,7 +19,7 @@ The included BitGenerators are: and can be advanced by an arbitrary amount. See the documentation for :meth:`~.PCG64.advance`. PCG-64 has a period of :math:`2^{128}`. See the `PCG author's page`_ for more details about this class of PRNG. -* MT19937 - The standard Python BitGenerator. Adds a `~mt19937.MT19937.jumped` +* MT19937 - The standard Python BitGenerator. Adds a `MT19937.jumped` function that returns a new generator with state as-if :math:`2^{128}` draws have been made. * Philox - A counter-based generator capable of being advanced an @@ -35,14 +33,18 @@ The included BitGenerators are: .. _`Random123`: https://www.deshawresearch.com/resources_random123.html .. _`SFC author's page`: http://pracrand.sourceforge.net/RNG_engines.txt +.. autosummary:: + :toctree: generated/ + + BitGenerator + .. toctree:: - :maxdepth: 1 + :maxdepth: 1 - BitGenerator <bitgenerators> - MT19937 <mt19937> - PCG64 <pcg64> - Philox <philox> - SFC64 <sfc64> + MT19937 <mt19937> + PCG64 <pcg64> + Philox <philox> + SFC64 <sfc64> Seeding and Entropy ------------------- @@ -53,14 +55,14 @@ seed. All of the provided BitGenerators will take an arbitrary-sized non-negative integer, or a list of such integers, as a seed. BitGenerators need to take those inputs and process them into a high-quality internal state for the BitGenerator. All of the BitGenerators in numpy delegate that task to -`~SeedSequence`, which uses hashing techniques to ensure that even low-quality +`SeedSequence`, which uses hashing techniques to ensure that even low-quality seeds generate high-quality initial states. .. code-block:: python - from numpy.random import PCG64 + from numpy.random import PCG64 - bg = PCG64(12345678903141592653589793) + bg = PCG64(12345678903141592653589793) .. end_block @@ -75,14 +77,14 @@ user, which is up to you. .. code-block:: python - from numpy.random import PCG64, SeedSequence + from numpy.random import PCG64, SeedSequence - # Get the user's seed somehow, maybe through `argparse`. - # If the user did not provide a seed, it should return `None`. - seed = get_user_seed() - ss = SeedSequence(seed) - print('seed = {}'.format(ss.entropy)) - bg = PCG64(ss) + # Get the user's seed somehow, maybe through `argparse`. + # If the user did not provide a seed, it should return `None`. + seed = get_user_seed() + ss = SeedSequence(seed) + print('seed = {}'.format(ss.entropy)) + bg = PCG64(ss) .. end_block @@ -104,9 +106,6 @@ or using ``secrets.randbits(128)`` from the standard library are both convenient ways. .. autosummary:: - :toctree: generated/ + :toctree: generated/ SeedSequence - bit_generator.ISeedSequence - bit_generator.ISpawnableSeedSequence - bit_generator.SeedlessSeedSequence diff --git a/doc/source/reference/random/bit_generators/pcg64.rst b/doc/source/reference/random/bit_generators/pcg64.rst index 5881b7008..edac4620b 100644 --- a/doc/source/reference/random/bit_generators/pcg64.rst +++ b/doc/source/reference/random/bit_generators/pcg64.rst @@ -1,5 +1,5 @@ -Parallel Congruent Generator (64-bit, PCG64) --------------------------------------------- +Permuted Congruential Generator (64-bit, PCG64) +----------------------------------------------- .. currentmodule:: numpy.random diff --git a/doc/source/reference/random/c-api.rst b/doc/source/reference/random/c-api.rst new file mode 100644 index 000000000..0d60f4d9e --- /dev/null +++ b/doc/source/reference/random/c-api.rst @@ -0,0 +1,192 @@ +Cython API for random +--------------------- + +.. currentmodule:: numpy.random + +Typed versions of many of the `Generator` and `BitGenerator` methods as well as +the classes themselves can be accessed directly from Cython via + +.. code-block:: cython + + cimport numpy.random + +C API for random +---------------- + +Access to various distributions is available via Cython or C-wrapper libraries +like CFFI. All the functions accept a :c:type:`bitgen_t` as their first argument. + +.. c:type:: bitgen_t + + The :c:type:`bitgen_t` holds the current state of the BitGenerator and + pointers to functions that return standard C types while advancing the + state. + + .. code-block:: c + + struct bitgen: + void *state + npy_uint64 (*next_uint64)(void *st) nogil + uint32_t (*next_uint32)(void *st) nogil + double (*next_double)(void *st) nogil + npy_uint64 (*next_raw)(void *st) nogil + + ctypedef bitgen bitgen_t + +See :doc:`extending` for examples of using these functions. + +The functions are named with the following conventions: + +- "standard" refers to the reference values for any parameters. For instance + "standard_uniform" means a uniform distribution on the interval ``0.0`` to + ``1.0`` + +- "fill" functions will fill the provided ``out`` with ``cnt`` values. + +- The functions without "standard" in their name require additional parameters + to describe the distributions. + +- ``zig`` in the name are based on a ziggurat lookup algorithm is used instead + of calculating the ``log``, which is significantly faster. The non-ziggurat + variants are used in corner cases and for legacy compatibility. + + +.. c:function:: double random_standard_uniform(bitgen_t *bitgen_state) + +.. c:function:: void random_standard_uniform_fill(bitgen_t* bitgen_state, npy_intp cnt, double *out) + +.. c:function:: double random_standard_exponential(bitgen_t *bitgen_state) + +.. c:function:: void random_standard_exponential_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) + +.. c:function:: double random_standard_normal(bitgen_t* bitgen_state) + +.. c:function:: void random_standard_normal_fill(bitgen_t *bitgen_state, npy_intp count, double *out) + +.. c:function:: void random_standard_normal_fill_f(bitgen_t *bitgen_state, npy_intp count, float *out) + +.. c:function:: double random_standard_gamma(bitgen_t *bitgen_state, double shape) + +.. c:function:: float random_standard_uniform_f(bitgen_t *bitgen_state) + +.. c:function:: void random_standard_uniform_fill_f(bitgen_t* bitgen_state, npy_intp cnt, float *out) + +.. c:function:: float random_standard_exponential_f(bitgen_t *bitgen_state) + +.. c:function:: void random_standard_exponential_fill_f(bitgen_t *bitgen_state, npy_intp cnt, float *out) + +.. c:function:: float random_standard_normal_f(bitgen_t* bitgen_state) + +.. c:function:: float random_standard_gamma_f(bitgen_t *bitgen_state, float shape) + +.. c:function:: double random_normal(bitgen_t *bitgen_state, double loc, double scale) + +.. c:function:: double random_gamma(bitgen_t *bitgen_state, double shape, double scale) + +.. c:function:: float random_gamma_f(bitgen_t *bitgen_state, float shape, float scale) + +.. c:function:: double random_exponential(bitgen_t *bitgen_state, double scale) + +.. c:function:: double random_uniform(bitgen_t *bitgen_state, double lower, double range) + +.. c:function:: double random_beta(bitgen_t *bitgen_state, double a, double b) + +.. c:function:: double random_chisquare(bitgen_t *bitgen_state, double df) + +.. c:function:: double random_f(bitgen_t *bitgen_state, double dfnum, double dfden) + +.. c:function:: double random_standard_cauchy(bitgen_t *bitgen_state) + +.. c:function:: double random_pareto(bitgen_t *bitgen_state, double a) + +.. c:function:: double random_weibull(bitgen_t *bitgen_state, double a) + +.. c:function:: double random_power(bitgen_t *bitgen_state, double a) + +.. c:function:: double random_laplace(bitgen_t *bitgen_state, double loc, double scale) + +.. c:function:: double random_gumbel(bitgen_t *bitgen_state, double loc, double scale) + +.. c:function:: double random_logistic(bitgen_t *bitgen_state, double loc, double scale) + +.. c:function:: double random_lognormal(bitgen_t *bitgen_state, double mean, double sigma) + +.. c:function:: double random_rayleigh(bitgen_t *bitgen_state, double mode) + +.. c:function:: double random_standard_t(bitgen_t *bitgen_state, double df) + +.. c:function:: double random_noncentral_chisquare(bitgen_t *bitgen_state, double df, double nonc) +.. c:function:: double random_noncentral_f(bitgen_t *bitgen_state, double dfnum, double dfden, double nonc) +.. c:function:: double random_wald(bitgen_t *bitgen_state, double mean, double scale) + +.. c:function:: double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) + +.. c:function:: double random_triangular(bitgen_t *bitgen_state, double left, double mode, double right) + +.. c:function:: npy_int64 random_poisson(bitgen_t *bitgen_state, double lam) + +.. c:function:: npy_int64 random_negative_binomial(bitgen_t *bitgen_state, double n, double p) + +.. c:type:: binomial_t + + .. code-block:: c + + typedef struct s_binomial_t { + int has_binomial; /* !=0: following parameters initialized for binomial */ + double psave; + RAND_INT_TYPE nsave; + double r; + double q; + double fm; + RAND_INT_TYPE m; + double p1; + double xm; + double xl; + double xr; + double c; + double laml; + double lamr; + double p2; + double p3; + double p4; + } binomial_t; + + +.. c:function:: npy_int64 random_binomial(bitgen_t *bitgen_state, double p, npy_int64 n, binomial_t *binomial) + +.. c:function:: npy_int64 random_logseries(bitgen_t *bitgen_state, double p) + +.. c:function:: npy_int64 random_geometric_search(bitgen_t *bitgen_state, double p) + +.. c:function:: npy_int64 random_geometric_inversion(bitgen_t *bitgen_state, double p) + +.. c:function:: npy_int64 random_geometric(bitgen_t *bitgen_state, double p) + +.. c:function:: npy_int64 random_zipf(bitgen_t *bitgen_state, double a) + +.. c:function:: npy_int64 random_hypergeometric(bitgen_t *bitgen_state, npy_int64 good, npy_int64 bad, npy_int64 sample) + +.. c:function:: npy_uint64 random_interval(bitgen_t *bitgen_state, npy_uint64 max) + +.. c:function:: void random_multinomial(bitgen_t *bitgen_state, npy_int64 n, npy_int64 *mnix, double *pix, npy_intp d, binomial_t *binomial) + +.. c:function:: int random_multivariate_hypergeometric_count(bitgen_t *bitgen_state, npy_int64 total, size_t num_colors, npy_int64 *colors, npy_int64 nsample, size_t num_variates, npy_int64 *variates) + +.. c:function:: void random_multivariate_hypergeometric_marginals(bitgen_t *bitgen_state, npy_int64 total, size_t num_colors, npy_int64 *colors, npy_int64 nsample, size_t num_variates, npy_int64 *variates) + +Generate a single integer + +.. c:function:: npy_int64 random_positive_int64(bitgen_t *bitgen_state) + +.. c:function:: npy_int32 random_positive_int32(bitgen_t *bitgen_state) + +.. c:function:: npy_int64 random_positive_int(bitgen_t *bitgen_state) + +.. c:function:: npy_uint64 random_uint(bitgen_t *bitgen_state) + + +Generate random uint64 numbers in closed interval [off, off + rng]. + +.. c:function:: npy_uint64 random_bounded_uint64(bitgen_t *bitgen_state, npy_uint64 off, npy_uint64 rng, npy_uint64 mask, bint use_masked) + + diff --git a/doc/source/reference/random/examples/cffi.rst b/doc/source/reference/random/examples/cffi.rst new file mode 100644 index 000000000..04d52203b --- /dev/null +++ b/doc/source/reference/random/examples/cffi.rst @@ -0,0 +1,5 @@ +Extending via CFFI +------------------ + +.. literalinclude:: ../../../../../numpy/random/_examples/cffi/extending.py + :language: python diff --git a/doc/source/reference/random/examples/cython/extending.pyx b/doc/source/reference/random/examples/cython/extending.pyx new file mode 100644 index 000000000..0cfbc146f --- /dev/null +++ b/doc/source/reference/random/examples/cython/extending.pyx @@ -0,0 +1,4 @@ +extending.pyx +------------- + +.. include:: ../../../../../../numpy/random/examples/extending.pyx diff --git a/doc/source/reference/random/examples/cython/extending.pyx.rst b/doc/source/reference/random/examples/cython/extending.pyx.rst new file mode 100644 index 000000000..e2bba5aa4 --- /dev/null +++ b/doc/source/reference/random/examples/cython/extending.pyx.rst @@ -0,0 +1,5 @@ +extending.pyx +------------- + +.. literalinclude:: ../../../../../../numpy/random/_examples/cython/extending.pyx + :language: cython diff --git a/doc/source/reference/random/examples/cython/extending_distributions.pyx.rst b/doc/source/reference/random/examples/cython/extending_distributions.pyx.rst new file mode 100644 index 000000000..f64921c67 --- /dev/null +++ b/doc/source/reference/random/examples/cython/extending_distributions.pyx.rst @@ -0,0 +1,5 @@ +extending_distributions.pyx +--------------------------- + +.. literalinclude:: ../../../../../../numpy/random/_examples/cython/extending_distributions.pyx + :language: cython diff --git a/doc/source/reference/random/examples/cython/index.rst b/doc/source/reference/random/examples/cython/index.rst new file mode 100644 index 000000000..368f5fcd5 --- /dev/null +++ b/doc/source/reference/random/examples/cython/index.rst @@ -0,0 +1,11 @@ + +.. _extending_cython_example: + +Extending `numpy.random` via Cython +----------------------------------- + + +.. toctree:: + setup.py.rst + extending.pyx + extending_distributions.pyx diff --git a/doc/source/reference/random/examples/cython/setup.py.rst b/doc/source/reference/random/examples/cython/setup.py.rst new file mode 100644 index 000000000..bc7a74c59 --- /dev/null +++ b/doc/source/reference/random/examples/cython/setup.py.rst @@ -0,0 +1,5 @@ +setup.py +-------- + +.. literalinclude:: ../../../../../../numpy/random/_examples/cython/setup.py + :language: python diff --git a/doc/source/reference/random/examples/numba.rst b/doc/source/reference/random/examples/numba.rst new file mode 100644 index 000000000..b41a02568 --- /dev/null +++ b/doc/source/reference/random/examples/numba.rst @@ -0,0 +1,5 @@ +Extending via Numba +------------------- + +.. literalinclude:: ../../../../../numpy/random/_examples/numba/extending.py + :language: python diff --git a/doc/source/reference/random/examples/numba_cffi.rst b/doc/source/reference/random/examples/numba_cffi.rst new file mode 100644 index 000000000..fb2f85cce --- /dev/null +++ b/doc/source/reference/random/examples/numba_cffi.rst @@ -0,0 +1,5 @@ +Extending via Numba and CFFI +---------------------------- + +.. literalinclude:: ../../../../../numpy/random/_examples/numba/extending_distributions.py + :language: python diff --git a/doc/source/reference/random/extending.rst b/doc/source/reference/random/extending.rst new file mode 100644 index 000000000..4adb90c06 --- /dev/null +++ b/doc/source/reference/random/extending.rst @@ -0,0 +1,109 @@ +.. currentmodule:: numpy.random + +.. _extending: + +Extending +--------- +The BitGenerators have been designed to be extendable using standard tools for +high-performance Python -- numba and Cython. The `~Generator` object can also +be used with user-provided BitGenerators as long as these export a small set of +required functions. + +Numba +===== +Numba can be used with either CTypes or CFFI. The current iteration of the +BitGenerators all export a small set of functions through both interfaces. + +This example shows how numba can be used to produce gaussian samples using +a pure Python implementation which is then compiled. The random numbers are +provided by ``ctypes.next_double``. + +.. literalinclude:: ../../../../numpy/random/_examples/numba/extending.py + :language: python + :end-before: example 2 + +Both CTypes and CFFI allow the more complicated distributions to be used +directly in Numba after compiling the file distributions.c into a ``DLL`` or +``so``. An example showing the use of a more complicated distribution is in +the `examples` section below. + +.. _random_cython: + +Cython +====== + +Cython can be used to unpack the ``PyCapsule`` provided by a BitGenerator. +This example uses `PCG64` and the example from above. The usual caveats +for writing high-performance code using Cython -- removing bounds checks and +wrap around, providing array alignment information -- still apply. + +.. literalinclude:: ../../../../numpy/random/_examples/cython/extending_distributions.pyx + :language: cython + :end-before: example 2 + +The BitGenerator can also be directly accessed using the members of the basic +RNG structure. + +.. literalinclude:: ../../../../numpy/random/_examples/cython/extending_distributions.pyx + :language: cython + :start-after: example 2 + +See :ref:`extending_cython_example` for a complete working example including a +minimal setup and cython files. + +CFFI +==== + +CFFI can be used to directly access the functions in +``include/numpy/random/distributions.h``. Some "massaging" of the header +file is required: + +.. literalinclude:: ../../../../numpy/random/_examples/cffi/extending.py + :language: python + :end-before: dlopen + +Once the header is parsed by ``ffi.cdef``, the functions can be accessed +directly from the ``_generator`` shared object, using the `BitGenerator.cffi` interface. + +.. literalinclude:: ../../../../numpy/random/_examples/cffi/extending.py + :language: python + :start-after: dlopen + + +New Basic RNGs +============== +`~Generator` can be used with other user-provided BitGenerators. The simplest +way to write a new BitGenerator is to examine the pyx file of one of the +existing BitGenerators. The key structure that must be provided is the +``capsule`` which contains a ``PyCapsule`` to a struct pointer of type +``bitgen_t``, + +.. code-block:: c + + typedef struct bitgen { + void *state; + uint64_t (*next_uint64)(void *st); + uint32_t (*next_uint32)(void *st); + double (*next_double)(void *st); + uint64_t (*next_raw)(void *st); + } bitgen_t; + +which provides 5 pointers. The first is an opaque pointer to the data structure +used by the BitGenerators. The next three are function pointers which return +the next 64- and 32-bit unsigned integers, the next random double and the next +raw value. This final function is used for testing and so can be set to +the next 64-bit unsigned integer function if not needed. Functions inside +``Generator`` use this structure as in + +.. code-block:: c + + bitgen_state->next_uint64(bitgen_state->state) + +Examples +======== + +.. toctree:: + Numba <examples/numba> + CFFI + Numba <examples/numba_cffi> + Cython <examples/cython/index> + CFFI <examples/cffi> diff --git a/doc/source/reference/random/generator.rst b/doc/source/reference/random/generator.rst index c3803bcab..a2cbb493a 100644 --- a/doc/source/reference/random/generator.rst +++ b/doc/source/reference/random/generator.rst @@ -22,63 +22,64 @@ Accessing the BitGenerator .. autosummary:: :toctree: generated/ - ~Generator.bit_generator + ~numpy.random.Generator.bit_generator Simple random data ================== .. autosummary:: :toctree: generated/ - ~Generator.integers - ~Generator.random - ~Generator.choice - ~Generator.bytes + ~numpy.random.Generator.integers + ~numpy.random.Generator.random + ~numpy.random.Generator.choice + ~numpy.random.Generator.bytes Permutations ============ .. autosummary:: :toctree: generated/ - ~Generator.shuffle - ~Generator.permutation + ~numpy.random.Generator.shuffle + ~numpy.random.Generator.permutation Distributions ============= .. autosummary:: :toctree: generated/ - ~Generator.beta - ~Generator.binomial - ~Generator.chisquare - ~Generator.dirichlet - ~Generator.exponential - ~Generator.f - ~Generator.gamma - ~Generator.geometric - ~Generator.gumbel - ~Generator.hypergeometric - ~Generator.laplace - ~Generator.logistic - ~Generator.lognormal - ~Generator.logseries - ~Generator.multinomial - ~Generator.multivariate_normal - ~Generator.negative_binomial - ~Generator.noncentral_chisquare - ~Generator.noncentral_f - ~Generator.normal - ~Generator.pareto - ~Generator.poisson - ~Generator.power - ~Generator.rayleigh - ~Generator.standard_cauchy - ~Generator.standard_exponential - ~Generator.standard_gamma - ~Generator.standard_normal - ~Generator.standard_t - ~Generator.triangular - ~Generator.uniform - ~Generator.vonmises - ~Generator.wald - ~Generator.weibull - ~Generator.zipf + ~numpy.random.Generator.beta + ~numpy.random.Generator.binomial + ~numpy.random.Generator.chisquare + ~numpy.random.Generator.dirichlet + ~numpy.random.Generator.exponential + ~numpy.random.Generator.f + ~numpy.random.Generator.gamma + ~numpy.random.Generator.geometric + ~numpy.random.Generator.gumbel + ~numpy.random.Generator.hypergeometric + ~numpy.random.Generator.laplace + ~numpy.random.Generator.logistic + ~numpy.random.Generator.lognormal + ~numpy.random.Generator.logseries + ~numpy.random.Generator.multinomial + ~numpy.random.Generator.multivariate_hypergeometric + ~numpy.random.Generator.multivariate_normal + ~numpy.random.Generator.negative_binomial + ~numpy.random.Generator.noncentral_chisquare + ~numpy.random.Generator.noncentral_f + ~numpy.random.Generator.normal + ~numpy.random.Generator.pareto + ~numpy.random.Generator.poisson + ~numpy.random.Generator.power + ~numpy.random.Generator.rayleigh + ~numpy.random.Generator.standard_cauchy + ~numpy.random.Generator.standard_exponential + ~numpy.random.Generator.standard_gamma + ~numpy.random.Generator.standard_normal + ~numpy.random.Generator.standard_t + ~numpy.random.Generator.triangular + ~numpy.random.Generator.uniform + ~numpy.random.Generator.vonmises + ~numpy.random.Generator.wald + ~numpy.random.Generator.weibull + ~numpy.random.Generator.zipf diff --git a/doc/source/reference/random/index.rst b/doc/source/reference/random/index.rst index 641c2164a..bda9c4d96 100644 --- a/doc/source/reference/random/index.rst +++ b/doc/source/reference/random/index.rst @@ -22,34 +22,44 @@ Since Numpy version 1.17.0 the Generator can be initialized with a number of different BitGenerators. It exposes many different probability distributions. See `NEP 19 <https://www.numpy.org/neps/ nep-0019-rng-policy.html>`_ for context on the updated random Numpy number -routines. The legacy `.RandomState` random number routines are still +routines. The legacy `RandomState` random number routines are still available, but limited to a single BitGenerator. -For convenience and backward compatibility, a single `~.RandomState` +For convenience and backward compatibility, a single `RandomState` instance's methods are imported into the numpy.random namespace, see :ref:`legacy` for the complete list. +.. _random-quick-start: + Quick Start ----------- -By default, `~Generator` uses bits provided by `~pcg64.PCG64` which -has better statistical properties than the legacy mt19937 random -number generator in `~.RandomState`. +Call `default_rng` to get a new instance of a `Generator`, then call its +methods to obtain samples from different distributions. By default, +`Generator` uses bits provided by `PCG64` which has better statistical +properties than the legacy `MT19937` used in `RandomState`. .. code-block:: python - # Uses the old numpy.random.RandomState + # Do this + from numpy.random import default_rng + rng = default_rng() + vals = rng.standard_normal(10) + more_vals = rng.standard_normal(10) + + # instead of this from numpy import random - random.standard_normal() + vals = random.standard_normal(10) + more_vals = random.standard_normal(10) -`~Generator` can be used as a replacement for `~.RandomState`. Both class -instances now hold a internal `BitGenerator` instance to provide the bit +`Generator` can be used as a replacement for `RandomState`. Both class +instances hold a internal `BitGenerator` instance to provide the bit stream, it is accessible as ``gen.bit_generator``. Some long-overdue API cleanup means that legacy and compatibility methods have been removed from -`~.Generator` +`Generator` =================== ============== ============ -`~.RandomState` `~.Generator` Notes +`RandomState` `Generator` Notes ------------------- -------------- ------------ ``random_sample``, ``random`` Compatible with `random.random` ``rand`` @@ -57,22 +67,13 @@ cleanup means that legacy and compatibility methods have been removed from ``randint``, ``integers`` Add an ``endpoint`` kwarg ``random_integers`` ------------------- -------------- ------------ -``tomaxint`` removed Use ``integers(0, np.iinfo(np.int).max,`` - ``endpoint=False)`` +``tomaxint`` removed Use ``integers(0, np.iinfo(np.int_).max,`` + ``endpoint=False)`` ------------------- -------------- ------------ -``seed`` removed Use `~.SeedSequence.spawn` +``seed`` removed Use `SeedSequence.spawn` =================== ============== ============ -See `new-or-different` for more information - -.. code-block:: python - - # As replacement for RandomState(); default_rng() instantiates Generator with - # the default PCG64 BitGenerator. - from numpy.random import default_rng - rg = default_rng() - rg.standard_normal() - rg.bit_generator +See :ref:`new-or-different` for more information Something like the following code can be used to support both ``RandomState`` and ``Generator``, with the understanding that the interfaces are slightly @@ -87,9 +88,9 @@ different a = rg_integers(1000) Seeds can be passed to any of the BitGenerators. The provided value is mixed -via `~.SeedSequence` to spread a possible sequence of seeds across a wider -range of initialization states for the BitGenerator. Here `~.PCG64` is used and -is wrapped with a `~.Generator`. +via `SeedSequence` to spread a possible sequence of seeds across a wider +range of initialization states for the BitGenerator. Here `PCG64` is used and +is wrapped with a `Generator`. .. code-block:: python @@ -100,7 +101,7 @@ is wrapped with a `~.Generator`. Introduction ------------ The new infrastructure takes a different approach to producing random numbers -from the `~.RandomState` object. Random number generation is separated into +from the `RandomState` object. Random number generation is separated into two components, a bit generator and a random generator. The `BitGenerator` has a limited set of responsibilities. It manages state @@ -113,8 +114,8 @@ distributions, e.g., simulated normal random values. This structure allows alternative bit generators to be used with little code duplication. The `Generator` is the user-facing object that is nearly identical to -`.RandomState`. The canonical method to initialize a generator passes a -`~.PCG64` bit generator as the sole argument. +`RandomState`. The canonical method to initialize a generator passes a +`PCG64` bit generator as the sole argument. .. code-block:: python @@ -123,7 +124,7 @@ The `Generator` is the user-facing object that is nearly identical to rg.random() One can also instantiate `Generator` directly with a `BitGenerator` instance. -To use the older `~mt19937.MT19937` algorithm, one can instantiate it directly +To use the older `MT19937` algorithm, one can instantiate it directly and pass it to `Generator`. .. code-block:: python @@ -139,9 +140,9 @@ What's New or Different The Box-Muller method used to produce NumPy's normals is no longer available in `Generator`. It is not possible to reproduce the exact random values using Generator for the normal distribution or any other - distribution that relies on the normal such as the `.RandomState.gamma` or - `.RandomState.standard_t`. If you require bitwise backward compatible - streams, use `.RandomState`. + distribution that relies on the normal such as the `RandomState.gamma` or + `RandomState.standard_t`. If you require bitwise backward compatible + streams, use `RandomState`. * The Generator's normal, exponential and gamma functions use 256-step Ziggurat methods which are 2-10 times faster than NumPy's Box-Muller or inverse CDF @@ -152,16 +153,20 @@ What's New or Different * Optional ``out`` argument that allows existing arrays to be filled for select distributions * All BitGenerators can produce doubles, uint64s and uint32s via CTypes -* `~.Generator.integers` is now the canonical way to generate integer + (`PCG64.ctypes`) and CFFI (`PCG64.cffi`). This allows the bit generators + to be used in numba. +* The bit generators can be used in downstream projects via + :ref:`Cython <random_cython>`. +* `Generator.integers` is now the canonical way to generate integer random numbers from a discrete uniform distribution. The ``rand`` and - ``randn`` methods are only available through the legacy `~.RandomState`. + ``randn`` methods are only available through the legacy `RandomState`. The ``endpoint`` keyword can be used to specify open or closed intervals. This replaces both ``randint`` and the deprecated ``random_integers``. -* `~.Generator.random` is now the canonical way to generate floating-point - random numbers, which replaces `.RandomState.random_sample`, - `.RandomState.sample`, and `.RandomState.ranf`. This is consistent with +* `Generator.random` is now the canonical way to generate floating-point + random numbers, which replaces `RandomState.random_sample`, + `RandomState.sample`, and `RandomState.ranf`. This is consistent with Python's `random.random`. -* All BitGenerators in numpy use `~SeedSequence` to convert seeds into +* All BitGenerators in numpy use `SeedSequence` to convert seeds into initialized states. See :ref:`new-or-different` for a complete list of improvements and @@ -195,9 +200,12 @@ Features Multithreaded Generation <multithreading> new-or-different Comparing Performance <performance> + c-api + Examples of using Numba, Cython, CFFI <extending> -Original Source -~~~~~~~~~~~~~~~ +Original Source of the Generator and BitGenerators +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This package was developed independently of NumPy and was integrated in version 1.17.0. The original repo is at https://github.com/bashtage/randomgen. + diff --git a/doc/source/reference/random/legacy.rst b/doc/source/reference/random/legacy.rst index 413a42727..922d76a9a 100644 --- a/doc/source/reference/random/legacy.rst +++ b/doc/source/reference/random/legacy.rst @@ -121,3 +121,71 @@ Distributions ~RandomState.wald ~RandomState.weibull ~RandomState.zipf + +Functions in `numpy.random` +=========================== +Many of the RandomState methods above are exported as functions in +`numpy.random` This usage is discouraged, as it is implemented via a gloabl +`RandomState` instance which is not advised on two counts: + +- It uses global state, which means results will change as the code changes + +- It uses a `RandomState` rather than the more modern `Generator`. + +For backward compatible legacy reasons, we cannot change this. See +`random-quick-start`. + +.. autosummary:: + :toctree: generated/ + + beta + binomial + bytes + chisquare + choice + dirichlet + exponential + f + gamma + geometric + get_state + gumbel + hypergeometric + laplace + logistic + lognormal + logseries + multinomial + multivariate_normal + negative_binomial + noncentral_chisquare + noncentral_f + normal + pareto + permutation + poisson + power + rand + randint + randn + random + random_integers + random_sample + ranf + rayleigh + sample + seed + set_state + shuffle + standard_cauchy + standard_exponential + standard_gamma + standard_normal + standard_t + triangular + uniform + vonmises + wald + weibull + zipf + diff --git a/doc/source/reference/random/multithreading.rst b/doc/source/reference/random/multithreading.rst index 6883d3672..a0a31d0ea 100644 --- a/doc/source/reference/random/multithreading.rst +++ b/doc/source/reference/random/multithreading.rst @@ -41,7 +41,7 @@ seed will produce the same outputs. self.n = n self.executor = concurrent.futures.ThreadPoolExecutor(threads) self.values = np.empty(n) - self.step = np.ceil(n / threads).astype(np.int) + self.step = np.ceil(n / threads).astype(np.int_) def fill(self): def _fill(random_state, out, first, last): diff --git a/doc/source/reference/random/new-or-different.rst b/doc/source/reference/random/new-or-different.rst index 402f24ab0..b3bddb443 100644 --- a/doc/source/reference/random/new-or-different.rst +++ b/doc/source/reference/random/new-or-different.rst @@ -10,9 +10,10 @@ What's New or Different The Box-Muller method used to produce NumPy's normals is no longer available in `Generator`. It is not possible to reproduce the exact random values using ``Generator`` for the normal distribution or any other - distribution that relies on the normal such as the `gamma` or - `standard_t`. If you require bitwise backward compatible - streams, use `RandomState`. + distribution that relies on the normal such as the `Generator.gamma` or + `Generator.standard_t`. If you require bitwise backward compatible + streams, use `RandomState`, i.e., `RandomState.gamma` or + `RandomState.standard_t`. Quick comparison of legacy `mtrand <legacy>`_ to the new `Generator` @@ -20,9 +21,9 @@ Quick comparison of legacy `mtrand <legacy>`_ to the new `Generator` Feature Older Equivalent Notes ------------------ -------------------- ------------- `~.Generator` `~.RandomState` ``Generator`` requires a stream - source, called a `BitGenerator - <bit_generators>` A number of these - are provided. ``RandomState`` uses + source, called a `BitGenerator` + A number of these are provided. + ``RandomState`` uses the Mersenne Twister `~.MT19937` by default, but can also be instantiated with any BitGenerator. @@ -56,12 +57,13 @@ And in more detail: ``randn`` methods are only available through the legacy `~.RandomState`. This replaces both ``randint`` and the deprecated ``random_integers``. * The Box-Muller method used to produce NumPy's normals is no longer available. -* All bit generators can produce doubles, uint64s and uint32s via CTypes - (`~PCG64.ctypes`) and CFFI (`~PCG64.cffi`). This allows these bit generators - to be used in numba. +* All bit generators can produce doubles, uint64s and + uint32s via CTypes (`~PCG64.ctypes`) and CFFI (`~PCG64.cffi`). + This allows these bit generators to be used in numba. * The bit generators can be used in downstream projects via Cython. + .. ipython:: python from numpy.random import Generator, PCG64 diff --git a/doc/source/reference/random/parallel.rst b/doc/source/reference/random/parallel.rst index 2f79f22d8..721584014 100644 --- a/doc/source/reference/random/parallel.rst +++ b/doc/source/reference/random/parallel.rst @@ -18,10 +18,10 @@ a `~BitGenerator`. It uses hashing techniques to ensure that low-quality seeds are turned into high quality initial states (at least, with very high probability). -For example, `~mt19937.MT19937` has a state consisting of 624 +For example, `MT19937` has a state consisting of 624 `uint32` integers. A naive way to take a 32-bit integer seed would be to just set the last element of the state to the 32-bit seed and leave the rest 0s. This is -a valid state for `~mt19937.MT19937`, but not a good one. The Mersenne Twister +a valid state for `MT19937`, but not a good one. The Mersenne Twister algorithm `suffers if there are too many 0s`_. Similarly, two adjacent 32-bit integer seeds (i.e. ``12345`` and ``12346``) would produce very similar streams. @@ -91,15 +91,15 @@ territory ([2]_). .. [2] In this calculation, we can ignore the amount of numbers drawn from each stream. Each of the PRNGs we provide has some extra protection built in that avoids overlaps if the `~SeedSequence` pools differ in the - slightest bit. `~pcg64.PCG64` has :math:`2^{127}` separate cycles + slightest bit. `PCG64` has :math:`2^{127}` separate cycles determined by the seed in addition to the position in the :math:`2^{128}` long period for each cycle, so one has to both get on or near the same cycle *and* seed a nearby position in the cycle. - `~philox.Philox` has completely independent cycles determined by the seed. - `~sfc64.SFC64` incorporates a 64-bit counter so every unique seed is at + `Philox` has completely independent cycles determined by the seed. + `SFC64` incorporates a 64-bit counter so every unique seed is at least :math:`2^{64}` iterations away from any other seed. And - finally, `~mt19937.MT19937` has just an unimaginably huge period. Getting - a collision internal to `~SeedSequence` is the way a failure would be + finally, `MT19937` has just an unimaginably huge period. Getting + a collision internal to `SeedSequence` is the way a failure would be observed. .. _`implements an algorithm`: http://www.pcg-random.org/posts/developing-a-seed_seq-alternative.html @@ -113,10 +113,10 @@ territory ([2]_). Independent Streams ------------------- -:class:`~philox.Philox` is a counter-based RNG based which generates values by +`Philox` is a counter-based RNG based which generates values by encrypting an incrementing counter using weak cryptographic primitives. The seed determines the key that is used for the encryption. Unique keys create -unique, independent streams. :class:`~philox.Philox` lets you bypass the +unique, independent streams. `Philox` lets you bypass the seeding algorithm to directly set the 128-bit key. Similar, but different, keys will still create independent streams. diff --git a/doc/source/reference/random/performance.rst b/doc/source/reference/random/performance.rst index 2d5fca496..d70dd064a 100644 --- a/doc/source/reference/random/performance.rst +++ b/doc/source/reference/random/performance.rst @@ -5,21 +5,21 @@ Performance Recommendation ************** -The recommended generator for general use is :class:`~pcg64.PCG64`. It is +The recommended generator for general use is `PCG64`. It is statistically high quality, full-featured, and fast on most platforms, but somewhat slow when compiled for 32-bit processes. -:class:`~philox.Philox` is fairly slow, but its statistical properties have +`Philox` is fairly slow, but its statistical properties have very high quality, and it is easy to get assuredly-independent stream by using unique keys. If that is the style you wish to use for parallel streams, or you are porting from another system that uses that style, then -:class:`~philox.Philox` is your choice. +`Philox` is your choice. -:class:`~sfc64.SFC64` is statistically high quality and very fast. However, it +`SFC64` is statistically high quality and very fast. However, it lacks jumpability. If you are not using that capability and want lots of speed, even on 32-bit processes, this is your choice. -:class:`~mt19937.MT19937` `fails some statistical tests`_ and is not especially +`MT19937` `fails some statistical tests`_ and is not especially fast compared to modern PRNGs. For these reasons, we mostly do not recommend using it on its own, only through the legacy `~.RandomState` for reproducing old results. That said, it has a very long history as a default in @@ -31,20 +31,20 @@ Timings ******* The timings below are the time in ns to produce 1 random value from a -specific distribution. The original :class:`~mt19937.MT19937` generator is +specific distribution. The original `MT19937` generator is much slower since it requires 2 32-bit values to equal the output of the faster generators. Integer performance has a similar ordering. The pattern is similar for other, more complex generators. The normal -performance of the legacy :class:`~.RandomState` generator is much +performance of the legacy `RandomState` generator is much lower than the other since it uses the Box-Muller transformation rather than the Ziggurat generator. The performance gap for Exponentials is also large due to the cost of computing the log function to invert the CDF. The column labeled MT19973 is used the same 32-bit generator as -:class:`~.RandomState` but produces random values using -:class:`~Generator`. +`RandomState` but produces random values using +`Generator`. .. csv-table:: :header: ,MT19937,PCG64,Philox,SFC64,RandomState @@ -61,7 +61,7 @@ The column labeled MT19973 is used the same 32-bit generator as Poissons,67.6,52.4,69.2,46.4,78.1 The next table presents the performance in percentage relative to values -generated by the legacy generator, `RandomState(MT19937())`. The overall +generated by the legacy generator, ``RandomState(MT19937())``. The overall performance was computed using a geometric mean. .. csv-table:: diff --git a/doc/source/reference/routines.array-manipulation.rst b/doc/source/reference/routines.array-manipulation.rst index cc93d1029..bf43232ef 100644 --- a/doc/source/reference/routines.array-manipulation.rst +++ b/doc/source/reference/routines.array-manipulation.rst @@ -9,6 +9,7 @@ Basic operations :toctree: generated/ copyto + shape Changing array shape ==================== diff --git a/doc/source/reference/routines.char.rst b/doc/source/reference/routines.char.rst index 513f975e7..ed8393855 100644 --- a/doc/source/reference/routines.char.rst +++ b/doc/source/reference/routines.char.rst @@ -58,6 +58,7 @@ comparison. less_equal greater less + compare_chararrays String information ------------------ diff --git a/doc/source/reference/routines.ma.rst b/doc/source/reference/routines.ma.rst index 491bb6bff..5b2098c7a 100644 --- a/doc/source/reference/routines.ma.rst +++ b/doc/source/reference/routines.ma.rst @@ -264,17 +264,6 @@ Conversion operations ma.MaskedArray.tobytes -Pickling and unpickling -~~~~~~~~~~~~~~~~~~~~~~~ -.. autosummary:: - :toctree: generated/ - - ma.dump - ma.dumps - ma.load - ma.loads - - Filling a masked array ~~~~~~~~~~~~~~~~~~~~~~ .. autosummary:: diff --git a/doc/source/reference/routines.other.rst b/doc/source/reference/routines.other.rst index 0a3677904..28c9a1ad1 100644 --- a/doc/source/reference/routines.other.rst +++ b/doc/source/reference/routines.other.rst @@ -21,6 +21,7 @@ Memory ranges shares_memory may_share_memory + byte_bounds Array mixins ------------ @@ -35,3 +36,21 @@ NumPy version comparison :toctree: generated/ lib.NumpyVersion + +Utility +------- + +.. autosummary:: + :toctree: generated/ + + get_include + deprecate + deprecate_with_doc + +Matlab-like Functions +--------------------- +.. autosummary:: + :toctree: generated/ + + who + disp
\ No newline at end of file diff --git a/doc/source/reference/routines.testing.rst b/doc/source/reference/routines.testing.rst index c676dec07..98ce3f377 100644 --- a/doc/source/reference/routines.testing.rst +++ b/doc/source/reference/routines.testing.rst @@ -37,11 +37,11 @@ Decorators .. autosummary:: :toctree: generated/ - decorators.deprecated - decorators.knownfailureif - decorators.setastest - decorators.skipif - decorators.slow + dec.deprecated + dec.knownfailureif + dec.setastest + dec.skipif + dec.slow decorate_methods Test Running diff --git a/doc/source/reference/ufuncs.rst b/doc/source/reference/ufuncs.rst index d00e88b34..361cf11b9 100644 --- a/doc/source/reference/ufuncs.rst +++ b/doc/source/reference/ufuncs.rst @@ -100,7 +100,7 @@ is true: - *d* acts like a (5,6) array where the single value is repeated. -.. _ufuncs.output-type: +.. _ufuncs-output-type: Output type determination ========================= @@ -228,46 +228,47 @@ can generate this table for your system with the code given in the Figure. .. admonition:: Figure - Code segment showing the "can cast safely" table for a 32-bit system. + Code segment showing the "can cast safely" table for a 64-bit system. + Generally the output depends on the system; your system might result in + a different table. + >>> mark = {False: ' -', True: ' Y'} >>> def print_table(ntypes): - ... print 'X', - ... for char in ntypes: print char, - ... print + ... print('X ' + ' '.join(ntypes)) ... for row in ntypes: - ... print row, + ... print(row, end='') ... for col in ntypes: - ... print int(np.can_cast(row, col)), - ... print + ... print(mark[np.can_cast(row, col)], end='') + ... print() + ... >>> print_table(np.typecodes['All']) X ? b h i l q p B H I L Q P e f d g F D G S U V O M m - ? 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 - b 0 1 1 1 1 1 1 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 - h 0 0 1 1 1 1 1 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 0 0 - i 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 1 1 0 0 - l 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 1 1 0 0 - q 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 1 1 0 0 - p 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 1 1 0 0 - B 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 - H 0 0 0 1 1 1 1 0 1 1 1 1 1 0 1 1 1 1 1 1 1 1 1 1 0 0 - I 0 0 0 0 1 1 1 0 0 1 1 1 1 0 0 1 1 0 1 1 1 1 1 1 0 0 - L 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 1 1 0 1 1 1 1 1 1 0 0 - Q 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 1 1 0 1 1 1 1 1 1 0 0 - P 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 1 1 0 1 1 1 1 1 1 0 0 - e 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 - f 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 0 0 - d 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 1 1 0 0 - g 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 1 1 1 1 0 0 - F 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 0 0 - D 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 - G 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 0 0 - S 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 - U 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 - V 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 - O 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 - M 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 - m 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 - + ? Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y + b - Y Y Y Y Y Y - - - - - - Y Y Y Y Y Y Y Y Y Y Y - Y + h - - Y Y Y Y Y - - - - - - - Y Y Y Y Y Y Y Y Y Y - Y + i - - - Y Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y + l - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y + q - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y + p - - - - Y Y Y - - - - - - - - Y Y - Y Y Y Y Y Y - Y + B - - Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y - Y + H - - - Y Y Y Y - Y Y Y Y Y - Y Y Y Y Y Y Y Y Y Y - Y + I - - - - Y Y Y - - Y Y Y Y - - Y Y - Y Y Y Y Y Y - Y + L - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - - + Q - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - - + P - - - - - - - - - - Y Y Y - - Y Y - Y Y Y Y Y Y - - + e - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y Y - - + f - - - - - - - - - - - - - - Y Y Y Y Y Y Y Y Y Y - - + d - - - - - - - - - - - - - - - Y Y - Y Y Y Y Y Y - - + g - - - - - - - - - - - - - - - - Y - - Y Y Y Y Y - - + F - - - - - - - - - - - - - - - - - Y Y Y Y Y Y Y - - + D - - - - - - - - - - - - - - - - - - Y Y Y Y Y Y - - + G - - - - - - - - - - - - - - - - - - - Y Y Y Y Y - - + S - - - - - - - - - - - - - - - - - - - - Y Y Y Y - - + U - - - - - - - - - - - - - - - - - - - - - Y Y Y - - + V - - - - - - - - - - - - - - - - - - - - - - Y Y - - + O - - - - - - - - - - - - - - - - - - - - - - Y Y - - + M - - - - - - - - - - - - - - - - - - - - - - Y Y Y - + m - - - - - - - - - - - - - - - - - - - - - - Y Y - Y You should note that, while included in the table for completeness, the 'S', 'U', and 'V' types cannot be operated on by ufuncs. Also, @@ -319,7 +320,7 @@ advanced usage and will not typically be used. .. versionadded:: 1.10 The 'out' keyword argument is expected to be a tuple with one entry per - output (which can be `None` for arrays to be allocated by the ufunc). + output (which can be None for arrays to be allocated by the ufunc). For ufuncs with a single output, passing a single array (instead of a tuple holding a single array) is also valid. @@ -493,7 +494,7 @@ keyword, and an *out* keyword, and the arrays must all have dimension >= 1. The *axis* keyword specifies the axis of the array over which the reduction will take place (with negative values counting backwards). Generally, it is an integer, though for :meth:`ufunc.reduce`, it can also be a tuple of `int` to -reduce over several axes at once, or `None`, to reduce over all axes. +reduce over several axes at once, or None, to reduce over all axes. The *dtype* keyword allows you to manage a very common problem that arises when naively using :meth:`ufunc.reduce`. Sometimes you may have an array of a certain data type and wish to add up all of its diff --git a/doc/source/release.rst b/doc/source/release.rst index febc96dc5..0343275a5 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -2,58 +2,60 @@ Release Notes ************* -.. include:: ../release/1.17.5-notes.rst -.. include:: ../release/1.17.4-notes.rst -.. include:: ../release/1.17.3-notes.rst -.. include:: ../release/1.17.2-notes.rst -.. include:: ../release/1.17.1-notes.rst -.. include:: ../release/1.17.0-notes.rst -.. include:: ../release/1.16.6-notes.rst -.. include:: ../release/1.16.5-notes.rst -.. include:: ../release/1.16.4-notes.rst -.. include:: ../release/1.16.3-notes.rst -.. include:: ../release/1.16.2-notes.rst -.. include:: ../release/1.16.1-notes.rst -.. include:: ../release/1.16.0-notes.rst -.. include:: ../release/1.15.4-notes.rst -.. include:: ../release/1.15.3-notes.rst -.. include:: ../release/1.15.2-notes.rst -.. include:: ../release/1.15.1-notes.rst -.. include:: ../release/1.15.0-notes.rst -.. include:: ../release/1.14.6-notes.rst -.. include:: ../release/1.14.5-notes.rst -.. include:: ../release/1.14.4-notes.rst -.. include:: ../release/1.14.3-notes.rst -.. include:: ../release/1.14.2-notes.rst -.. include:: ../release/1.14.1-notes.rst -.. include:: ../release/1.14.0-notes.rst -.. include:: ../release/1.13.3-notes.rst -.. include:: ../release/1.13.2-notes.rst -.. include:: ../release/1.13.1-notes.rst -.. include:: ../release/1.13.0-notes.rst -.. include:: ../release/1.12.1-notes.rst -.. include:: ../release/1.12.0-notes.rst -.. include:: ../release/1.11.3-notes.rst -.. include:: ../release/1.11.2-notes.rst -.. include:: ../release/1.11.1-notes.rst -.. include:: ../release/1.11.0-notes.rst -.. include:: ../release/1.10.4-notes.rst -.. include:: ../release/1.10.3-notes.rst -.. include:: ../release/1.10.2-notes.rst -.. include:: ../release/1.10.1-notes.rst -.. include:: ../release/1.10.0-notes.rst -.. include:: ../release/1.9.2-notes.rst -.. include:: ../release/1.9.1-notes.rst -.. include:: ../release/1.9.0-notes.rst -.. include:: ../release/1.8.2-notes.rst -.. include:: ../release/1.8.1-notes.rst -.. include:: ../release/1.8.0-notes.rst -.. include:: ../release/1.7.2-notes.rst -.. include:: ../release/1.7.1-notes.rst -.. include:: ../release/1.7.0-notes.rst -.. include:: ../release/1.6.2-notes.rst -.. include:: ../release/1.6.1-notes.rst -.. include:: ../release/1.6.0-notes.rst -.. include:: ../release/1.5.0-notes.rst -.. include:: ../release/1.4.0-notes.rst -.. include:: ../release/1.3.0-notes.rst +.. toctree:: + :maxdepth: 3 + + 1.18.0 <release/1.18.0-notes> + 1.17.4 <release/1.17.4-notes> + 1.17.3 <release/1.17.3-notes> + 1.17.2 <release/1.17.2-notes> + 1.17.1 <release/1.17.1-notes> + 1.17.0 <release/1.17.0-notes> + 1.16.5 <release/1.16.5-notes> + 1.16.4 <release/1.16.4-notes> + 1.16.3 <release/1.16.3-notes> + 1.16.2 <release/1.16.2-notes> + 1.16.1 <release/1.16.1-notes> + 1.16.0 <release/1.16.0-notes> + 1.15.4 <release/1.15.4-notes> + 1.15.3 <release/1.15.3-notes> + 1.15.2 <release/1.15.2-notes> + 1.15.1 <release/1.15.1-notes> + 1.15.0 <release/1.15.0-notes> + 1.14.6 <release/1.14.6-notes> + 1.14.5 <release/1.14.5-notes> + 1.14.4 <release/1.14.4-notes> + 1.14.3 <release/1.14.3-notes> + 1.14.2 <release/1.14.2-notes> + 1.14.1 <release/1.14.1-notes> + 1.14.0 <release/1.14.0-notes> + 1.13.3 <release/1.13.3-notes> + 1.13.2 <release/1.13.2-notes> + 1.13.1 <release/1.13.1-notes> + 1.13.0 <release/1.13.0-notes> + 1.12.1 <release/1.12.1-notes> + 1.12.0 <release/1.12.0-notes> + 1.11.3 <release/1.11.3-notes> + 1.11.2 <release/1.11.2-notes> + 1.11.1 <release/1.11.1-notes> + 1.11.0 <release/1.11.0-notes> + 1.10.4 <release/1.10.4-notes> + 1.10.3 <release/1.10.3-notes> + 1.10.2 <release/1.10.2-notes> + 1.10.1 <release/1.10.1-notes> + 1.10.0 <release/1.10.0-notes> + 1.9.2 <release/1.9.2-notes> + 1.9.1 <release/1.9.1-notes> + 1.9.0 <release/1.9.0-notes> + 1.8.2 <release/1.8.2-notes> + 1.8.1 <release/1.8.1-notes> + 1.8.0 <release/1.8.0-notes> + 1.7.2 <release/1.7.2-notes> + 1.7.1 <release/1.7.1-notes> + 1.7.0 <release/1.7.0-notes> + 1.6.2 <release/1.6.2-notes> + 1.6.1 <release/1.6.1-notes> + 1.6.0 <release/1.6.0-notes> + 1.5.0 <release/1.5.0-notes> + 1.4.0 <release/1.4.0-notes> + 1.3.0 <release/1.3.0-notes> diff --git a/doc/release/1.10.0-notes.rst b/doc/source/release/1.10.0-notes.rst index 88062e463..88062e463 100644 --- a/doc/release/1.10.0-notes.rst +++ b/doc/source/release/1.10.0-notes.rst diff --git a/doc/release/1.10.1-notes.rst b/doc/source/release/1.10.1-notes.rst index 4e541d279..4e541d279 100644 --- a/doc/release/1.10.1-notes.rst +++ b/doc/source/release/1.10.1-notes.rst diff --git a/doc/release/1.10.2-notes.rst b/doc/source/release/1.10.2-notes.rst index 8c26b463c..8c26b463c 100644 --- a/doc/release/1.10.2-notes.rst +++ b/doc/source/release/1.10.2-notes.rst diff --git a/doc/release/1.10.3-notes.rst b/doc/source/release/1.10.3-notes.rst index 0d4df4ce6..0d4df4ce6 100644 --- a/doc/release/1.10.3-notes.rst +++ b/doc/source/release/1.10.3-notes.rst diff --git a/doc/release/1.10.4-notes.rst b/doc/source/release/1.10.4-notes.rst index 481928ca7..481928ca7 100644 --- a/doc/release/1.10.4-notes.rst +++ b/doc/source/release/1.10.4-notes.rst diff --git a/doc/release/1.11.0-notes.rst b/doc/source/release/1.11.0-notes.rst index 166502ac5..36cd1d65a 100644 --- a/doc/release/1.11.0-notes.rst +++ b/doc/source/release/1.11.0-notes.rst @@ -85,7 +85,7 @@ times in UTC. By default, creating a datetime64 object from a string or printing it would convert from or to local time:: # old behavior - >>>> np.datetime64('2000-01-01T00:00:00') + >>> np.datetime64('2000-01-01T00:00:00') numpy.datetime64('2000-01-01T00:00:00-0800') # note the timezone offset -08:00 @@ -96,7 +96,7 @@ type is preferred, similar to the ``datetime.datetime`` type in the Python standard library. Accordingly, datetime64 no longer assumes that input is in local time, nor does it print local times:: - >>>> np.datetime64('2000-01-01T00:00:00') + >>> np.datetime64('2000-01-01T00:00:00') numpy.datetime64('2000-01-01T00:00:00') For backwards compatibility, datetime64 still parses timezone offsets, which @@ -200,7 +200,7 @@ New Features * A ``dtype`` parameter has been added to ``np.random.randint`` Random ndarrays of the following types can now be generated: - - ``np.bool``, + - ``np.bool_``, - ``np.int8``, ``np.uint8``, - ``np.int16``, ``np.uint16``, - ``np.int32``, ``np.uint32``, diff --git a/doc/release/1.11.1-notes.rst b/doc/source/release/1.11.1-notes.rst index 6303c32f0..6303c32f0 100644 --- a/doc/release/1.11.1-notes.rst +++ b/doc/source/release/1.11.1-notes.rst diff --git a/doc/release/1.11.2-notes.rst b/doc/source/release/1.11.2-notes.rst index c954089d5..c954089d5 100644 --- a/doc/release/1.11.2-notes.rst +++ b/doc/source/release/1.11.2-notes.rst diff --git a/doc/release/1.11.3-notes.rst b/doc/source/release/1.11.3-notes.rst index 8381a97f7..8381a97f7 100644 --- a/doc/release/1.11.3-notes.rst +++ b/doc/source/release/1.11.3-notes.rst diff --git a/doc/release/1.12.0-notes.rst b/doc/source/release/1.12.0-notes.rst index e735d2d77..711055d16 100644 --- a/doc/release/1.12.0-notes.rst +++ b/doc/source/release/1.12.0-notes.rst @@ -1,5 +1,3 @@ -.. 1.12.0: - ========================== NumPy 1.12.0 Release Notes ========================== diff --git a/doc/release/1.12.1-notes.rst b/doc/source/release/1.12.1-notes.rst index f67dab108..f67dab108 100644 --- a/doc/release/1.12.1-notes.rst +++ b/doc/source/release/1.12.1-notes.rst diff --git a/doc/release/1.13.0-notes.rst b/doc/source/release/1.13.0-notes.rst index 3b719db09..5d8c932fe 100644 --- a/doc/release/1.13.0-notes.rst +++ b/doc/source/release/1.13.0-notes.rst @@ -275,7 +275,7 @@ In particular ``np.gradient`` can now take: This means that, e.g., it is now possible to do the following:: - >>> f = np.array([[1, 2, 6], [3, 4, 5]], dtype=np.float) + >>> f = np.array([[1, 2, 6], [3, 4, 5]], dtype=np.float_) >>> dx = 2. >>> y = [1., 1.5, 3.5] >>> np.gradient(f, dx, y) diff --git a/doc/release/1.13.1-notes.rst b/doc/source/release/1.13.1-notes.rst index 88a4bc3dd..88a4bc3dd 100644 --- a/doc/release/1.13.1-notes.rst +++ b/doc/source/release/1.13.1-notes.rst diff --git a/doc/release/1.13.2-notes.rst b/doc/source/release/1.13.2-notes.rst index f2f9120f5..f2f9120f5 100644 --- a/doc/release/1.13.2-notes.rst +++ b/doc/source/release/1.13.2-notes.rst diff --git a/doc/release/1.13.3-notes.rst b/doc/source/release/1.13.3-notes.rst index 7f7170bcc..7f7170bcc 100644 --- a/doc/release/1.13.3-notes.rst +++ b/doc/source/release/1.13.3-notes.rst diff --git a/doc/release/1.14.0-notes.rst b/doc/source/release/1.14.0-notes.rst index 462631de6..462631de6 100644 --- a/doc/release/1.14.0-notes.rst +++ b/doc/source/release/1.14.0-notes.rst diff --git a/doc/release/1.14.1-notes.rst b/doc/source/release/1.14.1-notes.rst index 7b95c2e28..7b95c2e28 100644 --- a/doc/release/1.14.1-notes.rst +++ b/doc/source/release/1.14.1-notes.rst diff --git a/doc/release/1.14.2-notes.rst b/doc/source/release/1.14.2-notes.rst index 3f47cb5f5..3f47cb5f5 100644 --- a/doc/release/1.14.2-notes.rst +++ b/doc/source/release/1.14.2-notes.rst diff --git a/doc/release/1.14.3-notes.rst b/doc/source/release/1.14.3-notes.rst index 60b631168..60b631168 100644 --- a/doc/release/1.14.3-notes.rst +++ b/doc/source/release/1.14.3-notes.rst diff --git a/doc/release/1.14.4-notes.rst b/doc/source/release/1.14.4-notes.rst index 3fb94383b..3fb94383b 100644 --- a/doc/release/1.14.4-notes.rst +++ b/doc/source/release/1.14.4-notes.rst diff --git a/doc/release/1.14.5-notes.rst b/doc/source/release/1.14.5-notes.rst index 9a97cc033..9a97cc033 100644 --- a/doc/release/1.14.5-notes.rst +++ b/doc/source/release/1.14.5-notes.rst diff --git a/doc/release/1.14.6-notes.rst b/doc/source/release/1.14.6-notes.rst index ac6a78272..ac6a78272 100644 --- a/doc/release/1.14.6-notes.rst +++ b/doc/source/release/1.14.6-notes.rst diff --git a/doc/release/1.15.0-notes.rst b/doc/source/release/1.15.0-notes.rst index 7235ca915..7235ca915 100644 --- a/doc/release/1.15.0-notes.rst +++ b/doc/source/release/1.15.0-notes.rst diff --git a/doc/release/1.15.1-notes.rst b/doc/source/release/1.15.1-notes.rst index ddb83303c..ddb83303c 100644 --- a/doc/release/1.15.1-notes.rst +++ b/doc/source/release/1.15.1-notes.rst diff --git a/doc/release/1.15.2-notes.rst b/doc/source/release/1.15.2-notes.rst index a3e61fccd..a3e61fccd 100644 --- a/doc/release/1.15.2-notes.rst +++ b/doc/source/release/1.15.2-notes.rst diff --git a/doc/release/1.15.3-notes.rst b/doc/source/release/1.15.3-notes.rst index 753eecec9..753eecec9 100644 --- a/doc/release/1.15.3-notes.rst +++ b/doc/source/release/1.15.3-notes.rst diff --git a/doc/release/1.15.4-notes.rst b/doc/source/release/1.15.4-notes.rst index 033bd5828..033bd5828 100644 --- a/doc/release/1.15.4-notes.rst +++ b/doc/source/release/1.15.4-notes.rst diff --git a/doc/release/1.16.0-notes.rst b/doc/source/release/1.16.0-notes.rst index 1034d6e6c..e78e270f4 100644 --- a/doc/release/1.16.0-notes.rst +++ b/doc/source/release/1.16.0-notes.rst @@ -506,7 +506,7 @@ become more conspicuous. Now no warnings will be emitted. Umath and multiarray c-extension modules merged into a single module -------------------------------------------------------------------- The two modules were merged, according to `NEP 15`_. Previously `np.core.umath` -and `np.core.multiarray` were seperate c-extension modules. They are now python +and `np.core.multiarray` were separate c-extension modules. They are now python wrappers to the single `np.core/_multiarray_math` c-extension module. .. _`NEP 15` : http://www.numpy.org/neps/nep-0015-merge-multiarray-umath.html diff --git a/doc/release/1.16.1-notes.rst b/doc/source/release/1.16.1-notes.rst index 2a190ef91..d6fc25b44 100644 --- a/doc/release/1.16.1-notes.rst +++ b/doc/source/release/1.16.1-notes.rst @@ -58,7 +58,7 @@ Enhancements Compatibility notes =================== -* The changed error message emited by array comparison testing functions may +* The changed error message emitted by array comparison testing functions may affect doctests. See below for detail. * Casting from double and single denormals to float16 has been corrected. In diff --git a/doc/release/1.16.2-notes.rst b/doc/source/release/1.16.2-notes.rst index 62b90dc40..62b90dc40 100644 --- a/doc/release/1.16.2-notes.rst +++ b/doc/source/release/1.16.2-notes.rst diff --git a/doc/release/1.16.3-notes.rst b/doc/source/release/1.16.3-notes.rst index 181a7264d..181a7264d 100644 --- a/doc/release/1.16.3-notes.rst +++ b/doc/source/release/1.16.3-notes.rst diff --git a/doc/release/1.16.4-notes.rst b/doc/source/release/1.16.4-notes.rst index a236b05c8..a236b05c8 100644 --- a/doc/release/1.16.4-notes.rst +++ b/doc/source/release/1.16.4-notes.rst diff --git a/doc/release/1.16.5-notes.rst b/doc/source/release/1.16.5-notes.rst index 3fbe2d7e8..5b6eb585b 100644 --- a/doc/release/1.16.5-notes.rst +++ b/doc/source/release/1.16.5-notes.rst @@ -1,5 +1,3 @@ -.. currentmodule:: numpy - ========================== NumPy 1.16.5 Release Notes ========================== diff --git a/doc/release/1.17.0-notes.rst b/doc/source/release/1.17.0-notes.rst index 8d69e36d9..a0e737982 100644 --- a/doc/release/1.17.0-notes.rst +++ b/doc/source/release/1.17.0-notes.rst @@ -239,7 +239,7 @@ New extensible `numpy.random` module with selectable random number generators ----------------------------------------------------------------------------- A new extensible `numpy.random` module along with four selectable random number generators and improved seeding designed for use in parallel processes has been -added. The currently available :ref:`Bit Generators <bit_generator>` are +added. The currently available `Bit Generators` are `~mt19937.MT19937`, `~pcg64.PCG64`, `~philox.Philox`, and `~sfc64.SFC64`. ``PCG64`` is the new default while ``MT19937`` is retained for backwards compatibility. Note that the legacy random module is unchanged and is now diff --git a/doc/release/1.17.1-notes.rst b/doc/source/release/1.17.1-notes.rst index bd837ee5b..bd837ee5b 100644 --- a/doc/release/1.17.1-notes.rst +++ b/doc/source/release/1.17.1-notes.rst diff --git a/doc/release/1.17.2-notes.rst b/doc/source/release/1.17.2-notes.rst index 65cdaf903..65cdaf903 100644 --- a/doc/release/1.17.2-notes.rst +++ b/doc/source/release/1.17.2-notes.rst diff --git a/doc/release/1.17.3-notes.rst b/doc/source/release/1.17.3-notes.rst index 9f20c2fdc..e33ca1917 100644 --- a/doc/release/1.17.3-notes.rst +++ b/doc/source/release/1.17.3-notes.rst @@ -5,7 +5,7 @@ NumPy 1.17.3 Release Notes ========================== This release contains fixes for bugs reported against NumPy 1.17.2 along with a -some documentation improvements. The Python versions supported in this release +some documentation improvements. The Python versions supported in this release are 3.5-3.8. Downstream developers should use Cython >= 0.29.13 for Python 3.8 support and @@ -23,10 +23,6 @@ Compatibility notes =================== - The seldom used ``PyArray_DescrCheck`` macro has been changed/fixed. -- The use of the new ``numpy.random`` features from Cython and Numba - was not well documented and parts have been removed or refactored. - We plan to finish the refactor and fully document it in 1.18.0 - Contributors diff --git a/doc/release/1.17.4-notes.rst b/doc/source/release/1.17.4-notes.rst index 47f4725f9..47f4725f9 100644 --- a/doc/release/1.17.4-notes.rst +++ b/doc/source/release/1.17.4-notes.rst diff --git a/doc/source/release/1.18.0-notes.rst b/doc/source/release/1.18.0-notes.rst new file mode 100644 index 000000000..ad02d74d3 --- /dev/null +++ b/doc/source/release/1.18.0-notes.rst @@ -0,0 +1,381 @@ +.. currentmodule:: numpy + +================================ +NumPy NumPy 1.18.0 Release Notes +================================ + +In addition to the usual bug fixes, this NumPy release cleans up and documents +the new random C-API, expires a large number of old deprecations, and improves +the appearance of the documentation. The Python versions supported are 3.5-3.8. +This is the last NumPy release series that will support Python 3.5. + +Downstream developers should use Cython >= 0.29.13 for Python 3.8 support and +OpenBLAS >= 3.7 to avoid problems on the Skylake +architecture. + + +Highlights +========== + +* The C-API for ``numpy.random`` has been defined and documented. +* Basic infrastructure for linking with 64 bit BLAS and LAPACK libraries. +* Many documentation improvements. + + +New functions +============= + +Multivariate hypergeometric distribution added to ``numpy.random`` +------------------------------------------------------------------ +The method ``multivariate_hypergeometric`` has been added to the class +`numpy.random.Generator`. This method generates random variates from +the multivariate hypergeometric probability distribution. +(`gh-13794 <https://github.com/numpy/numpy/pull/13794>`__) + + +Deprecations +============ + +``np.fromfile`` and ``np.fromstring`` will error on bad data +------------------------------------------------------------ + +In future numpy releases, the functions ``np.fromfile`` and ``np.fromstring`` +will throw an error when parsing bad data. +This will now give a ``DeprecationWarning`` where previously partial or +even invalid data was silently returned. This deprecation also affects +the C defined functions ``PyArray_FromString`` and ``PyArray_FromFile`` +(`gh-13605 <https://github.com/numpy/numpy/pull/13605>`__) + +Deprecate non-scalar arrays as fill values in ``ma.fill_value`` +--------------------------------------------------------------- +Setting a ``MaskedArray.fill_value`` to a non-scalar array is deprecated +since the logic to broadcast the fill value to the array is fragile, +especially when slicing. +(`gh-13698 <https://github.com/numpy/numpy/pull/13698>`__) + +Deprecate ``PyArray_As1D``, ``PyArray_As2D`` +-------------------------------------------- +``PyArray_As1D``, ``PyArray_As2D`` are deprecated, use +``PyArray_AsCArray`` instead +(`gh-14036 <https://github.com/numpy/numpy/pull/14036>`__) + +Deprecate ``np.alen`` +--------------------- +``np.alen`` was deprecated. Use ``len`` instead. +(`gh-14181 <https://github.com/numpy/numpy/pull/14181>`__) + +Deprecate the financial functions +--------------------------------- +In accordance with +`NEP-32 <https://numpy.org/neps/nep-0032-remove-financial-functions.html>`_, +the financial functions ``fv`` ``ipmt``, ``irr``, ``mirr``, ``nper``, +``npv``, ``pmt``, ``ppmt``, ``pv`` and ``rate`` are deprecated, and will be +removed from NumPy 1.20.The replacement for these functions is the Python package +`numpy-financial <https://pypi.org/project/numpy-financial>`_. +(`gh-14720 <https://github.com/numpy/numpy/pull/14720>`__) + +The ``axis`` argument to ``numpy.ma.mask_cols`` and ``numpy.ma.mask_row`` is deprecated +--------------------------------------------------------------------------------------- +This argument was always ignored. +(`gh-14996 <https://github.com/numpy/numpy/pull/14996>`__) + + +Expired deprecations +==================== + +* ``PyArray_As1D`` and ``PyArray_As2D`` have been removed in favor of + ``PyArray_AsCArray`` + (`gh-14036 <https://github.com/numpy/numpy/pull/14036>`__) + +* ``np.rank`` has been removed. This was deprecated in NumPy 1.10 + and has been replaced by ``np.ndim``. + (`gh-14039 <https://github.com/numpy/numpy/pull/14039>`__) + +* The deprecation of ``expand_dims`` out-of-range axes in 1.13.0 has + expired. + (`gh-14051 <https://github.com/numpy/numpy/pull/14051>`__) + +* ``PyArray_FromDimsAndDataAndDescr`` and ``PyArray_FromDims`` have been + removed (they will always raise an error). Use ``PyArray_NewFromDescr`` + and ``PyArray_SimpleNew`` instead. + (`gh-14100 <https://github.com/numpy/numpy/pull/14100>`__) + +* ``numeric.loads``, ``numeric.load``, ``np.ma.dump``, + ``np.ma.dumps``, ``np.ma.load``, ``np.ma.loads`` are removed, + use ``pickle`` methods instead + (`gh-14256 <https://github.com/numpy/numpy/pull/14256>`__) + +* ``arrayprint.FloatFormat``, ``arrayprint.LongFloatFormat`` has been removed, + use ``FloatingFormat`` instead + +* ``arrayprint.ComplexFormat``, ``arrayprint.LongComplexFormat`` has been + removed, use ``ComplexFloatingFormat`` instead + +* ``arrayprint.StructureFormat`` has been removed, use ``StructureVoidFormat`` + instead + (`gh-14259 <https://github.com/numpy/numpy/pull/14259>`__) + +* ``np.testing.rand`` has been removed. This was deprecated in NumPy 1.11 + and has been replaced by ``np.random.rand``. + (`gh-14325 <https://github.com/numpy/numpy/pull/14325>`__) + +* Class ``SafeEval`` in ``numpy/lib/utils.py`` has been removed. + This was deprecated in NumPy 1.10. Use ``np.safe_eval`` instead. + (`gh-14335 <https://github.com/numpy/numpy/pull/14335>`__) + +* Remove deprecated support for boolean and empty condition lists in + ``np.select`` + (`gh-14583 <https://github.com/numpy/numpy/pull/14583>`__) + +* Array order only accepts 'C', 'F', 'A', and 'K'. More permissive options + were deprecated in NumPy 1.11. + (`gh-14596 <https://github.com/numpy/numpy/pull/14596>`__) + +* np.linspace parameter ``num`` must be an integer. Deprecated in NumPy 1.12. + (`gh-14620 <https://github.com/numpy/numpy/pull/14620>`__) + +* UFuncs with multiple outputs must use a tuple for the ``out`` kwarg. This + finishes a deprecation started in NumPy 1.10. + (`gh-14682 <https://github.com/numpy/numpy/pull/14682>`__) + +The files ``numpy/testing/decorators.py``, ``numpy/testing/noseclasses.py`` +and ``numpy/testing/nosetester.py`` have been removed. They were never +meant to be public (all relevant objects are present in the +``numpy.testing`` namespace), and importing them has given a deprecation +warning since NumPy 1.15.0 +(`gh-14567 <https://github.com/numpy/numpy/pull/14567>`__) + + +Compatibility notes +=================== + +`numpy.lib.recfunctions.drop_fields` can no longer return None +-------------------------------------------------------------- +If ``drop_fields`` is used to drop all fields, previously the array would +be completely discarded and None returned. Now it returns an array of the +same shape as the input, but with no fields. The old behavior can be retained +with:: + + dropped_arr = drop_fields(arr, ['a', 'b']) + if dropped_arr.dtype.names == (): + dropped_arr = None + +converting the empty recarray to None +(`gh-14510 <https://github.com/numpy/numpy/pull/14510>`__) + +``numpy.argmin/argmax/min/max`` returns ``NaT`` if it exists in array +--------------------------------------------------------------------- +``numpy.argmin``, ``numpy.argmax``, ``numpy.min``, and ``numpy.max`` will return +``NaT`` if it exists in the array. +(`gh-14717 <https://github.com/numpy/numpy/pull/14717>`__) + +``np.can_cast(np.uint64, np.timedelta64, casting='safe')`` is now ``False`` +--------------------------------------------------------------------------- +Previously this was ``True`` - however, this was inconsistent with ``uint64`` +not being safely castable to ``int64``, and resulting in strange type +resolution. + +If this impacts your code, cast ``uint64`` to ``int64`` first. +(`gh-14718 <https://github.com/numpy/numpy/pull/14718>`__) + +Changed random variate stream from ``numpy.random.Generator.integers`` +---------------------------------------------------------------------- +There was a bug in ``numpy.random.Generator.integers`` that caused biased +sampling of 8 and 16 bit integer types. Fixing that bug has changed the +output stream from what it was in previous releases. +(`gh-14777 <https://github.com/numpy/numpy/pull/14777>`__) + +Add more ufunc loops for ``datetime64``, ``timedelta64`` +-------------------------------------------------------- +``np.datetime('NaT')`` should behave more like ``float('Nan')``. Add needed +infrastructure so ``np.isinf(a)`` and ``np.isnan(a)`` will run on +``datetime64`` and ``timedelta64`` dtypes. Also added specific loops for +``numpy.fmin`` and ``numpy.fmax`` that mask ``NaT``. This may require +adjustment to user- facing code. Specifically, code that either disallowed the +calls to ``numpy.isinf`` or ``numpy.isnan`` or checked that they raised an +exception will require adaptation, and code that mistakenly called +``numpy.fmax`` and ``numpy.fmin`` instead of ``numpy.maximum`` or +``numpy.minimum`` respectively will requre adjustment. This also affects +``numpy.nanmax`` and ``numpy.nanmin``. +(`gh-14841 <https://github.com/numpy/numpy/pull/14841>`__) + + +C API changes +============= + +``PyDataType_ISUNSIZED(descr)`` now returns False for structured datatypes +-------------------------------------------------------------------------- +Previously this returned True for any datatype of itemsize 0, but now this +returns false for the non-flexible datatype with itemsize 0, ``np.dtype([])``. +(`gh-14393 <https://github.com/numpy/numpy/pull/14393>`__) + + +New Features +============ + +Add our own ``*.pxd`` cython import file +---------------------------------------- +Added a ``numpy/__init__.pxd`` file. It will be used for ``cimport numpy`` +(`gh-12284 <https://github.com/numpy/numpy/pull/12284>`__) + +A tuple of axes can now be input to ``expand_dims`` +--------------------------------------------------- +The ``numpy.expand_dims`` ``axis`` keyword can now accept a tuple of +axes. Previously, ``axis`` was required to be an integer. +(`gh-14051 <https://github.com/numpy/numpy/pull/14051>`__) + +Support for 64-bit OpenBLAS +--------------------------- +Added support for 64-bit (ILP64) OpenBLAS. See ``site.cfg.example`` +for details. +(`gh-15012 <https://github.com/numpy/numpy/pull/15012>`__) + +Add ``--f2cmap`` option to F2PY +------------------------------- +Allow specifying a file to load Fortran-to-C type map +customizations from. +(`gh-15113 <https://github.com/numpy/numpy/pull/15113>`__) + + +Improvements +============ + +Different C numeric types of the same size have unique names +------------------------------------------------------------ +On any given platform, two of ``np.intc``, ``np.int_``, and ``np.longlong`` +would previously appear indistinguishable through their ``repr``, despite +their corresponding ``dtype`` having different properties. +A similar problem existed for the unsigned counterparts to these types, and on +some platforms for ``np.double`` and ``np.longdouble`` + +These types now always print with a unique ``__name__``. +(`gh-10151 <https://github.com/numpy/numpy/pull/10151>`__) + +``argwhere`` now produces a consistent result on 0d arrays +---------------------------------------------------------- +On N-d arrays, ``numpy.argwhere`` now always produces an array of shape +``(n_non_zero, arr.ndim)``, even when ``arr.ndim == 0``. Previously, the +last axis would have a dimension of 1 in this case. +(`gh-13610 <https://github.com/numpy/numpy/pull/13610>`__) + +Add ``axis`` argument for ``random.permutation`` and ``random.shuffle`` +----------------------------------------------------------------------- + +Previously the ``random.permutation`` and ``random.shuffle`` functions +can only shuffle an array along the first axis; they now have a +new argument ``axis`` which allows shuffle along a specified axis. +(`gh-13829 <https://github.com/numpy/numpy/pull/13829>`__) + +``method`` keyword argument for ``np.random.multivariate_normal`` +----------------------------------------------------------------- +A ``method`` keyword argument is now available for +``np.random.multivariate_normal`` with possible values +``{'svd', 'eigh', 'cholesky'}``. To use it, write +``np.random.multivariate_normal(..., method=<method>)``. +(`gh-14197 <https://github.com/numpy/numpy/pull/14197>`__) + +Add complex number support for ``numpy.fromstring`` +--------------------------------------------------- +Now ``numpy.fromstring`` can read complex numbers. +(`gh-14227 <https://github.com/numpy/numpy/pull/14227>`__) + +``numpy.unique`` has consistent axes order when ``axis`` is not None +-------------------------------------------------------------------- +Using ``moveaxis`` instead of ``swapaxes`` in ``numpy.unique``, so that the ordering of axes +except the axis in arguments will not be broken. +(`gh-14255 <https://github.com/numpy/numpy/pull/14255>`__) + +``numpy.matmul`` with boolean output now converts to boolean values +------------------------------------------------------------------- +Calling ``numpy.matmul`` where the output is a boolean array would fill the array +with uint8 equivalents of the result, rather than 0/1. Now it forces the output +to 0 or 1 (``NPY_TRUE`` or ``NPY_FALSE``). +(`gh-14464 <https://github.com/numpy/numpy/pull/14464>`__) + +``numpy.random.randint`` produced incorrect value when the range was ``2**32`` +------------------------------------------------------------------------------ +The implementation introduced in 1.17.0 had an incorrect check when +determining whether to use the 32-bit path or the full 64-bit +path that incorrectly redirected random integer generation with a high - low +range of ``2**32`` to the 64-bit generator. +(`gh-14501 <https://github.com/numpy/numpy/pull/14501>`__) + +Add complex number support for ``numpy.fromfile`` +------------------------------------------------- +Now ``numpy.fromfile`` can read complex numbers. +(`gh-14730 <https://github.com/numpy/numpy/pull/14730>`__) + +``std=c99`` added if compiler is named ``gcc`` +---------------------------------------------- +GCC before version 5 requires the ``-std=c99`` command line argument. Newer +compilers automatically turn on C99 mode. The compiler setup code will +automatically add the code if the compiler name has ``gcc`` in it. +(`gh-14771 <https://github.com/numpy/numpy/pull/14771>`__) + + +Changes +======= + + +``NaT`` now sorts to the end of arrays +-------------------------------------- +``NaT`` is now effectively treated as the largest integer for sorting +purposes, so that it sorts to the end of arrays. This change is for consistency +with ``NaN`` sorting behavior. +(`gh-12658 <https://github.com/numpy/numpy/pull/12658>`__) +(`gh-15068 <https://github.com/numpy/numpy/pull/15068>`__) + +Incorrect ``threshold`` in ``np.set_printoptions`` raises ``TypeError`` or ``ValueError`` +----------------------------------------------------------------------------------------- +Previously an incorrect ``threshold`` raised ``ValueError``; it now raises ``TypeError`` +for non-numeric types and ``ValueError`` for ``nan`` values. +(`gh-13899 <https://github.com/numpy/numpy/pull/13899>`__) + +Warn when saving a dtype with metadata +-------------------------------------- +A ``UserWarning`` will be emitted when saving an array via ``numpy.save`` with +``metadata``. Saving such an array may not preserve metadata, and if metadata +is preserved, loading it will cause a ``ValueError``. This shortcoming in save +and load will be addressed in a future release. +(`gh-14142 <https://github.com/numpy/numpy/pull/14142>`__) + +``numpy.distutils`` append behavior changed for LDFLAGS and similar +------------------------------------------------------------------- +`numpy.distutils` has always overridden rather than appended to ``LDFLAGS`` and +other similar such environment variables for compiling Fortran extensions. Now +the default behavior has changed to appending - which is the expected behavior +in most situations. To preserve the old (overwriting) behavior, set the +``NPY_DISTUTILS_APPEND_FLAGS`` environment variable to 0. This applies to: +``LDFLAGS``, ``F77FLAGS``, ``F90FLAGS``, ``FREEFLAGS``, ``FOPT``, ``FDEBUG``, +and ``FFLAGS``. NumPy 1.16 and 1.17 gave build warnings in situations where this +change in behavior would have affected the compile flags used. +(`gh-14248 <https://github.com/numpy/numpy/pull/14248>`__) + +Remove ``numpy.random.entropy`` without a deprecation +----------------------------------------------------- + +``numpy.random.entropy`` was added to the ``numpy.random`` namespace in 1.17.0. +It was meant to be a private c-extension module, but was exposed as public. +It has been replaced by ``numpy.random.SeedSequence`` so the module was +completely removed. +(`gh-14498 <https://github.com/numpy/numpy/pull/14498>`__) + +Add options to quiet build configuration and build with ``-Werror`` +------------------------------------------------------------------- +Added two new configuration options. During the ``build_src`` subcommand, as +part of configuring NumPy, the files ``_numpyconfig.h`` and ``config.h`` are +created by probing support for various runtime functions and routines. +Previously, the very verbose compiler output during this stage clouded more +important information. By default the output is silenced. Running ``runtests.py +--debug-info`` will add ``--verbose-cfg`` to the ``build_src`` subcommand, +which will restore the previous behaviour. + +Adding ``CFLAGS=-Werror`` to turn warnings into errors would trigger errors +during the configuration. Now ``runtests.py --warn-error`` will add +``--warn-error`` to the ``build`` subcommand, which will percolate to the +``build_ext`` and ``build_lib`` subcommands. This will add the compiler flag +to those stages and turn compiler warnings into errors while actually building +NumPy itself, avoiding the ``build_src`` subcommand compiler calls. + +(`gh-14527 <https://github.com/numpy/numpy/pull/14527>`__) +(`gh-14518 <https://github.com/numpy/numpy/pull/14518>`__) diff --git a/doc/release/1.3.0-notes.rst b/doc/source/release/1.3.0-notes.rst index 239714246..239714246 100644 --- a/doc/release/1.3.0-notes.rst +++ b/doc/source/release/1.3.0-notes.rst diff --git a/doc/release/1.4.0-notes.rst b/doc/source/release/1.4.0-notes.rst index 9480a054e..9480a054e 100644 --- a/doc/release/1.4.0-notes.rst +++ b/doc/source/release/1.4.0-notes.rst diff --git a/doc/release/1.5.0-notes.rst b/doc/source/release/1.5.0-notes.rst index a2184ab13..a2184ab13 100644 --- a/doc/release/1.5.0-notes.rst +++ b/doc/source/release/1.5.0-notes.rst diff --git a/doc/release/1.6.0-notes.rst b/doc/source/release/1.6.0-notes.rst index c5f53a0eb..c5f53a0eb 100644 --- a/doc/release/1.6.0-notes.rst +++ b/doc/source/release/1.6.0-notes.rst diff --git a/doc/release/1.6.1-notes.rst b/doc/source/release/1.6.1-notes.rst index 05fcb4ab9..05fcb4ab9 100644 --- a/doc/release/1.6.1-notes.rst +++ b/doc/source/release/1.6.1-notes.rst diff --git a/doc/release/1.6.2-notes.rst b/doc/source/release/1.6.2-notes.rst index 8f0b06f98..8f0b06f98 100644 --- a/doc/release/1.6.2-notes.rst +++ b/doc/source/release/1.6.2-notes.rst diff --git a/doc/release/1.7.0-notes.rst b/doc/source/release/1.7.0-notes.rst index f111f80dc..f111f80dc 100644 --- a/doc/release/1.7.0-notes.rst +++ b/doc/source/release/1.7.0-notes.rst diff --git a/doc/release/1.7.1-notes.rst b/doc/source/release/1.7.1-notes.rst index 04216b0df..04216b0df 100644 --- a/doc/release/1.7.1-notes.rst +++ b/doc/source/release/1.7.1-notes.rst diff --git a/doc/release/1.7.2-notes.rst b/doc/source/release/1.7.2-notes.rst index b0951bd72..b0951bd72 100644 --- a/doc/release/1.7.2-notes.rst +++ b/doc/source/release/1.7.2-notes.rst diff --git a/doc/release/1.8.0-notes.rst b/doc/source/release/1.8.0-notes.rst index 80c39f8bc..80c39f8bc 100644 --- a/doc/release/1.8.0-notes.rst +++ b/doc/source/release/1.8.0-notes.rst diff --git a/doc/release/1.8.1-notes.rst b/doc/source/release/1.8.1-notes.rst index ea34e75ac..ea34e75ac 100644 --- a/doc/release/1.8.1-notes.rst +++ b/doc/source/release/1.8.1-notes.rst diff --git a/doc/release/1.8.2-notes.rst b/doc/source/release/1.8.2-notes.rst index 71e549526..71e549526 100644 --- a/doc/release/1.8.2-notes.rst +++ b/doc/source/release/1.8.2-notes.rst diff --git a/doc/release/1.9.0-notes.rst b/doc/source/release/1.9.0-notes.rst index 7ea29e354..7ea29e354 100644 --- a/doc/release/1.9.0-notes.rst +++ b/doc/source/release/1.9.0-notes.rst diff --git a/doc/release/1.9.1-notes.rst b/doc/source/release/1.9.1-notes.rst index 4558237f4..4558237f4 100644 --- a/doc/release/1.9.1-notes.rst +++ b/doc/source/release/1.9.1-notes.rst diff --git a/doc/release/1.9.2-notes.rst b/doc/source/release/1.9.2-notes.rst index 268f3aa64..268f3aa64 100644 --- a/doc/release/1.9.2-notes.rst +++ b/doc/source/release/1.9.2-notes.rst diff --git a/doc/release/template.rst b/doc/source/release/template.rst index fdfec2be9..cde7646df 100644 --- a/doc/release/template.rst +++ b/doc/source/release/template.rst @@ -1,3 +1,5 @@ +:orphan: + ========================== NumPy 1.xx.x Release Notes ========================== diff --git a/doc/source/user/basics.indexing.rst b/doc/source/user/basics.indexing.rst index 8844adcae..0dca4b884 100644 --- a/doc/source/user/basics.indexing.rst +++ b/doc/source/user/basics.indexing.rst @@ -4,6 +4,10 @@ Indexing ******** -.. seealso:: :ref:`Indexing routines <routines.indexing>` +.. seealso:: + + :ref:`Indexing <arrays.indexing>` + + :ref:`Indexing routines <routines.indexing>` .. automodule:: numpy.doc.indexing diff --git a/doc/source/user/basics.io.genfromtxt.rst b/doc/source/user/basics.io.genfromtxt.rst index 6ef80bf8e..3fce6a8aa 100644 --- a/doc/source/user/basics.io.genfromtxt.rst +++ b/doc/source/user/basics.io.genfromtxt.rst @@ -27,13 +27,13 @@ Defining the input ================== The only mandatory argument of :func:`~numpy.genfromtxt` is the source of -the data. It can be a string, a list of strings, or a generator. If a -single string is provided, it is assumed to be the name of a local or -remote file, or an open file-like object with a :meth:`read` method, for -example, a file or :class:`io.StringIO` object. If a list of strings -or a generator returning strings is provided, each string is treated as one -line in a file. When the URL of a remote file is passed, the file is -automatically downloaded to the current directory and opened. +the data. It can be a string, a list of strings, a generator or an open +file-like object with a :meth:`read` method, for example, a file or +:class:`io.StringIO` object. If a single string is provided, it is assumed +to be the name of a local or remote file. If a list of strings or a generator +returning strings is provided, each string is treated as one line in a file. +When the URL of a remote file is passed, the file is automatically downloaded +to the current directory and opened. Recognized file types are text files and archives. Currently, the function recognizes :class:`gzip` and :class:`bz2` (`bzip2`) archives. The type of @@ -98,13 +98,11 @@ This behavior can be overwritten by setting the optional argument >>> # Without autostrip >>> np.genfromtxt(StringIO(data), delimiter=",", dtype="|U5") array([['1', ' abc ', ' 2'], - ['3', ' xxx', ' 4']], - dtype='|U5') + ['3', ' xxx', ' 4']], dtype='<U5') >>> # With autostrip >>> np.genfromtxt(StringIO(data), delimiter=",", dtype="|U5", autostrip=True) array([['1', 'abc', '2'], - ['3', 'xxx', '4']], - dtype='|U5') + ['3', 'xxx', '4']], dtype='<U5') The ``comments`` argument @@ -127,11 +125,11 @@ marker(s) is simply ignored:: ... 9, 0 ... """ >>> np.genfromtxt(StringIO(data), comments="#", delimiter=",") - [[ 1. 2.] - [ 3. 4.] - [ 5. 6.] - [ 7. 8.] - [ 9. 0.]] + array([[1., 2.], + [3., 4.], + [5., 6.], + [7., 8.], + [9., 0.]]) .. versionadded:: 1.7.0 @@ -376,12 +374,12 @@ single element of the wanted type. In the following example, the second column is converted from as string representing a percentage to a float between 0 and 1:: - >>> convertfunc = lambda x: float(x.strip("%"))/100. + >>> convertfunc = lambda x: float(x.strip(b"%"))/100. >>> data = u"1, 2.3%, 45.\n6, 78.9%, 0" >>> names = ("i", "p", "n") >>> # General case ..... >>> np.genfromtxt(StringIO(data), delimiter=",", names=names) - array([(1.0, nan, 45.0), (6.0, nan, 0.0)], + array([(1., nan, 45.), (6., nan, 0.)], dtype=[('i', '<f8'), ('p', '<f8'), ('n', '<f8')]) We need to keep in mind that by default, ``dtype=float``. A float is diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst index a13e1160a..1588de964 100644 --- a/doc/source/user/building.rst +++ b/doc/source/user/building.rst @@ -56,7 +56,7 @@ Basic Installation To install NumPy run:: - python setup.py install + pip install . To perform an in-place build that can be run from the source folder run:: @@ -69,6 +69,15 @@ Using ``virtualenv`` should work as expected. *Note: for build instructions to do development work on NumPy itself, see* :ref:`development-environment`. +Testing +------- + +Make sure to test your builds. To ensure everything stays in shape, see if all tests pass:: + + $ python runtests.py -v -m full + +For detailed info on testing, see :ref:`testing-builds`. + .. _parallel-builds: Parallel builds @@ -186,6 +195,39 @@ or:: BLAS=None LAPACK=None ATLAS=None python setup.py build +64-bit BLAS and LAPACK +~~~~~~~~~~~~~~~~~~~~~~ + +You can tell Numpy to use 64-bit BLAS/LAPACK libraries by setting the +environment variable:: + + NPY_USE_BLAS_ILP64=1 + +when building Numpy. The following 64-bit BLAS/LAPACK libraries are +supported: + +1. OpenBLAS ILP64 with ``64_`` symbol suffix (``openblas64_``) +2. OpenBLAS ILP64 without symbol suffix (``openblas_ilp64``) + +The order in which they are preferred is determined by +``NPY_BLAS_ILP64_ORDER`` and ``NPY_LAPACK_ILP64_ORDER`` environment +variables. The default value is ``openblas64_,openblas_ilp64``. + +.. note:: + + Using non-symbol-suffixed 64-bit BLAS/LAPACK in a program that also + uses 32-bit BLAS/LAPACK can cause crashes under certain conditions + (e.g. with embedded Python interpreters on Linux). + + The 64-bit OpenBLAS with ``64_`` symbol suffix is obtained by + compiling OpenBLAS with settings:: + + make INTERFACE64=1 SYMBOLSUFFIX=64_ + + The symbol suffix avoids the symbol name clashes between 32-bit and + 64-bit BLAS/LAPACK libraries. + + Supplying additional compiler flags ----------------------------------- diff --git a/doc/source/user/c-info.beyond-basics.rst b/doc/source/user/c-info.beyond-basics.rst index d4d941a5e..62e8139fe 100644 --- a/doc/source/user/c-info.beyond-basics.rst +++ b/doc/source/user/c-info.beyond-basics.rst @@ -217,14 +217,13 @@ type will behave much like a regular data-type except ufuncs must have 1-d loops registered to handle it separately. Also checking for whether or not other data-types can be cast "safely" to and from this new type or not will always return "can cast" unless you also register -which types your new data-type can be cast to and from. Adding -data-types is one of the less well-tested areas for NumPy 1.0, so -there may be bugs remaining in the approach. Only add a new data-type -if you can't do what you want to do using the OBJECT or VOID -data-types that are already available. As an example of what I -consider a useful application of the ability to add data-types is the -possibility of adding a data-type of arbitrary precision floats to -NumPy. +which types your new data-type can be cast to and from. + +The NumPy source code includes an example of a custom data-type as part +of its test suite. The file ``_rational_tests.c.src`` in the source code +directory ``numpy/numpy/core/src/umath/`` contains an implementation of +a data-type that represents a rational number as the ratio of two 32 bit +integers. .. index:: pair: dtype; adding new @@ -300,9 +299,10 @@ An example castfunc is: static void double_to_float(double *from, float* to, npy_intp n, - void* ig1, void* ig2); - while (n--) { - (*to++) = (double) *(from++); + void* ignore1, void* ignore2) { + while (n--) { + (*to++) = (double) *(from++); + } } This could then be registered to convert doubles to floats using the diff --git a/doc/source/user/c-info.how-to-extend.rst b/doc/source/user/c-info.how-to-extend.rst index 3961325fb..00ef8ab74 100644 --- a/doc/source/user/c-info.how-to-extend.rst +++ b/doc/source/user/c-info.how-to-extend.rst @@ -342,7 +342,7 @@ The method is to 4. If you are writing the algorithm, then I recommend that you use the stride information contained in the array to access the elements of - the array (the :c:func:`PyArray_GETPTR` macros make this painless). Then, + the array (the :c:func:`PyArray_GetPtr` macros make this painless). Then, you can relax your requirements so as not to force a single-segment array and the data-copying that might result. @@ -463,7 +463,7 @@ writeable). The syntax is This flag is useful to specify an array that will be used for both input and output. :c:func:`PyArray_ResolveWritebackIfCopy` - must be called before :func:`Py_DECREF` at + must be called before :c:func:`Py_DECREF` at the end of the interface routine to write back the temporary data into the original array passed in. Use of the :c:data:`NPY_ARRAY_WRITEBACKIFCOPY` or @@ -530,7 +530,7 @@ specific element of the array is determined only by the array of npy_intp variables, :c:func:`PyArray_STRIDES` (obj). In particular, this c-array of integers shows how many **bytes** must be added to the current element pointer to get to the next element in each dimension. -For arrays less than 4-dimensions there are :c:func:`PyArray_GETPTR{k}` +For arrays less than 4-dimensions there are ``PyArray_GETPTR{k}`` (obj, ...) macros where {k} is the integer 1, 2, 3, or 4 that make using the array strides easier. The arguments .... represent {k} non- negative integer indices into the array. For example, suppose ``E`` is @@ -543,7 +543,7 @@ contiguous arrays have particular striding patterns. Two array flags whether or not the striding pattern of a particular array matches the C-style contiguous or Fortran-style contiguous or neither. Whether or not the striding pattern matches a standard C or Fortran one can be -tested Using :c:func:`PyArray_ISCONTIGUOUS` (obj) and +tested Using :c:func:`PyArray_IS_C_CONTIGUOUS` (obj) and :c:func:`PyArray_ISFORTRAN` (obj) respectively. Most third-party libraries expect contiguous arrays. But, often it is not difficult to support general-purpose striding. I encourage you to use the striding diff --git a/doc/source/user/c-info.python-as-glue.rst b/doc/source/user/c-info.python-as-glue.rst index 01d2a64d1..7b9b096af 100644 --- a/doc/source/user/c-info.python-as-glue.rst +++ b/doc/source/user/c-info.python-as-glue.rst @@ -387,7 +387,7 @@ distribution of the ``add.f`` module (as part of the package Installation of the new package is easy using:: - python setup.py install + pip install . assuming you have the proper permissions to write to the main site- packages directory for the version of Python you are using. For the @@ -744,14 +744,14 @@ around this restriction that allow ctypes to integrate with other objects. 1. Don't set the argtypes attribute of the function object and define an - :obj:`_as_parameter_` method for the object you want to pass in. The - :obj:`_as_parameter_` method must return a Python int which will be passed + ``_as_parameter_`` method for the object you want to pass in. The + ``_as_parameter_`` method must return a Python int which will be passed directly to the function. 2. Set the argtypes attribute to a list whose entries contain objects with a classmethod named from_param that knows how to convert your object to an object that ctypes can understand (an int/long, string, - unicode, or object with the :obj:`_as_parameter_` attribute). + unicode, or object with the ``_as_parameter_`` attribute). NumPy uses both methods with a preference for the second method because it can be safer. The ctypes attribute of the ndarray returns @@ -764,7 +764,7 @@ correct type, shape, and has the correct flags set or risk nasty crashes if the data-pointer to inappropriate arrays are passed in. To implement the second method, NumPy provides the class-factory -function :func:`ndpointer` in the :mod:`ctypeslib` module. This +function :func:`ndpointer` in the :mod:`numpy.ctypeslib` module. This class-factory function produces an appropriate class that can be placed in an argtypes attribute entry of a ctypes function. The class will contain a from_param method which ctypes will use to convert any diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index 09647be86..6211d0c69 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -25,7 +25,7 @@ The Basics NumPy's main object is the homogeneous multidimensional array. It is a table of elements (usually numbers), all of the same type, indexed by a -tuple of positive integers. In NumPy dimensions are called *axes*. +tuple of non-negative integers. In NumPy dimensions are called *axes*. For example, the coordinates of a point in 3D space ``[1, 2, 1]`` has one axis. That axis has 3 elements in it, so we say it has a length @@ -206,8 +206,8 @@ of elements that we want, instead of the step:: `empty_like`, `arange`, `linspace`, - `numpy.random.rand`, - `numpy.random.randn`, + `numpy.random.RandomState.rand`, + `numpy.random.RandomState.randn`, `fromfunction`, `fromfile` @@ -270,7 +270,7 @@ can change the printing options using ``set_printoptions``. :: - >>> np.set_printoptions(threshold=np.nan) + >>> np.set_printoptions(threshold=sys.maxsize) # sys module should be imported Basic Operations @@ -732,9 +732,9 @@ stacks 1D arrays as columns into a 2D array. It is equivalent to array([[ 4., 3.], [ 2., 8.]]) -On the other hand, the function `row_stack` is equivalent to `vstack` +On the other hand, the function `ma.row_stack` is equivalent to `vstack` for any input arrays. -In general, for arrays of with more than two dimensions, +In general, for arrays with more than two dimensions, `hstack` stacks along their second axes, `vstack` stacks along their first axes, and `concatenate` diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd new file mode 100644 index 000000000..23bd22e36 --- /dev/null +++ b/numpy/__init__.pxd @@ -0,0 +1,978 @@ +# NumPy static imports for Cython +# +# If any of the PyArray_* functions are called, import_array must be +# called first. +# +# This also defines backwards-compatibility buffer acquisition +# code for use in Python 2.x (or Python <= 2.5 when NumPy starts +# implementing PEP-3118 directly). +# +# Because of laziness, the format string of the buffer is statically +# allocated. Increase the size if this is not enough, or submit a +# patch to do this properly. +# +# Author: Dag Sverre Seljebotn +# + +DEF _buffer_format_string_len = 255 + +cimport cpython.buffer as pybuf +from cpython.ref cimport Py_INCREF +from cpython.mem cimport PyObject_Malloc, PyObject_Free +from cpython.object cimport PyObject, PyTypeObject +from cpython.buffer cimport PyObject_GetBuffer +from cpython.type cimport type +cimport libc.stdio as stdio + +cdef extern from "Python.h": + ctypedef int Py_intptr_t + +cdef extern from "numpy/arrayobject.h": + ctypedef Py_intptr_t npy_intp + ctypedef size_t npy_uintp + + cdef enum NPY_TYPES: + NPY_BOOL + NPY_BYTE + NPY_UBYTE + NPY_SHORT + NPY_USHORT + NPY_INT + NPY_UINT + NPY_LONG + NPY_ULONG + NPY_LONGLONG + NPY_ULONGLONG + NPY_FLOAT + NPY_DOUBLE + NPY_LONGDOUBLE + NPY_CFLOAT + NPY_CDOUBLE + NPY_CLONGDOUBLE + NPY_OBJECT + NPY_STRING + NPY_UNICODE + NPY_VOID + NPY_DATETIME + NPY_TIMEDELTA + NPY_NTYPES + NPY_NOTYPE + + NPY_INT8 + NPY_INT16 + NPY_INT32 + NPY_INT64 + NPY_INT128 + NPY_INT256 + NPY_UINT8 + NPY_UINT16 + NPY_UINT32 + NPY_UINT64 + NPY_UINT128 + NPY_UINT256 + NPY_FLOAT16 + NPY_FLOAT32 + NPY_FLOAT64 + NPY_FLOAT80 + NPY_FLOAT96 + NPY_FLOAT128 + NPY_FLOAT256 + NPY_COMPLEX32 + NPY_COMPLEX64 + NPY_COMPLEX128 + NPY_COMPLEX160 + NPY_COMPLEX192 + NPY_COMPLEX256 + NPY_COMPLEX512 + + NPY_INTP + + ctypedef enum NPY_ORDER: + NPY_ANYORDER + NPY_CORDER + NPY_FORTRANORDER + NPY_KEEPORDER + + ctypedef enum NPY_CASTING: + NPY_NO_CASTING + NPY_EQUIV_CASTING + NPY_SAFE_CASTING + NPY_SAME_KIND_CASTING + NPY_UNSAFE_CASTING + + ctypedef enum NPY_CLIPMODE: + NPY_CLIP + NPY_WRAP + NPY_RAISE + + ctypedef enum NPY_SCALARKIND: + NPY_NOSCALAR, + NPY_BOOL_SCALAR, + NPY_INTPOS_SCALAR, + NPY_INTNEG_SCALAR, + NPY_FLOAT_SCALAR, + NPY_COMPLEX_SCALAR, + NPY_OBJECT_SCALAR + + ctypedef enum NPY_SORTKIND: + NPY_QUICKSORT + NPY_HEAPSORT + NPY_MERGESORT + + ctypedef enum NPY_SEARCHSIDE: + NPY_SEARCHLEFT + NPY_SEARCHRIGHT + + enum: + # DEPRECATED since NumPy 1.7 ! Do not use in new code! + NPY_C_CONTIGUOUS + NPY_F_CONTIGUOUS + NPY_CONTIGUOUS + NPY_FORTRAN + NPY_OWNDATA + NPY_FORCECAST + NPY_ENSURECOPY + NPY_ENSUREARRAY + NPY_ELEMENTSTRIDES + NPY_ALIGNED + NPY_NOTSWAPPED + NPY_WRITEABLE + NPY_UPDATEIFCOPY + NPY_ARR_HAS_DESCR + + NPY_BEHAVED + NPY_BEHAVED_NS + NPY_CARRAY + NPY_CARRAY_RO + NPY_FARRAY + NPY_FARRAY_RO + NPY_DEFAULT + + NPY_IN_ARRAY + NPY_OUT_ARRAY + NPY_INOUT_ARRAY + NPY_IN_FARRAY + NPY_OUT_FARRAY + NPY_INOUT_FARRAY + + NPY_UPDATE_ALL + + enum: + # Added in NumPy 1.7 to replace the deprecated enums above. + NPY_ARRAY_C_CONTIGUOUS + NPY_ARRAY_F_CONTIGUOUS + NPY_ARRAY_OWNDATA + NPY_ARRAY_FORCECAST + NPY_ARRAY_ENSURECOPY + NPY_ARRAY_ENSUREARRAY + NPY_ARRAY_ELEMENTSTRIDES + NPY_ARRAY_ALIGNED + NPY_ARRAY_NOTSWAPPED + NPY_ARRAY_WRITEABLE + NPY_ARRAY_UPDATEIFCOPY + + NPY_ARRAY_BEHAVED + NPY_ARRAY_BEHAVED_NS + NPY_ARRAY_CARRAY + NPY_ARRAY_CARRAY_RO + NPY_ARRAY_FARRAY + NPY_ARRAY_FARRAY_RO + NPY_ARRAY_DEFAULT + + NPY_ARRAY_IN_ARRAY + NPY_ARRAY_OUT_ARRAY + NPY_ARRAY_INOUT_ARRAY + NPY_ARRAY_IN_FARRAY + NPY_ARRAY_OUT_FARRAY + NPY_ARRAY_INOUT_FARRAY + + NPY_ARRAY_UPDATE_ALL + + cdef enum: + NPY_MAXDIMS + + npy_intp NPY_MAX_ELSIZE + + ctypedef void (*PyArray_VectorUnaryFunc)(void *, void *, npy_intp, void *, void *) + + ctypedef struct PyArray_ArrayDescr: + # shape is a tuple, but Cython doesn't support "tuple shape" + # inside a non-PyObject declaration, so we have to declare it + # as just a PyObject*. + PyObject* shape + + ctypedef struct PyArray_Descr: + pass + + ctypedef class numpy.dtype [object PyArray_Descr, check_size ignore]: + # Use PyDataType_* macros when possible, however there are no macros + # for accessing some of the fields, so some are defined. + cdef PyTypeObject* typeobj + cdef char kind + cdef char type + # Numpy sometimes mutates this without warning (e.g. it'll + # sometimes change "|" to "<" in shared dtype objects on + # little-endian machines). If this matters to you, use + # PyArray_IsNativeByteOrder(dtype.byteorder) instead of + # directly accessing this field. + cdef char byteorder + cdef char flags + cdef int type_num + cdef int itemsize "elsize" + cdef int alignment + cdef dict fields + cdef tuple names + # Use PyDataType_HASSUBARRAY to test whether this field is + # valid (the pointer can be NULL). Most users should access + # this field via the inline helper method PyDataType_SHAPE. + cdef PyArray_ArrayDescr* subarray + + ctypedef extern class numpy.flatiter [object PyArrayIterObject, check_size ignore]: + # Use through macros + pass + + ctypedef extern class numpy.broadcast [object PyArrayMultiIterObject, check_size ignore]: + cdef int numiter + cdef npy_intp size, index + cdef int nd + cdef npy_intp *dimensions + cdef void **iters + + ctypedef struct PyArrayObject: + # For use in situations where ndarray can't replace PyArrayObject*, + # like PyArrayObject**. + pass + + ctypedef class numpy.ndarray [object PyArrayObject, check_size ignore]: + cdef __cythonbufferdefaults__ = {"mode": "strided"} + + cdef: + # Only taking a few of the most commonly used and stable fields. + # One should use PyArray_* macros instead to access the C fields. + char *data + int ndim "nd" + npy_intp *shape "dimensions" + npy_intp *strides + dtype descr # deprecated since NumPy 1.7 ! + PyObject* base + + # Note: This syntax (function definition in pxd files) is an + # experimental exception made for __getbuffer__ and __releasebuffer__ + # -- the details of this may change. + def __getbuffer__(ndarray self, Py_buffer* info, int flags): + PyObject_GetBuffer(<object>self, info, flags); + + def __releasebuffer__(ndarray self, Py_buffer* info): + # We should call a possible tp_bufferrelease(self, info) but no + # interface to that is exposed by cython or python. And currently + # the function is NULL in numpy, we rely on refcounting to release + # info when self is collected + pass + + + ctypedef unsigned char npy_bool + + ctypedef signed char npy_byte + ctypedef signed short npy_short + ctypedef signed int npy_int + ctypedef signed long npy_long + ctypedef signed long long npy_longlong + + ctypedef unsigned char npy_ubyte + ctypedef unsigned short npy_ushort + ctypedef unsigned int npy_uint + ctypedef unsigned long npy_ulong + ctypedef unsigned long long npy_ulonglong + + ctypedef float npy_float + ctypedef double npy_double + ctypedef long double npy_longdouble + + ctypedef signed char npy_int8 + ctypedef signed short npy_int16 + ctypedef signed int npy_int32 + ctypedef signed long long npy_int64 + ctypedef signed long long npy_int96 + ctypedef signed long long npy_int128 + + ctypedef unsigned char npy_uint8 + ctypedef unsigned short npy_uint16 + ctypedef unsigned int npy_uint32 + ctypedef unsigned long long npy_uint64 + ctypedef unsigned long long npy_uint96 + ctypedef unsigned long long npy_uint128 + + ctypedef float npy_float32 + ctypedef double npy_float64 + ctypedef long double npy_float80 + ctypedef long double npy_float96 + ctypedef long double npy_float128 + + ctypedef struct npy_cfloat: + double real + double imag + + ctypedef struct npy_cdouble: + double real + double imag + + ctypedef struct npy_clongdouble: + long double real + long double imag + + ctypedef struct npy_complex64: + float real + float imag + + ctypedef struct npy_complex128: + double real + double imag + + ctypedef struct npy_complex160: + long double real + long double imag + + ctypedef struct npy_complex192: + long double real + long double imag + + ctypedef struct npy_complex256: + long double real + long double imag + + ctypedef struct PyArray_Dims: + npy_intp *ptr + int len + + int _import_array() except -1 + + # + # Macros from ndarrayobject.h + # + bint PyArray_CHKFLAGS(ndarray m, int flags) + bint PyArray_IS_C_CONTIGUOUS(ndarray arr) + bint PyArray_IS_F_CONTIGUOUS(ndarray arr) + bint PyArray_ISCONTIGUOUS(ndarray m) + bint PyArray_ISWRITEABLE(ndarray m) + bint PyArray_ISALIGNED(ndarray m) + + int PyArray_NDIM(ndarray) + bint PyArray_ISONESEGMENT(ndarray) + bint PyArray_ISFORTRAN(ndarray) + int PyArray_FORTRANIF(ndarray) + + void* PyArray_DATA(ndarray) + char* PyArray_BYTES(ndarray) + npy_intp* PyArray_DIMS(ndarray) + npy_intp* PyArray_STRIDES(ndarray) + npy_intp PyArray_DIM(ndarray, size_t) + npy_intp PyArray_STRIDE(ndarray, size_t) + + PyObject *PyArray_BASE(ndarray) # returns borrowed reference! + PyArray_Descr *PyArray_DESCR(ndarray) # returns borrowed reference to dtype! + int PyArray_FLAGS(ndarray) + npy_intp PyArray_ITEMSIZE(ndarray) + int PyArray_TYPE(ndarray arr) + + object PyArray_GETITEM(ndarray arr, void *itemptr) + int PyArray_SETITEM(ndarray arr, void *itemptr, object obj) + + bint PyTypeNum_ISBOOL(int) + bint PyTypeNum_ISUNSIGNED(int) + bint PyTypeNum_ISSIGNED(int) + bint PyTypeNum_ISINTEGER(int) + bint PyTypeNum_ISFLOAT(int) + bint PyTypeNum_ISNUMBER(int) + bint PyTypeNum_ISSTRING(int) + bint PyTypeNum_ISCOMPLEX(int) + bint PyTypeNum_ISPYTHON(int) + bint PyTypeNum_ISFLEXIBLE(int) + bint PyTypeNum_ISUSERDEF(int) + bint PyTypeNum_ISEXTENDED(int) + bint PyTypeNum_ISOBJECT(int) + + bint PyDataType_ISBOOL(dtype) + bint PyDataType_ISUNSIGNED(dtype) + bint PyDataType_ISSIGNED(dtype) + bint PyDataType_ISINTEGER(dtype) + bint PyDataType_ISFLOAT(dtype) + bint PyDataType_ISNUMBER(dtype) + bint PyDataType_ISSTRING(dtype) + bint PyDataType_ISCOMPLEX(dtype) + bint PyDataType_ISPYTHON(dtype) + bint PyDataType_ISFLEXIBLE(dtype) + bint PyDataType_ISUSERDEF(dtype) + bint PyDataType_ISEXTENDED(dtype) + bint PyDataType_ISOBJECT(dtype) + bint PyDataType_HASFIELDS(dtype) + bint PyDataType_HASSUBARRAY(dtype) + + bint PyArray_ISBOOL(ndarray) + bint PyArray_ISUNSIGNED(ndarray) + bint PyArray_ISSIGNED(ndarray) + bint PyArray_ISINTEGER(ndarray) + bint PyArray_ISFLOAT(ndarray) + bint PyArray_ISNUMBER(ndarray) + bint PyArray_ISSTRING(ndarray) + bint PyArray_ISCOMPLEX(ndarray) + bint PyArray_ISPYTHON(ndarray) + bint PyArray_ISFLEXIBLE(ndarray) + bint PyArray_ISUSERDEF(ndarray) + bint PyArray_ISEXTENDED(ndarray) + bint PyArray_ISOBJECT(ndarray) + bint PyArray_HASFIELDS(ndarray) + + bint PyArray_ISVARIABLE(ndarray) + + bint PyArray_SAFEALIGNEDCOPY(ndarray) + bint PyArray_ISNBO(char) # works on ndarray.byteorder + bint PyArray_IsNativeByteOrder(char) # works on ndarray.byteorder + bint PyArray_ISNOTSWAPPED(ndarray) + bint PyArray_ISBYTESWAPPED(ndarray) + + bint PyArray_FLAGSWAP(ndarray, int) + + bint PyArray_ISCARRAY(ndarray) + bint PyArray_ISCARRAY_RO(ndarray) + bint PyArray_ISFARRAY(ndarray) + bint PyArray_ISFARRAY_RO(ndarray) + bint PyArray_ISBEHAVED(ndarray) + bint PyArray_ISBEHAVED_RO(ndarray) + + + bint PyDataType_ISNOTSWAPPED(dtype) + bint PyDataType_ISBYTESWAPPED(dtype) + + bint PyArray_DescrCheck(object) + + bint PyArray_Check(object) + bint PyArray_CheckExact(object) + + # Cannot be supported due to out arg: + # bint PyArray_HasArrayInterfaceType(object, dtype, object, object&) + # bint PyArray_HasArrayInterface(op, out) + + + bint PyArray_IsZeroDim(object) + # Cannot be supported due to ## ## in macro: + # bint PyArray_IsScalar(object, verbatim work) + bint PyArray_CheckScalar(object) + bint PyArray_IsPythonNumber(object) + bint PyArray_IsPythonScalar(object) + bint PyArray_IsAnyScalar(object) + bint PyArray_CheckAnyScalar(object) + ndarray PyArray_GETCONTIGUOUS(ndarray) + bint PyArray_SAMESHAPE(ndarray, ndarray) + npy_intp PyArray_SIZE(ndarray) + npy_intp PyArray_NBYTES(ndarray) + + object PyArray_FROM_O(object) + object PyArray_FROM_OF(object m, int flags) + object PyArray_FROM_OT(object m, int type) + object PyArray_FROM_OTF(object m, int type, int flags) + object PyArray_FROMANY(object m, int type, int min, int max, int flags) + object PyArray_ZEROS(int nd, npy_intp* dims, int type, int fortran) + object PyArray_EMPTY(int nd, npy_intp* dims, int type, int fortran) + void PyArray_FILLWBYTE(object, int val) + npy_intp PyArray_REFCOUNT(object) + object PyArray_ContiguousFromAny(op, int, int min_depth, int max_depth) + unsigned char PyArray_EquivArrTypes(ndarray a1, ndarray a2) + bint PyArray_EquivByteorders(int b1, int b2) + object PyArray_SimpleNew(int nd, npy_intp* dims, int typenum) + object PyArray_SimpleNewFromData(int nd, npy_intp* dims, int typenum, void* data) + #object PyArray_SimpleNewFromDescr(int nd, npy_intp* dims, dtype descr) + object PyArray_ToScalar(void* data, ndarray arr) + + void* PyArray_GETPTR1(ndarray m, npy_intp i) + void* PyArray_GETPTR2(ndarray m, npy_intp i, npy_intp j) + void* PyArray_GETPTR3(ndarray m, npy_intp i, npy_intp j, npy_intp k) + void* PyArray_GETPTR4(ndarray m, npy_intp i, npy_intp j, npy_intp k, npy_intp l) + + void PyArray_XDECREF_ERR(ndarray) + # Cannot be supported due to out arg + # void PyArray_DESCR_REPLACE(descr) + + + object PyArray_Copy(ndarray) + object PyArray_FromObject(object op, int type, int min_depth, int max_depth) + object PyArray_ContiguousFromObject(object op, int type, int min_depth, int max_depth) + object PyArray_CopyFromObject(object op, int type, int min_depth, int max_depth) + + object PyArray_Cast(ndarray mp, int type_num) + object PyArray_Take(ndarray ap, object items, int axis) + object PyArray_Put(ndarray ap, object items, object values) + + void PyArray_ITER_RESET(flatiter it) nogil + void PyArray_ITER_NEXT(flatiter it) nogil + void PyArray_ITER_GOTO(flatiter it, npy_intp* destination) nogil + void PyArray_ITER_GOTO1D(flatiter it, npy_intp ind) nogil + void* PyArray_ITER_DATA(flatiter it) nogil + bint PyArray_ITER_NOTDONE(flatiter it) nogil + + void PyArray_MultiIter_RESET(broadcast multi) nogil + void PyArray_MultiIter_NEXT(broadcast multi) nogil + void PyArray_MultiIter_GOTO(broadcast multi, npy_intp dest) nogil + void PyArray_MultiIter_GOTO1D(broadcast multi, npy_intp ind) nogil + void* PyArray_MultiIter_DATA(broadcast multi, npy_intp i) nogil + void PyArray_MultiIter_NEXTi(broadcast multi, npy_intp i) nogil + bint PyArray_MultiIter_NOTDONE(broadcast multi) nogil + + # Functions from __multiarray_api.h + + # Functions taking dtype and returning object/ndarray are disabled + # for now as they steal dtype references. I'm conservative and disable + # more than is probably needed until it can be checked further. + int PyArray_SetNumericOps (object) + object PyArray_GetNumericOps () + int PyArray_INCREF (ndarray) + int PyArray_XDECREF (ndarray) + void PyArray_SetStringFunction (object, int) + dtype PyArray_DescrFromType (int) + object PyArray_TypeObjectFromType (int) + char * PyArray_Zero (ndarray) + char * PyArray_One (ndarray) + #object PyArray_CastToType (ndarray, dtype, int) + int PyArray_CastTo (ndarray, ndarray) + int PyArray_CastAnyTo (ndarray, ndarray) + int PyArray_CanCastSafely (int, int) + npy_bool PyArray_CanCastTo (dtype, dtype) + int PyArray_ObjectType (object, int) + dtype PyArray_DescrFromObject (object, dtype) + #ndarray* PyArray_ConvertToCommonType (object, int *) + dtype PyArray_DescrFromScalar (object) + dtype PyArray_DescrFromTypeObject (object) + npy_intp PyArray_Size (object) + #object PyArray_Scalar (void *, dtype, object) + #object PyArray_FromScalar (object, dtype) + void PyArray_ScalarAsCtype (object, void *) + #int PyArray_CastScalarToCtype (object, void *, dtype) + #int PyArray_CastScalarDirect (object, dtype, void *, int) + object PyArray_ScalarFromObject (object) + #PyArray_VectorUnaryFunc * PyArray_GetCastFunc (dtype, int) + object PyArray_FromDims (int, int *, int) + #object PyArray_FromDimsAndDataAndDescr (int, int *, dtype, char *) + #object PyArray_FromAny (object, dtype, int, int, int, object) + object PyArray_EnsureArray (object) + object PyArray_EnsureAnyArray (object) + #object PyArray_FromFile (stdio.FILE *, dtype, npy_intp, char *) + #object PyArray_FromString (char *, npy_intp, dtype, npy_intp, char *) + #object PyArray_FromBuffer (object, dtype, npy_intp, npy_intp) + #object PyArray_FromIter (object, dtype, npy_intp) + object PyArray_Return (ndarray) + #object PyArray_GetField (ndarray, dtype, int) + #int PyArray_SetField (ndarray, dtype, int, object) + object PyArray_Byteswap (ndarray, npy_bool) + object PyArray_Resize (ndarray, PyArray_Dims *, int, NPY_ORDER) + int PyArray_MoveInto (ndarray, ndarray) + int PyArray_CopyInto (ndarray, ndarray) + int PyArray_CopyAnyInto (ndarray, ndarray) + int PyArray_CopyObject (ndarray, object) + object PyArray_NewCopy (ndarray, NPY_ORDER) + object PyArray_ToList (ndarray) + object PyArray_ToString (ndarray, NPY_ORDER) + int PyArray_ToFile (ndarray, stdio.FILE *, char *, char *) + int PyArray_Dump (object, object, int) + object PyArray_Dumps (object, int) + int PyArray_ValidType (int) + void PyArray_UpdateFlags (ndarray, int) + object PyArray_New (type, int, npy_intp *, int, npy_intp *, void *, int, int, object) + #object PyArray_NewFromDescr (type, dtype, int, npy_intp *, npy_intp *, void *, int, object) + #dtype PyArray_DescrNew (dtype) + dtype PyArray_DescrNewFromType (int) + double PyArray_GetPriority (object, double) + object PyArray_IterNew (object) + object PyArray_MultiIterNew (int, ...) + + int PyArray_PyIntAsInt (object) + npy_intp PyArray_PyIntAsIntp (object) + int PyArray_Broadcast (broadcast) + void PyArray_FillObjectArray (ndarray, object) + int PyArray_FillWithScalar (ndarray, object) + npy_bool PyArray_CheckStrides (int, int, npy_intp, npy_intp, npy_intp *, npy_intp *) + dtype PyArray_DescrNewByteorder (dtype, char) + object PyArray_IterAllButAxis (object, int *) + #object PyArray_CheckFromAny (object, dtype, int, int, int, object) + #object PyArray_FromArray (ndarray, dtype, int) + object PyArray_FromInterface (object) + object PyArray_FromStructInterface (object) + #object PyArray_FromArrayAttr (object, dtype, object) + #NPY_SCALARKIND PyArray_ScalarKind (int, ndarray*) + int PyArray_CanCoerceScalar (int, int, NPY_SCALARKIND) + object PyArray_NewFlagsObject (object) + npy_bool PyArray_CanCastScalar (type, type) + #int PyArray_CompareUCS4 (npy_ucs4 *, npy_ucs4 *, register size_t) + int PyArray_RemoveSmallest (broadcast) + int PyArray_ElementStrides (object) + void PyArray_Item_INCREF (char *, dtype) + void PyArray_Item_XDECREF (char *, dtype) + object PyArray_FieldNames (object) + object PyArray_Transpose (ndarray, PyArray_Dims *) + object PyArray_TakeFrom (ndarray, object, int, ndarray, NPY_CLIPMODE) + object PyArray_PutTo (ndarray, object, object, NPY_CLIPMODE) + object PyArray_PutMask (ndarray, object, object) + object PyArray_Repeat (ndarray, object, int) + object PyArray_Choose (ndarray, object, ndarray, NPY_CLIPMODE) + int PyArray_Sort (ndarray, int, NPY_SORTKIND) + object PyArray_ArgSort (ndarray, int, NPY_SORTKIND) + object PyArray_SearchSorted (ndarray, object, NPY_SEARCHSIDE) + object PyArray_ArgMax (ndarray, int, ndarray) + object PyArray_ArgMin (ndarray, int, ndarray) + object PyArray_Reshape (ndarray, object) + object PyArray_Newshape (ndarray, PyArray_Dims *, NPY_ORDER) + object PyArray_Squeeze (ndarray) + #object PyArray_View (ndarray, dtype, type) + object PyArray_SwapAxes (ndarray, int, int) + object PyArray_Max (ndarray, int, ndarray) + object PyArray_Min (ndarray, int, ndarray) + object PyArray_Ptp (ndarray, int, ndarray) + object PyArray_Mean (ndarray, int, int, ndarray) + object PyArray_Trace (ndarray, int, int, int, int, ndarray) + object PyArray_Diagonal (ndarray, int, int, int) + object PyArray_Clip (ndarray, object, object, ndarray) + object PyArray_Conjugate (ndarray, ndarray) + object PyArray_Nonzero (ndarray) + object PyArray_Std (ndarray, int, int, ndarray, int) + object PyArray_Sum (ndarray, int, int, ndarray) + object PyArray_CumSum (ndarray, int, int, ndarray) + object PyArray_Prod (ndarray, int, int, ndarray) + object PyArray_CumProd (ndarray, int, int, ndarray) + object PyArray_All (ndarray, int, ndarray) + object PyArray_Any (ndarray, int, ndarray) + object PyArray_Compress (ndarray, object, int, ndarray) + object PyArray_Flatten (ndarray, NPY_ORDER) + object PyArray_Ravel (ndarray, NPY_ORDER) + npy_intp PyArray_MultiplyList (npy_intp *, int) + int PyArray_MultiplyIntList (int *, int) + void * PyArray_GetPtr (ndarray, npy_intp*) + int PyArray_CompareLists (npy_intp *, npy_intp *, int) + #int PyArray_AsCArray (object*, void *, npy_intp *, int, dtype) + #int PyArray_As1D (object*, char **, int *, int) + #int PyArray_As2D (object*, char ***, int *, int *, int) + int PyArray_Free (object, void *) + #int PyArray_Converter (object, object*) + int PyArray_IntpFromSequence (object, npy_intp *, int) + object PyArray_Concatenate (object, int) + object PyArray_InnerProduct (object, object) + object PyArray_MatrixProduct (object, object) + object PyArray_CopyAndTranspose (object) + object PyArray_Correlate (object, object, int) + int PyArray_TypestrConvert (int, int) + #int PyArray_DescrConverter (object, dtype*) + #int PyArray_DescrConverter2 (object, dtype*) + int PyArray_IntpConverter (object, PyArray_Dims *) + #int PyArray_BufferConverter (object, chunk) + int PyArray_AxisConverter (object, int *) + int PyArray_BoolConverter (object, npy_bool *) + int PyArray_ByteorderConverter (object, char *) + int PyArray_OrderConverter (object, NPY_ORDER *) + unsigned char PyArray_EquivTypes (dtype, dtype) + #object PyArray_Zeros (int, npy_intp *, dtype, int) + #object PyArray_Empty (int, npy_intp *, dtype, int) + object PyArray_Where (object, object, object) + object PyArray_Arange (double, double, double, int) + #object PyArray_ArangeObj (object, object, object, dtype) + int PyArray_SortkindConverter (object, NPY_SORTKIND *) + object PyArray_LexSort (object, int) + object PyArray_Round (ndarray, int, ndarray) + unsigned char PyArray_EquivTypenums (int, int) + int PyArray_RegisterDataType (dtype) + int PyArray_RegisterCastFunc (dtype, int, PyArray_VectorUnaryFunc *) + int PyArray_RegisterCanCast (dtype, int, NPY_SCALARKIND) + #void PyArray_InitArrFuncs (PyArray_ArrFuncs *) + object PyArray_IntTupleFromIntp (int, npy_intp *) + int PyArray_TypeNumFromName (char *) + int PyArray_ClipmodeConverter (object, NPY_CLIPMODE *) + #int PyArray_OutputConverter (object, ndarray*) + object PyArray_BroadcastToShape (object, npy_intp *, int) + void _PyArray_SigintHandler (int) + void* _PyArray_GetSigintBuf () + #int PyArray_DescrAlignConverter (object, dtype*) + #int PyArray_DescrAlignConverter2 (object, dtype*) + int PyArray_SearchsideConverter (object, void *) + object PyArray_CheckAxis (ndarray, int *, int) + npy_intp PyArray_OverflowMultiplyList (npy_intp *, int) + int PyArray_CompareString (char *, char *, size_t) + int PyArray_SetBaseObject(ndarray, base) # NOTE: steals a reference to base! Use "set_array_base()" instead. + + +# Typedefs that matches the runtime dtype objects in +# the numpy module. + +# The ones that are commented out needs an IFDEF function +# in Cython to enable them only on the right systems. + +ctypedef npy_int8 int8_t +ctypedef npy_int16 int16_t +ctypedef npy_int32 int32_t +ctypedef npy_int64 int64_t +#ctypedef npy_int96 int96_t +#ctypedef npy_int128 int128_t + +ctypedef npy_uint8 uint8_t +ctypedef npy_uint16 uint16_t +ctypedef npy_uint32 uint32_t +ctypedef npy_uint64 uint64_t +#ctypedef npy_uint96 uint96_t +#ctypedef npy_uint128 uint128_t + +ctypedef npy_float32 float32_t +ctypedef npy_float64 float64_t +#ctypedef npy_float80 float80_t +#ctypedef npy_float128 float128_t + +ctypedef float complex complex64_t +ctypedef double complex complex128_t + +# The int types are mapped a bit surprising -- +# numpy.int corresponds to 'l' and numpy.long to 'q' +ctypedef npy_long int_t +ctypedef npy_longlong long_t +ctypedef npy_longlong longlong_t + +ctypedef npy_ulong uint_t +ctypedef npy_ulonglong ulong_t +ctypedef npy_ulonglong ulonglong_t + +ctypedef npy_intp intp_t +ctypedef npy_uintp uintp_t + +ctypedef npy_double float_t +ctypedef npy_double double_t +ctypedef npy_longdouble longdouble_t + +ctypedef npy_cfloat cfloat_t +ctypedef npy_cdouble cdouble_t +ctypedef npy_clongdouble clongdouble_t + +ctypedef npy_cdouble complex_t + +cdef inline object PyArray_MultiIterNew1(a): + return PyArray_MultiIterNew(1, <void*>a) + +cdef inline object PyArray_MultiIterNew2(a, b): + return PyArray_MultiIterNew(2, <void*>a, <void*>b) + +cdef inline object PyArray_MultiIterNew3(a, b, c): + return PyArray_MultiIterNew(3, <void*>a, <void*>b, <void*> c) + +cdef inline object PyArray_MultiIterNew4(a, b, c, d): + return PyArray_MultiIterNew(4, <void*>a, <void*>b, <void*>c, <void*> d) + +cdef inline object PyArray_MultiIterNew5(a, b, c, d, e): + return PyArray_MultiIterNew(5, <void*>a, <void*>b, <void*>c, <void*> d, <void*> e) + +cdef inline tuple PyDataType_SHAPE(dtype d): + if PyDataType_HASSUBARRAY(d): + return <tuple>d.subarray.shape + else: + return () + +cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: + # Recursive utility function used in __getbuffer__ to get format + # string. The new location in the format string is returned. + + cdef dtype child + cdef int endian_detector = 1 + cdef bint little_endian = ((<char*>&endian_detector)[0] != 0) + cdef tuple fields + + for childname in descr.names: + fields = descr.fields[childname] + child, new_offset = fields + + if (end - f) - <int>(new_offset - offset[0]) < 15: + raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") + + if ((child.byteorder == c'>' and little_endian) or + (child.byteorder == c'<' and not little_endian)): + raise ValueError(u"Non-native byte order not supported") + # One could encode it in the format string and have Cython + # complain instead, BUT: < and > in format strings also imply + # standardized sizes for datatypes, and we rely on native in + # order to avoid reencoding data types based on their size. + # + # A proper PEP 3118 exporter for other clients than Cython + # must deal properly with this! + + # Output padding bytes + while offset[0] < new_offset: + f[0] = 120 # "x"; pad byte + f += 1 + offset[0] += 1 + + offset[0] += child.itemsize + + if not PyDataType_HASFIELDS(child): + t = child.type_num + if end - f < 5: + raise RuntimeError(u"Format string allocated too short.") + + # Until ticket #99 is fixed, use integers to avoid warnings + if t == NPY_BYTE: f[0] = 98 #"b" + elif t == NPY_UBYTE: f[0] = 66 #"B" + elif t == NPY_SHORT: f[0] = 104 #"h" + elif t == NPY_USHORT: f[0] = 72 #"H" + elif t == NPY_INT: f[0] = 105 #"i" + elif t == NPY_UINT: f[0] = 73 #"I" + elif t == NPY_LONG: f[0] = 108 #"l" + elif t == NPY_ULONG: f[0] = 76 #"L" + elif t == NPY_LONGLONG: f[0] = 113 #"q" + elif t == NPY_ULONGLONG: f[0] = 81 #"Q" + elif t == NPY_FLOAT: f[0] = 102 #"f" + elif t == NPY_DOUBLE: f[0] = 100 #"d" + elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" + elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf + elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd + elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg + elif t == NPY_OBJECT: f[0] = 79 #"O" + else: + raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) + f += 1 + else: + # Cython ignores struct boundary information ("T{...}"), + # so don't output it + f = _util_dtypestring(child, f, end, offset) + return f + + +# +# ufunc API +# + +cdef extern from "numpy/ufuncobject.h": + + ctypedef void (*PyUFuncGenericFunction) (char **, npy_intp *, npy_intp *, void *) + + ctypedef extern class numpy.ufunc [object PyUFuncObject, check_size ignore]: + cdef: + int nin, nout, nargs + int identity + PyUFuncGenericFunction *functions + void **data + int ntypes + int check_return + char *name + char *types + char *doc + void *ptr + PyObject *obj + PyObject *userloops + + cdef enum: + PyUFunc_Zero + PyUFunc_One + PyUFunc_None + UFUNC_ERR_IGNORE + UFUNC_ERR_WARN + UFUNC_ERR_RAISE + UFUNC_ERR_CALL + UFUNC_ERR_PRINT + UFUNC_ERR_LOG + UFUNC_MASK_DIVIDEBYZERO + UFUNC_MASK_OVERFLOW + UFUNC_MASK_UNDERFLOW + UFUNC_MASK_INVALID + UFUNC_SHIFT_DIVIDEBYZERO + UFUNC_SHIFT_OVERFLOW + UFUNC_SHIFT_UNDERFLOW + UFUNC_SHIFT_INVALID + UFUNC_FPE_DIVIDEBYZERO + UFUNC_FPE_OVERFLOW + UFUNC_FPE_UNDERFLOW + UFUNC_FPE_INVALID + UFUNC_ERR_DEFAULT + UFUNC_ERR_DEFAULT2 + + object PyUFunc_FromFuncAndData(PyUFuncGenericFunction *, + void **, char *, int, int, int, int, char *, char *, int) + int PyUFunc_RegisterLoopForType(ufunc, int, + PyUFuncGenericFunction, int *, void *) + int PyUFunc_GenericFunction \ + (ufunc, PyObject *, PyObject *, PyArrayObject **) + void PyUFunc_f_f_As_d_d \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_d_d \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_f_f \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_g_g \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_F_F_As_D_D \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_F_F \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_D_D \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_G_G \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_O_O \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_ff_f_As_dd_d \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_ff_f \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_dd_d \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_gg_g \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_FF_F_As_DD_D \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_DD_D \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_FF_F \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_GG_G \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_OO_O \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_O_O_method \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_OO_O_method \ + (char **, npy_intp *, npy_intp *, void *) + void PyUFunc_On_Om \ + (char **, npy_intp *, npy_intp *, void *) + int PyUFunc_GetPyValues \ + (char *, int *, int *, PyObject **) + int PyUFunc_checkfperr \ + (int, PyObject *, int *) + void PyUFunc_clearfperr() + int PyUFunc_getfperr() + int PyUFunc_handlefperr \ + (int, PyObject *, int, int *) + int PyUFunc_ReplaceLoopBySignature \ + (ufunc, PyUFuncGenericFunction, int *, PyUFuncGenericFunction *) + object PyUFunc_FromFuncAndDataAndSignature \ + (PyUFuncGenericFunction *, void **, char *, int, int, int, + int, char *, char *, int, char *) + + int _import_umath() except -1 + +cdef inline void set_array_base(ndarray arr, object base): + Py_INCREF(base) # important to do this before stealing the reference below! + PyArray_SetBaseObject(arr, base) + +cdef inline object get_array_base(ndarray arr): + base = PyArray_BASE(arr) + if base is NULL: + return None + return <object>base + +# Versions of the import_* functions which are more suitable for +# Cython code. +cdef inline int import_array() except -1: + try: + _import_array() + except Exception: + raise ImportError("numpy.core.multiarray failed to import") + +cdef inline int import_umath() except -1: + try: + _import_umath() + except Exception: + raise ImportError("numpy.core.umath failed to import") + +cdef inline int import_ufunc() except -1: + try: + _import_umath() + except Exception: + raise ImportError("numpy.core.umath failed to import") diff --git a/numpy/__init__.py b/numpy/__init__.py index ba88c733f..349914b2f 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -143,7 +143,9 @@ else: from .core import * from . import compat from . import lib + # FIXME: why have numpy.lib if everything is imported here?? from .lib import * + from . import linalg from . import fft from . import polynomial @@ -156,6 +158,7 @@ else: # Make these accessible from numpy name-space # but not imported in from numpy import * + # TODO[gh-6103]: Deprecate these if sys.version_info[0] >= 3: from builtins import bool, int, float, complex, object, str unicode = str @@ -172,6 +175,19 @@ else: __all__.extend(lib.__all__) __all__.extend(['linalg', 'fft', 'random', 'ctypeslib', 'ma']) + # These are added by `from .core import *` and `core.__all__`, but we + # overwrite them above with builtins we do _not_ want to export. + __all__.remove('long') + __all__.remove('unicode') + + # Remove things that are in the numpy.lib but not in the numpy namespace + # Note that there is a test (numpy/tests/test_public_api.py:test_numpy_namespace) + # that prevents adding more things to the main namespace by accident. + # The list below will grow until the `from .lib import *` fixme above is + # taken care of + __all__.remove('Arrayterator') + del Arrayterator + # Filter out Cython harmless warnings warnings.filterwarnings("ignore", message="numpy.dtype size changed") warnings.filterwarnings("ignore", message="numpy.ufunc size changed") @@ -182,9 +198,34 @@ else: oldnumeric = 'removed' numarray = 'removed' - # We don't actually use this ourselves anymore, but I'm not 100% sure that - # no-one else in the world is using it (though I hope not) - from .testing import Tester + if sys.version_info[:2] >= (3, 7): + # Importing Tester requires importing all of UnitTest which is not a + # cheap import Since it is mainly used in test suits, we lazy import it + # here to save on the order of 10 ms of import time for most users + # + # The previous way Tester was imported also had a side effect of adding + # the full `numpy.testing` namespace + # + # module level getattr is only supported in 3.7 onwards + # https://www.python.org/dev/peps/pep-0562/ + def __getattr__(attr): + if attr == 'testing': + import numpy.testing as testing + return testing + elif attr == 'Tester': + from .testing import Tester + return Tester + else: + raise AttributeError("module {!r} has no attribute " + "{!r}".format(__name__, attr)) + + def __dir__(): + return list(globals().keys()) + ['Tester', 'testing'] + + else: + # We don't actually use this ourselves anymore, but I'm not 100% sure that + # no-one else in the world is using it (though I hope not) + from .testing import Tester # Pytest testing from numpy._pytesttester import PytestTester diff --git a/numpy/_pytesttester.py b/numpy/_pytesttester.py index 8d1a3811c..b25224c20 100644 --- a/numpy/_pytesttester.py +++ b/numpy/_pytesttester.py @@ -48,10 +48,9 @@ class PytestTester(object): """ Pytest test runner. - This class is made available in ``numpy.testing``, and a test function - is typically added to a package's __init__.py like so:: + A test function is typically added to a package's __init__.py like so:: - from numpy.testing import PytestTester + from numpy._pytesttester import PytestTester test = PytestTester(__name__).test del PytestTester @@ -68,6 +67,12 @@ class PytestTester(object): module_name : module name The name of the module to test. + Notes + ----- + Unlike the previous ``nose``-based implementation, this class is not + publicly exposed as it performs some ``numpy``-specific warning + suppression. + """ def __init__(self, module_name): self.module_name = module_name diff --git a/numpy/compat/py3k.py b/numpy/compat/py3k.py index c9ed9d52c..90e17d6d6 100644 --- a/numpy/compat/py3k.py +++ b/numpy/compat/py3k.py @@ -1,9 +1,15 @@ """ -Python 3 compatibility tools. +Python 3.X compatibility tools. -""" -from __future__ import division, absolute_import, print_function +While this file was originally intented for Python 2 -> 3 transition, +it is now used to create a compatibility layer between different +minor versions of Python 3. +While the active version of numpy may not support a given version of python, we +allow downstream libraries to continue to use these shims for forward +compatibility with numpy while they transition their code to newer versions of +Python. +""" __all__ = ['bytes', 'asbytes', 'isfileobj', 'getexception', 'strchar', 'unicode', 'asunicode', 'asbytes_nested', 'asunicode_nested', 'asstr', 'open_latin1', 'long', 'basestring', 'sixu', diff --git a/numpy/conftest.py b/numpy/conftest.py index 7834dd39d..1baf4adda 100644 --- a/numpy/conftest.py +++ b/numpy/conftest.py @@ -3,6 +3,8 @@ Pytest configuration and fixtures for the Numpy test suite. """ from __future__ import division, absolute_import, print_function +import os + import pytest import numpy @@ -17,9 +19,27 @@ def pytest_configure(config): config.addinivalue_line("markers", "valgrind_error: Tests that are known to error under valgrind.") config.addinivalue_line("markers", + "leaks_references: Tests that are known to leak references.") + config.addinivalue_line("markers", "slow: Tests that are very slow.") +def pytest_addoption(parser): + parser.addoption("--available-memory", action="store", default=None, + help=("Set amount of memory available for running the " + "test suite. This can result to tests requiring " + "especially large amounts of memory to be skipped. " + "Equivalent to setting environment variable " + "NPY_AVAILABLE_MEM. Default: determined" + "automatically.")) + + +def pytest_sessionstart(session): + available_mem = session.config.getoption('available_memory') + if available_mem is not None: + os.environ['NPY_AVAILABLE_MEM'] = available_mem + + #FIXME when yield tests are gone. @pytest.hookimpl() def pytest_itemcollected(item): diff --git a/numpy/core/__init__.py b/numpy/core/__init__.py index ce443bb22..c3b3f0392 100644 --- a/numpy/core/__init__.py +++ b/numpy/core/__init__.py @@ -1,6 +1,13 @@ +""" +Contains the core of NumPy: ndarray, ufuncs, dtypes, etc. + +Please note that this module is private. All functions and objects +are available in the main ``numpy`` namespace - use that instead. + +""" + from __future__ import division, absolute_import, print_function -from .info import __doc__ from numpy.version import version as __version__ import os diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 02700c8ca..2f1273904 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -94,7 +94,7 @@ add_newdoc('numpy.core', 'flatiter', ('coords', >>> fl = x.flat >>> fl.coords (0, 0) - >>> fl.next() + >>> next(fl) 0 >>> fl.coords (0, 1) @@ -113,7 +113,7 @@ add_newdoc('numpy.core', 'flatiter', ('index', >>> fl = x.flat >>> fl.index 0 - >>> fl.next() + >>> next(fl) 0 >>> fl.index 1 @@ -386,12 +386,12 @@ add_newdoc('numpy.core', 'nditer', >>> luf(lambda i,j:i*i + j/2, a, b) array([ 0.5, 1.5, 4.5, 9.5, 16.5]) - If operand flags `"writeonly"` or `"readwrite"` are used the operands may - be views into the original data with the `WRITEBACKIFCOPY` flag. In this case - nditer must be used as a context manager or the nditer.close - method must be called before using the result. The temporary - data will be written back to the original data when the `__exit__` - function is called but not before: + If operand flags `"writeonly"` or `"readwrite"` are used the + operands may be views into the original data with the + `WRITEBACKIFCOPY` flag. In this case `nditer` must be used as a + context manager or the `nditer.close` method must be called before + using the result. The temporary data will be written back to the + original data when the `__exit__` function is called but not before: >>> a = np.arange(6, dtype='i4')[::-2] >>> with np.nditer(a, [], @@ -413,6 +413,8 @@ add_newdoc('numpy.core', 'nditer', `x.data` will still point at some part of `a.data`, and writing to one will affect the other. + Context management and the `close` method appeared in version 1.15.0. + """) # nditer methods @@ -568,6 +570,8 @@ add_newdoc('numpy.core', 'nditer', ('close', Resolve all writeback semantics in writeable operands. + .. versionadded:: 1.15.0 + See Also -------- @@ -666,7 +670,7 @@ add_newdoc('numpy.core', 'broadcast', ('iters', >>> y = np.array([[4], [5], [6]]) >>> b = np.broadcast(x, y) >>> row, col = b.iters - >>> row.next(), col.next() + >>> next(row), next(col) (1, 4) """)) @@ -795,8 +799,7 @@ add_newdoc('numpy.core.multiarray', 'array', dtype : data-type, optional The desired data-type for the array. If not given, then the type will be determined as the minimum type required to hold the objects in the - sequence. This argument can only be used to 'upcast' the array. For - downcasting, use the .astype(t) method. + sequence. copy : bool, optional If true (default), then the object is copied. Otherwise, a copy will only be made if __array__ returns a copy, if obj is a nested sequence, @@ -1033,7 +1036,12 @@ add_newdoc('numpy.core.multiarray', 'fromstring', A string containing the data. dtype : data-type, optional The data type of the array; default: float. For binary input data, - the data must be in exactly this format. + the data must be in exactly this format. Most builtin numeric types are + supported and extension types may be supported. + + .. versionadded:: 1.18.0 + Complex dtypes. + count : int, optional Read this number of `dtype` elements from the data. If this is negative (the default), the count will be determined from the @@ -1169,6 +1177,11 @@ add_newdoc('numpy.core.multiarray', 'fromfile', Data type of the returned array. For binary files, it is used to determine the size and byte-order of the items in the file. + Most builtin numeric types are supported and extension types may be supported. + + .. versionadded:: 1.18.0 + Complex dtypes. + count : int Number of items to read. ``-1`` means all items (i.e., the complete file). @@ -1193,7 +1206,7 @@ add_newdoc('numpy.core.multiarray', 'fromfile', Notes ----- Do not rely on the combination of `tofile` and `fromfile` for - data storage, as the binary files generated are are not platform + data storage, as the binary files generated are not platform independent. In particular, no byte-order or data-type information is saved. Data can be stored in the platform independent ``.npy`` format using `save` and `load` instead. @@ -1323,9 +1336,9 @@ add_newdoc('numpy.core.multiarray', 'arange', See Also -------- - linspace : Evenly spaced numbers with careful handling of endpoints. - ogrid: Arrays of evenly spaced numbers in N-dimensions. - mgrid: Grid-shaped arrays of evenly spaced numbers in N-dimensions. + numpy.linspace : Evenly spaced numbers with careful handling of endpoints. + numpy.ogrid: Arrays of evenly spaced numbers in N-dimensions. + numpy.mgrid: Grid-shaped arrays of evenly spaced numbers in N-dimensions. Examples -------- @@ -1343,7 +1356,7 @@ add_newdoc('numpy.core.multiarray', 'arange', add_newdoc('numpy.core.multiarray', '_get_ndarray_c_version', """_get_ndarray_c_version() - Return the compile time NDARRAY_VERSION number. + Return the compile time NPY_VERSION (formerly called NDARRAY_VERSION) number. """) @@ -2736,6 +2749,8 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('byteswap', Toggle between low-endian and big-endian data representation by returning a byteswapped array, optionally swapped in-place. + Arrays of byte-strings are not swapped. The real and imaginary + parts of a complex number are swapped individually. Parameters ---------- @@ -2758,13 +2773,24 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('byteswap', >>> list(map(hex, A)) ['0x100', '0x1', '0x3322'] - Arrays of strings are not swapped + Arrays of byte-strings are not swapped - >>> A = np.array(['ceg', 'fac']) + >>> A = np.array([b'ceg', b'fac']) >>> A.byteswap() - Traceback (most recent call last): - ... - UnicodeDecodeError: ... + array([b'ceg', b'fac'], dtype='|S3') + + ``A.newbyteorder().byteswap()`` produces an array with the same values + but different representation in memory + + >>> A = np.array([1, 2, 3]) + >>> A.view(np.uint8) + array([1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, + 0, 0], dtype=uint8) + >>> A.newbyteorder().byteswap(inplace=True) + array([1, 2, 3]) + >>> A.view(np.uint8) + array([0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, + 0, 3], dtype=uint8) """)) @@ -3690,10 +3716,10 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('sort', See Also -------- numpy.sort : Return a sorted copy of an array. - argsort : Indirect sort. - lexsort : Indirect stable sort on multiple keys. - searchsorted : Find elements in sorted array. - partition: Partial sort. + numpy.argsort : Indirect sort. + numpy.lexsort : Indirect stable sort on multiple keys. + numpy.searchsorted : Find elements in sorted array. + numpy.partition: Partial sort. Notes ----- @@ -3927,15 +3953,22 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('tolist', Examples -------- - For a 1D array, ``a.tolist()`` is almost the same as ``list(a)``: + For a 1D array, ``a.tolist()`` is almost the same as ``list(a)``, + except that ``tolist`` changes numpy scalars to Python scalars: - >>> a = np.array([1, 2]) - >>> list(a) + >>> a = np.uint32([1, 2]) + >>> a_list = list(a) + >>> a_list [1, 2] - >>> a.tolist() + >>> type(a_list[0]) + <class 'numpy.uint32'> + >>> a_tolist = a.tolist() + >>> a_tolist [1, 2] + >>> type(a_tolist[0]) + <class 'int'> - However, for a 2D array, ``tolist`` applies recursively: + Additionally, for a 2D array, ``tolist`` applies recursively: >>> a = np.array([[1, 2], [3, 4]]) >>> list(a) @@ -4220,7 +4253,7 @@ add_newdoc('numpy.core.umath', 'frompyfunc', See Also -------- - vectorize : evaluates pyfunc over input arrays using broadcasting rules of numpy + vectorize : Evaluates pyfunc over input arrays using broadcasting rules of numpy. Notes ----- @@ -4481,7 +4514,7 @@ add_newdoc('numpy.core', 'ufunc', Alternate array object(s) in which to put the result; if provided, it must have a shape that the inputs broadcast to. A tuple of arrays (possible only as a keyword argument) must have length equal to the - number of outputs; use `None` for uninitialized outputs to be + number of outputs; use None for uninitialized outputs to be allocated by the ufunc. where : array_like, optional This condition is broadcast over the input. At locations where the @@ -4675,7 +4708,7 @@ add_newdoc('numpy.core', 'ufunc', ('signature', ----- Generalized ufuncs are used internally in many linalg functions, and in the testing suite; the examples below are taken from these. - For ufuncs that operate on scalars, the signature is `None`, which is + For ufuncs that operate on scalars, the signature is None, which is equivalent to '()' for every argument. Examples @@ -4726,7 +4759,7 @@ add_newdoc('numpy.core', 'ufunc', ('reduce', .. versionadded:: 1.7.0 - If this is `None`, a reduction is performed over all the axes. + If this is None, a reduction is performed over all the axes. If this is a tuple of ints, a reduction is performed on multiple axes, instead of a single axis or all the axes as before. @@ -4739,7 +4772,7 @@ add_newdoc('numpy.core', 'ufunc', ('reduce', to the data-type of the output array if this is provided, or the data-type of the input array if no output array is provided. out : ndarray, None, or tuple of ndarray and None, optional - A location into which the result is stored. If not provided or `None`, + A location into which the result is stored. If not provided or None, a freshly-allocated array is returned. For consistency with ``ufunc.__call__``, if given as a keyword, this may be wrapped in a 1-element tuple. @@ -4856,7 +4889,7 @@ add_newdoc('numpy.core', 'ufunc', ('accumulate', to the data-type of the output array if such is provided, or the the data-type of the input array if no output array is provided. out : ndarray, None, or tuple of ndarray and None, optional - A location into which the result is stored. If not provided or `None`, + A location into which the result is stored. If not provided or None, a freshly-allocated array is returned. For consistency with ``ufunc.__call__``, if given as a keyword, this may be wrapped in a 1-element tuple. @@ -4938,7 +4971,7 @@ add_newdoc('numpy.core', 'ufunc', ('reduceat', to the data type of the output array if this is provided, or the data type of the input array if no output array is provided. out : ndarray, None, or tuple of ndarray and None, optional - A location into which the result is stored. If not provided or `None`, + A location into which the result is stored. If not provided or None, a freshly-allocated array is returned. For consistency with ``ufunc.__call__``, if given as a keyword, this may be wrapped in a 1-element tuple. @@ -5311,7 +5344,8 @@ add_newdoc('numpy.core.multiarray', 'dtype', ('descr', `__array_interface__` attribute. Warning: This attribute exists specifically for `__array_interface__`, - and is not a datatype description compatible with `np.dtype`. + and passing it directly to `np.dtype` will not accurately reconstruct + some dtypes (e.g., scalar and subarray dtypes). Examples -------- @@ -6837,3 +6871,4 @@ for float_name in ('half', 'single', 'double', 'longdouble'): >>> np.{ftype}(-.25).as_integer_ratio() (-1, 4) """.format(ftype=float_name))) + diff --git a/numpy/core/_aliased_types.py b/numpy/core/_aliased_types.py deleted file mode 100644 index e69de29bb..000000000 --- a/numpy/core/_aliased_types.py +++ /dev/null diff --git a/numpy/core/_dtype.py b/numpy/core/_dtype.py index 3a12c8fad..df1ff180e 100644 --- a/numpy/core/_dtype.py +++ b/numpy/core/_dtype.py @@ -252,7 +252,7 @@ def _is_packed(dtype): from a list of the field names and dtypes with no additional dtype parameters. - Duplicates the C `is_dtype_struct_simple_unaligned_layout` functio. + Duplicates the C `is_dtype_struct_simple_unaligned_layout` function. """ total_offset = 0 for name in dtype.names: @@ -316,26 +316,39 @@ def _subarray_str(dtype): ) +def _name_includes_bit_suffix(dtype): + if dtype.type == np.object_: + # pointer size varies by system, best to omit it + return False + elif dtype.type == np.bool_: + # implied + return False + elif np.issubdtype(dtype, np.flexible) and _isunsized(dtype): + # unspecified + return False + else: + return True + + def _name_get(dtype): - # provides dtype.name.__get__ + # provides dtype.name.__get__, documented as returning a "bit name" if dtype.isbuiltin == 2: # user dtypes don't promise to do anything special return dtype.type.__name__ - # Builtin classes are documented as returning a "bit name" - name = dtype.type.__name__ - - # handle bool_, str_, etc - if name[-1] == '_': - name = name[:-1] + if issubclass(dtype.type, np.void): + # historically, void subclasses preserve their name, eg `record64` + name = dtype.type.__name__ + else: + name = _kind_name(dtype) - # append bit counts to str, unicode, and void - if np.issubdtype(dtype, np.flexible) and not _isunsized(dtype): + # append bit counts + if _name_includes_bit_suffix(dtype): name += "{}".format(dtype.itemsize * 8) # append metadata to datetimes - elif dtype.type in (np.datetime64, np.timedelta64): + if dtype.type in (np.datetime64, np.timedelta64): name += _datetime_metadata_str(dtype) return name diff --git a/numpy/core/_exceptions.py b/numpy/core/_exceptions.py index b3805af04..88a45561f 100644 --- a/numpy/core/_exceptions.py +++ b/numpy/core/_exceptions.py @@ -147,6 +147,54 @@ class _ArrayMemoryError(MemoryError): self.shape = shape self.dtype = dtype - def __str__(self): - return "Unable to allocate array with shape {} and data type {}".format(self.shape, self.dtype) + @property + def _total_size(self): + num_bytes = self.dtype.itemsize + for dim in self.shape: + num_bytes *= dim + return num_bytes + + @staticmethod + def _size_to_string(num_bytes): + """ Convert a number of bytes into a binary size string """ + import math + + # https://en.wikipedia.org/wiki/Binary_prefix + LOG2_STEP = 10 + STEP = 1024 + units = ['bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB'] + + unit_i = max(num_bytes.bit_length() - 1, 1) // LOG2_STEP + unit_val = 1 << (unit_i * LOG2_STEP) + n_units = num_bytes / unit_val + del unit_val + + # ensure we pick a unit that is correct after rounding + if round(n_units) == STEP: + unit_i += 1 + n_units /= STEP + + # deal with sizes so large that we don't have units for them + if unit_i >= len(units): + new_unit_i = len(units) - 1 + n_units *= 1 << ((unit_i - new_unit_i) * LOG2_STEP) + unit_i = new_unit_i + + unit_name = units[unit_i] + # format with a sensible number of digits + if unit_i == 0: + # no decimal point on bytes + return '{:.0f} {}'.format(n_units, unit_name) + elif round(n_units) < 1000: + # 3 significant figures, if none are dropped to the left of the . + return '{:#.3g} {}'.format(n_units, unit_name) + else: + # just give all the digits otherwise + return '{:#.0f} {}'.format(n_units, unit_name) + def __str__(self): + size_str = self._size_to_string(self._total_size) + return ( + "Unable to allocate {} for an array with shape {} and data type {}" + .format(size_str, self.shape, self.dtype) + ) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 5fd643505..05e401e0b 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -313,7 +313,7 @@ class _ctypes(object): crashing. User Beware! The value of this attribute is exactly the same as ``self._array_interface_['data'][0]``. - Note that unlike `data_as`, a reference will not be kept to the array: + Note that unlike ``data_as``, a reference will not be kept to the array: code like ``ctypes.c_void_p((a + b).ctypes.data)`` will result in a pointer to a deallocated array, and should be spelt ``(a + b).ctypes.data_as(ctypes.c_void_p)`` diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 8251beae8..401018015 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -89,8 +89,10 @@ def _make_options_dict(precision=None, threshold=None, edgeitems=None, "`False`", stacklevel=3) if threshold is not None: # forbid the bad threshold arg suggested by stack overflow, gh-12351 - if not isinstance(threshold, numbers.Number) or np.isnan(threshold): - raise ValueError("threshold must be numeric and non-NAN, try " + if not isinstance(threshold, numbers.Number): + raise TypeError("threshold must be numeric") + if np.isnan(threshold): + raise ValueError("threshold must be non-NAN, try " "sys.maxsize for untruncated representation") return options @@ -109,11 +111,12 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, ---------- precision : int or None, optional Number of digits of precision for floating point output (default 8). - May be `None` if `floatmode` is not `fixed`, to print as many digits as + May be None if `floatmode` is not `fixed`, to print as many digits as necessary to uniquely specify the value. threshold : int, optional Total number of array elements which trigger summarization rather than full repr (default 1000). + To always use the full repr without summarization, pass `sys.maxsize`. edgeitems : int, optional Number of array items in summary at beginning and end of each dimension (default 3). @@ -191,12 +194,14 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, See Also -------- - get_printoptions, set_string_function, array2string + get_printoptions, printoptions, set_string_function, array2string Notes ----- `formatter` is always reset with a call to `set_printoptions`. + Use `printoptions` as a context manager to set the values temporarily. + Examples -------- Floating point precision can be set: @@ -233,9 +238,16 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, To put back the default options, you can use: - >>> np.set_printoptions(edgeitems=3,infstr='inf', + >>> np.set_printoptions(edgeitems=3, infstr='inf', ... linewidth=75, nanstr='nan', precision=8, ... suppress=False, threshold=1000, formatter=None) + + Also to temporarily override options, use `printoptions` as a context manager: + + >>> with np.printoptions(precision=2, suppress=True, threshold=5): + ... np.linspace(0, 10, 10) + array([ 0. , 1.11, 2.22, ..., 7.78, 8.89, 10. ]) + """ legacy = kwarg.pop('legacy', None) if kwarg: @@ -282,7 +294,7 @@ def get_printoptions(): See Also -------- - set_printoptions, set_string_function + set_printoptions, printoptions, set_string_function """ return _format_options.copy() @@ -981,20 +993,6 @@ class FloatingFormat(object): pad_left=self.pad_left, pad_right=self.pad_right) -# for back-compatibility, we keep the classes for each float type too -class FloatFormat(FloatingFormat): - def __init__(self, *args, **kwargs): - warnings.warn("FloatFormat has been replaced by FloatingFormat", - DeprecationWarning, stacklevel=2) - super(FloatFormat, self).__init__(*args, **kwargs) - - -class LongFloatFormat(FloatingFormat): - def __init__(self, *args, **kwargs): - warnings.warn("LongFloatFormat has been replaced by FloatingFormat", - DeprecationWarning, stacklevel=2) - super(LongFloatFormat, self).__init__(*args, **kwargs) - @set_module('numpy') def format_float_scientific(x, precision=None, unique=True, trim='k', @@ -1193,21 +1191,6 @@ class ComplexFloatingFormat(object): return r + i -# for back-compatibility, we keep the classes for each complex type too -class ComplexFormat(ComplexFloatingFormat): - def __init__(self, *args, **kwargs): - warnings.warn( - "ComplexFormat has been replaced by ComplexFloatingFormat", - DeprecationWarning, stacklevel=2) - super(ComplexFormat, self).__init__(*args, **kwargs) - -class LongComplexFormat(ComplexFloatingFormat): - def __init__(self, *args, **kwargs): - warnings.warn( - "LongComplexFormat has been replaced by ComplexFloatingFormat", - DeprecationWarning, stacklevel=2) - super(LongComplexFormat, self).__init__(*args, **kwargs) - class _TimelikeFormat(object): def __init__(self, data): @@ -1318,16 +1301,6 @@ class StructuredVoidFormat(object): return "({})".format(", ".join(str_fields)) -# for backwards compatibility -class StructureFormat(StructuredVoidFormat): - def __init__(self, *args, **kwargs): - # NumPy 1.14, 2018-02-14 - warnings.warn( - "StructureFormat has been replaced by StructuredVoidFormat", - DeprecationWarning, stacklevel=2) - super(StructureFormat, self).__init__(*args, **kwargs) - - def _void_scalar_repr(x): """ Implements the repr for structured-void scalars. It is called from the @@ -1506,7 +1479,11 @@ def array_repr(arr, max_line_width=None, precision=None, suppress_small=None): arr, max_line_width, precision, suppress_small) -_guarded_str = _recursive_guard()(str) +@_recursive_guard() +def _guarded_repr_or_str(v): + if isinstance(v, bytes): + return repr(v) + return str(v) def _array_str_implementation( @@ -1524,7 +1501,7 @@ def _array_str_implementation( # obtain a scalar and call str on it, avoiding problems for subclasses # for which indexing with () returns a 0d instead of a scalar by using # ndarray's getindex. Also guard against recursive 0d object arrays. - return _guarded_str(np.ndarray.__getitem__(a, ())) + return _guarded_repr_or_str(np.ndarray.__getitem__(a, ())) return array2string(a, max_line_width, precision, suppress_small, ' ', "") @@ -1641,5 +1618,5 @@ def set_string_function(f, repr=True): else: return multiarray.set_string_function(f, repr) -set_string_function(_default_array_str, 0) -set_string_function(_default_array_repr, 1) +set_string_function(_default_array_str, False) +set_string_function(_default_array_repr, True) diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index 00f10df57..72d2af8b9 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -47,4 +47,6 @@ # Deprecate PyArray_SetNumericOps and PyArray_GetNumericOps, # Add fields core_dim_flags and core_dim_sizes to PyUFuncObject. # Add PyUFunc_FromFuncAndDataAndSignatureAndIdentity to ufunc_funcs_api. +# Version 13 (NumPy 1.17) No change. +# Version 13 (NumPy 1.18) No change. 0x0000000d = 5b0e8bbded00b166125974fc71e80a33 diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 923c34425..22afa0320 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -8,8 +8,11 @@ specified. """ from __future__ import division, absolute_import, print_function +from numpy.distutils.conv_template import process_file as process_c_file + import sys, os, re import hashlib +import io import textwrap @@ -215,7 +218,10 @@ def find_functions(filename, tag='API'): This function does foo... */ """ - fo = open(filename, 'r') + if filename.endswith(('.c.src', '.h.src')): + fo = io.StringIO(process_c_file(filename)) + else: + fo = open(filename, 'r') functions = [] return_type = None function_name = None @@ -259,7 +265,8 @@ def find_functions(filename, tag='API'): elif state == STATE_ARGS: if line.startswith('{'): # finished - fargs_str = ' '.join(function_args).rstrip(' )') + # remove any white space and the closing bracket: + fargs_str = ' '.join(function_args).rstrip()[:-1].rstrip() fargs = split_arguments(fargs_str) f = Function(function_name, return_type, fargs, '\n'.join(doclist)) diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py index bf1747272..6d76f7ca2 100644 --- a/numpy/core/code_generators/generate_umath.py +++ b/numpy/core/code_generators/generate_umath.py @@ -226,7 +226,9 @@ chartoname = { 'P': 'OBJECT', } -all = '?bBhHiIlLqQefdgFDGOMm' +noobj = '?bBhHiIlLqQefdgFDGmM' +all = '?bBhHiIlLqQefdgFDGOmM' + O = 'O' P = 'P' ints = 'bBhHiIlLqQ' @@ -246,10 +248,8 @@ inexactvec = 'fd' noint = inexact+O nointP = inexact+P allP = bints+times+flts+cmplxP -nobool = all[1:] -noobj = all[:-3]+all[-2:] -nobool_or_obj = all[1:-3]+all[-2:] -nobool_or_datetime = all[1:-2]+all[-1:] +nobool_or_obj = noobj[1:] +nobool_or_datetime = noobj[1:-1] + O # includes m - timedelta64 intflt = ints+flts intfltcmplx = ints+flts+cmplx nocmplx = bints+times+flts @@ -287,7 +287,7 @@ defdict = { Ufunc(2, 1, None, # Zero is only a unit to the right, not the left docstrings.get('numpy.core.umath.subtract'), 'PyUFunc_SubtractionTypeResolver', - TD(notimes_or_obj, simd=[('avx2', ints)]), + TD(ints + inexact, simd=[('avx2', ints)]), [TypeDescription('M', FullTypeDescr, 'Mm', 'M'), TypeDescription('m', FullTypeDescr, 'mm', 'm'), TypeDescription('M', FullTypeDescr, 'MM', 'm'), @@ -358,14 +358,14 @@ defdict = { Ufunc(1, 1, None, docstrings.get('numpy.core.umath.square'), None, - TD(ints+inexact, simd=[('avx2', ints)]), + TD(ints+inexact, simd=[('avx2', ints), ('fma', 'fd'), ('avx512f', 'fd')]), TD(O, f='Py_square'), ), 'reciprocal': Ufunc(1, 1, None, docstrings.get('numpy.core.umath.reciprocal'), None, - TD(ints+inexact, simd=[('avx2', ints)]), + TD(ints+inexact, simd=[('avx2', ints), ('fma', 'fd'), ('avx512f','fd')]), TD(O, f='Py_reciprocal'), ), # This is no longer used as numpy.ones_like, however it is @@ -395,7 +395,7 @@ defdict = { Ufunc(1, 1, None, docstrings.get('numpy.core.umath.absolute'), 'PyUFunc_AbsoluteTypeResolver', - TD(bints+flts+timedeltaonly), + TD(bints+flts+timedeltaonly, simd=[('fma', 'fd'), ('avx512f', 'fd')]), TD(cmplx, out=('f', 'd', 'g')), TD(O, f='PyNumber_Absolute'), ), @@ -409,7 +409,7 @@ defdict = { Ufunc(1, 1, None, docstrings.get('numpy.core.umath.negative'), 'PyUFunc_NegativeTypeResolver', - TD(bints+flts+timedeltaonly, simd=[('avx2', ints)]), + TD(ints+flts+timedeltaonly, simd=[('avx2', ints)]), TD(cmplx, f='neg'), TD(O, f='PyNumber_Negative'), ), @@ -433,6 +433,7 @@ defdict = { 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(all, out='?', simd=[('avx2', ints)]), [TypeDescription('O', FullTypeDescr, 'OO', 'O')], + TD('O', out='?'), ), 'greater_equal': Ufunc(2, 1, None, @@ -440,6 +441,7 @@ defdict = { 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(all, out='?', simd=[('avx2', ints)]), [TypeDescription('O', FullTypeDescr, 'OO', 'O')], + TD('O', out='?'), ), 'less': Ufunc(2, 1, None, @@ -447,6 +449,7 @@ defdict = { 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(all, out='?', simd=[('avx2', ints)]), [TypeDescription('O', FullTypeDescr, 'OO', 'O')], + TD('O', out='?'), ), 'less_equal': Ufunc(2, 1, None, @@ -454,6 +457,7 @@ defdict = { 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(all, out='?', simd=[('avx2', ints)]), [TypeDescription('O', FullTypeDescr, 'OO', 'O')], + TD('O', out='?'), ), 'equal': Ufunc(2, 1, None, @@ -461,6 +465,7 @@ defdict = { 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(all, out='?', simd=[('avx2', ints)]), [TypeDescription('O', FullTypeDescr, 'OO', 'O')], + TD('O', out='?'), ), 'not_equal': Ufunc(2, 1, None, @@ -468,6 +473,7 @@ defdict = { 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(all, out='?', simd=[('avx2', ints)]), [TypeDescription('O', FullTypeDescr, 'OO', 'O')], + TD('O', out='?'), ), 'logical_and': Ufunc(2, 1, True_, @@ -475,6 +481,7 @@ defdict = { 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(nodatetime_or_obj, out='?', simd=[('avx2', ints)]), TD(O, f='npy_ObjectLogicalAnd'), + TD(O, f='npy_ObjectLogicalAnd', out='?'), ), 'logical_not': Ufunc(1, 1, None, @@ -482,6 +489,7 @@ defdict = { None, TD(nodatetime_or_obj, out='?', simd=[('avx2', ints)]), TD(O, f='npy_ObjectLogicalNot'), + TD(O, f='npy_ObjectLogicalNot', out='?'), ), 'logical_or': Ufunc(2, 1, False_, @@ -489,6 +497,7 @@ defdict = { 'PyUFunc_SimpleBinaryComparisonTypeResolver', TD(nodatetime_or_obj, out='?', simd=[('avx2', ints)]), TD(O, f='npy_ObjectLogicalOr'), + TD(O, f='npy_ObjectLogicalOr', out='?'), ), 'logical_xor': Ufunc(2, 1, False_, @@ -662,14 +671,18 @@ defdict = { Ufunc(1, 1, None, docstrings.get('numpy.core.umath.cos'), None, - TD(inexact, f='cos', astype={'e':'f'}), + TD('e', f='cos', astype={'e':'f'}), + TD('f', simd=[('fma', 'f'), ('avx512f', 'f')]), + TD('fdg' + cmplx, f='cos'), TD(P, f='cos'), ), 'sin': Ufunc(1, 1, None, docstrings.get('numpy.core.umath.sin'), None, - TD(inexact, f='sin', astype={'e':'f'}), + TD('e', f='sin', astype={'e':'f'}), + TD('f', simd=[('fma', 'f'), ('avx512f', 'f')]), + TD('fdg' + cmplx, f='sin'), TD(P, f='sin'), ), 'tan': @@ -705,8 +718,8 @@ defdict = { docstrings.get('numpy.core.umath.exp'), None, TD('e', f='exp', astype={'e':'f'}), - TD('f', simd=[('avx2', 'f'), ('avx512f', 'f')]), - TD(inexact, f='exp', astype={'e':'f'}), + TD('f', simd=[('fma', 'f'), ('avx512f', 'f')]), + TD('fdg' + cmplx, f='exp'), TD(P, f='exp'), ), 'exp2': @@ -728,8 +741,8 @@ defdict = { docstrings.get('numpy.core.umath.log'), None, TD('e', f='log', astype={'e':'f'}), - TD('f', simd=[('avx2', 'f'), ('avx512f', 'f')]), - TD(inexact, f='log', astype={'e':'f'}), + TD('f', simd=[('fma', 'f'), ('avx512f', 'f')]), + TD('fdg' + cmplx, f='log'), TD(P, f='log'), ), 'log2': @@ -758,8 +771,8 @@ defdict = { docstrings.get('numpy.core.umath.sqrt'), None, TD('e', f='sqrt', astype={'e':'f'}), - TD(inexactvec), - TD(inexact, f='sqrt', astype={'e':'f'}), + TD(inexactvec, simd=[('fma', 'fd'), ('avx512f', 'fd')]), + TD('fdg' + cmplx, f='sqrt'), TD(P, f='sqrt'), ), 'cbrt': @@ -773,14 +786,18 @@ defdict = { Ufunc(1, 1, None, docstrings.get('numpy.core.umath.ceil'), None, - TD(flts, f='ceil', astype={'e':'f'}), + TD('e', f='ceil', astype={'e':'f'}), + TD(inexactvec, simd=[('fma', 'fd'), ('avx512f', 'fd')]), + TD('fdg', f='ceil'), TD(O, f='npy_ObjectCeil'), ), 'trunc': Ufunc(1, 1, None, docstrings.get('numpy.core.umath.trunc'), None, - TD(flts, f='trunc', astype={'e':'f'}), + TD('e', f='trunc', astype={'e':'f'}), + TD(inexactvec, simd=[('fma', 'fd'), ('avx512f', 'fd')]), + TD('fdg', f='trunc'), TD(O, f='npy_ObjectTrunc'), ), 'fabs': @@ -794,14 +811,18 @@ defdict = { Ufunc(1, 1, None, docstrings.get('numpy.core.umath.floor'), None, - TD(flts, f='floor', astype={'e':'f'}), + TD('e', f='floor', astype={'e':'f'}), + TD(inexactvec, simd=[('fma', 'fd'), ('avx512f', 'fd')]), + TD('fdg', f='floor'), TD(O, f='npy_ObjectFloor'), ), 'rint': Ufunc(1, 1, None, docstrings.get('numpy.core.umath.rint'), None, - TD(inexact, f='rint', astype={'e':'f'}), + TD('e', f='rint', astype={'e':'f'}), + TD(inexactvec, simd=[('fma', 'fd'), ('avx512f', 'fd')]), + TD('fdg' + cmplx, f='rint'), TD(P, f='rint'), ), 'arctan2': @@ -837,8 +858,8 @@ defdict = { 'isnan': Ufunc(1, 1, None, docstrings.get('numpy.core.umath.isnan'), - None, - TD(nodatetime_or_obj, out='?'), + 'PyUFunc_IsFiniteTypeResolver', + TD(noobj, out='?'), ), 'isnat': Ufunc(1, 1, None, @@ -849,8 +870,8 @@ defdict = { 'isinf': Ufunc(1, 1, None, docstrings.get('numpy.core.umath.isinf'), - None, - TD(nodatetime_or_obj, out='?'), + 'PyUFunc_IsFiniteTypeResolver', + TD(noobj, out='?'), ), 'isfinite': Ufunc(1, 1, None, diff --git a/numpy/core/code_generators/ufunc_docstrings.py b/numpy/core/code_generators/ufunc_docstrings.py index fb418aadc..4dec73505 100644 --- a/numpy/core/code_generators/ufunc_docstrings.py +++ b/numpy/core/code_generators/ufunc_docstrings.py @@ -22,7 +22,7 @@ subst = { 'PARAMS': textwrap.dedent(""" out : ndarray, None, or tuple of ndarray and None, optional A location into which the result is stored. If provided, it must have - a shape that the inputs broadcast to. If not provided or `None`, + a shape that the inputs broadcast to. If not provided or None, a freshly-allocated array is returned. A tuple (possible only as a keyword argument) must have length equal to the number of outputs. where : array_like, optional @@ -2183,7 +2183,7 @@ add_newdoc('numpy.core.umath', 'logical_and', Returns ------- y : ndarray or bool - Boolean result of the logical OR operation applied to the elements + Boolean result of the logical AND operation applied to the elements of `x1` and `x2`; the shape is determined by broadcasting. $OUT_SCALAR_2 @@ -2596,7 +2596,7 @@ add_newdoc('numpy.core.umath', 'matmul', out : ndarray, optional A location into which the result is stored. If provided, it must have a shape that matches the signature `(n,k),(k,m)->(n,m)`. If not - provided or `None`, a freshly-allocated array is returned. + provided or None, a freshly-allocated array is returned. **kwargs For other keyword-only arguments, see the :ref:`ufunc docs <ufuncs.kwargs>`. diff --git a/numpy/core/defchararray.py b/numpy/core/defchararray.py index d7ecce1b4..2d89d6fe0 100644 --- a/numpy/core/defchararray.py +++ b/numpy/core/defchararray.py @@ -82,7 +82,7 @@ def _clean_args(*args): Many of the Python string operations that have optional arguments do not use 'None' to indicate a default value. In these cases, - we need to remove all `None` arguments, and those following them. + we need to remove all None arguments, and those following them. """ newargs = [] for chk in args: @@ -1333,7 +1333,7 @@ def rsplit(a, sep=None, maxsplit=None): a : array_like of str or unicode sep : str or unicode, optional - If `sep` is not specified or `None`, any whitespace string + If `sep` is not specified or None, any whitespace string is a separator. maxsplit : int, optional If `maxsplit` is given, at most `maxsplit` splits are done, @@ -1417,7 +1417,7 @@ def split(a, sep=None, maxsplit=None): a : array_like of str or unicode sep : str or unicode, optional - If `sep` is not specified or `None`, any whitespace string is a + If `sep` is not specified or None, any whitespace string is a separator. maxsplit : int, optional @@ -1840,7 +1840,7 @@ class chararray(ndarray): This constructor creates the array, using `buffer` (with `offset` and `strides`) if it is not ``None``. If `buffer` is ``None``, then constructs a new array with `strides` in "C order", unless both - ``len(shape) >= 2`` and ``order='Fortran'``, in which case `strides` + ``len(shape) >= 2`` and ``order='F'``, in which case `strides` is in "Fortran order". Methods @@ -2659,7 +2659,7 @@ def array(obj, itemsize=None, copy=True, unicode=None, order=None): unicode : bool, optional When true, the resulting `chararray` can contain Unicode characters, when false only 8-bit characters. If unicode is - `None` and `obj` is one of the following: + None and `obj` is one of the following: - a `chararray`, - an ndarray of type `str` or `unicode` @@ -2799,7 +2799,7 @@ def asarray(obj, itemsize=None, unicode=None, order=None): unicode : bool, optional When true, the resulting `chararray` can contain Unicode characters, when false only 8-bit characters. If unicode is - `None` and `obj` is one of the following: + None and `obj` is one of the following: - a `chararray`, - an ndarray of type `str` or 'unicode` diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 08f17aae4..d454480a8 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -25,7 +25,7 @@ __all__ = [ 'argmin', 'argpartition', 'argsort', 'around', 'choose', 'clip', 'compress', 'cumprod', 'cumproduct', 'cumsum', 'diagonal', 'mean', 'ndim', 'nonzero', 'partition', 'prod', 'product', 'ptp', 'put', - 'rank', 'ravel', 'repeat', 'reshape', 'resize', 'round_', + 'ravel', 'repeat', 'reshape', 'resize', 'round_', 'searchsorted', 'shape', 'size', 'sometrue', 'sort', 'squeeze', 'std', 'sum', 'swapaxes', 'take', 'trace', 'transpose', 'var', ] @@ -380,6 +380,7 @@ def choose(a, choices, out=None, mode='raise'): See Also -------- ndarray.choose : equivalent method + numpy.take_along_axis : Preferable if `choices` is an array Notes ----- @@ -795,7 +796,9 @@ def argpartition(a, kth, axis=-1, kind='introselect', order=None): -------- partition : Describes partition algorithms used. ndarray.partition : Inplace partition. - argsort : Full indirect sort + argsort : Full indirect sort. + take_along_axis : Apply ``index_array`` from argpartition + to an array as if by calling partition. Notes ----- @@ -815,6 +818,14 @@ def argpartition(a, kth, axis=-1, kind='introselect', order=None): >>> np.array(x)[np.argpartition(x, 3)] array([2, 1, 3, 4]) + Multi-dimensional array: + + >>> x = np.array([[3, 4, 2], [1, 3, 1]]) + >>> index_array = np.argpartition(x, kth=1, axis=-1) + >>> np.take_along_axis(x, index_array, axis=-1) # same as np.partition(x, kth=1) + array([[2, 3, 4], + [1, 1, 3]]) + """ return _wrapfunc(a, 'argpartition', kth, axis=axis, kind=kind, order=order) @@ -908,14 +919,18 @@ def sort(a, axis=-1, kind=None, order=None): .. versionadded:: 1.12.0 - quicksort has been changed to an introsort which will switch - heapsort when it does not make enough progress. This makes its - worst case O(n*log(n)). - - 'stable' automatically choses the best stable sorting algorithm - for the data type being sorted. It, along with 'mergesort' is - currently mapped to timsort or radix sort depending on the - data type. API forward compatibility currently limits the + quicksort has been changed to `introsort <https://en.wikipedia.org/wiki/Introsort>`_. + When sorting does not make enough progress it switches to + `heapsort <https://en.wikipedia.org/wiki/Heapsort>`_. + This implementation makes quicksort O(n*log(n)) in the worst case. + + 'stable' automatically chooses the best stable sorting algorithm + for the data type being sorted. + It, along with 'mergesort' is currently mapped to + `timsort <https://en.wikipedia.org/wiki/Timsort>`_ + or `radix sort <https://en.wikipedia.org/wiki/Radix_sort>`_ + depending on the data type. + API forward compatibility currently limits the ability to select the implementation and it is hardwired for the different data types. @@ -924,11 +939,15 @@ def sort(a, axis=-1, kind=None, order=None): Timsort is added for better performance on already or nearly sorted data. On random data timsort is almost identical to mergesort. It is now used for stable sort while quicksort is still the - default sort if none is chosen. For details of timsort, refer to + default sort if none is chosen. For timsort details, refer to `CPython listsort.txt <https://github.com/python/cpython/blob/3.7/Objects/listsort.txt>`_. 'mergesort' and 'stable' are mapped to radix sort for integer data types. Radix sort is an O(n) sort instead of O(n log n). + .. versionchanged:: 1.17.0 + + NaT now sorts to the end of arrays for consistency with NaN. + Examples -------- >>> a = np.array([[1,4],[3,1]]) @@ -1020,6 +1039,8 @@ def argsort(a, axis=-1, kind=None, order=None): lexsort : Indirect stable sort with multiple keys. ndarray.sort : Inplace sort. argpartition : Indirect partial sort. + take_along_axis : Apply ``index_array`` from argsort + to an array as if by calling sort. Notes ----- @@ -1115,6 +1136,8 @@ def argmax(a, axis=None, out=None): ndarray.argmax, argmin amax : The maximum value along a given axis. unravel_index : Convert a flat index into an index tuple. + take_along_axis : Apply ``np.expand_dims(index_array, axis)`` + from argmax to an array as if by calling max. Notes ----- @@ -1149,6 +1172,16 @@ def argmax(a, axis=None, out=None): >>> np.argmax(b) # Only the first occurrence is returned. 1 + >>> x = np.array([[4,2,3], [1,0,3]]) + >>> index_array = np.argmax(x, axis=-1) + >>> # Same as np.max(x, axis=-1, keepdims=True) + >>> np.take_along_axis(x, np.expand_dims(index_array, axis=-1), axis=-1) + array([[4], + [3]]) + >>> # Same as np.max(x, axis=-1) + >>> np.take_along_axis(x, np.expand_dims(index_array, axis=-1), axis=-1).squeeze(axis=-1) + array([4, 3]) + """ return _wrapfunc(a, 'argmax', axis=axis, out=out) @@ -1184,6 +1217,8 @@ def argmin(a, axis=None, out=None): ndarray.argmin, argmax amin : The minimum value along a given axis. unravel_index : Convert a flat index into an index tuple. + take_along_axis : Apply ``np.expand_dims(index_array, axis)`` + from argmin to an array as if by calling min. Notes ----- @@ -1218,6 +1253,16 @@ def argmin(a, axis=None, out=None): >>> np.argmin(b) # Only the first occurrence is returned. 0 + >>> x = np.array([[4,2,3], [1,0,3]]) + >>> index_array = np.argmin(x, axis=-1) + >>> # Same as np.min(x, axis=-1, keepdims=True) + >>> np.take_along_axis(x, np.expand_dims(index_array, axis=-1), axis=-1) + array([[2], + [0]]) + >>> # Same as np.max(x, axis=-1) + >>> np.take_along_axis(x, np.expand_dims(index_array, axis=-1), axis=-1).squeeze(axis=-1) + array([2, 0]) + """ return _wrapfunc(a, 'argmin', axis=axis, out=out) @@ -1404,7 +1449,7 @@ def squeeze(a, axis=None): Raises ------ ValueError - If `axis` is not `None`, and an axis being squeezed is not of length 1 + If `axis` is not None, and an axis being squeezed is not of length 1 See Also -------- @@ -1770,11 +1815,13 @@ def nonzero(a): which returns a row for each non-zero element. .. note:: - When called on a zero-d array or scalar, ``nonzero(a)`` is treated - as ``nonzero(atleast1d(a))``. - ..deprecated:: 1.17.0 - Use `atleast1d` explicitly if this behavior is deliberate. + When called on a zero-d array or scalar, ``nonzero(a)`` is treated + as ``nonzero(atleast1d(a))``. + + .. deprecated:: 1.17.0 + + Use `atleast1d` explicitly if this behavior is deliberate. Parameters ---------- @@ -1938,7 +1985,7 @@ def compress(condition, a, axis=None, out=None): take, choose, diag, diagonal, select ndarray.compress : Equivalent method in ndarray np.extract: Equivalent method when working on 1-D arrays - numpy.doc.ufuncs : Section "Output arguments" + ufuncs-output-type Examples -------- @@ -1988,14 +2035,14 @@ def clip(a, a_min, a_max, out=None, **kwargs): ---------- a : array_like Array containing elements to clip. - a_min : scalar or array_like or `None` - Minimum value. If `None`, clipping is not performed on lower + a_min : scalar or array_like or None + Minimum value. If None, clipping is not performed on lower interval edge. Not more than one of `a_min` and `a_max` may be - `None`. - a_max : scalar or array_like or `None` - Maximum value. If `None`, clipping is not performed on upper + None. + a_max : scalar or array_like or None + Maximum value. If None, clipping is not performed on upper interval edge. Not more than one of `a_min` and `a_max` may be - `None`. If `a_min` or `a_max` are array_like, then the three + None. If `a_min` or `a_max` are array_like, then the three arrays will be broadcasted to match their shapes. out : ndarray, optional The results will be placed in this array. It may be the input @@ -2016,7 +2063,7 @@ def clip(a, a_min, a_max, out=None, **kwargs): See Also -------- - numpy.doc.ufuncs : Section "Output arguments" + ufuncs-output-type Examples -------- @@ -2199,7 +2246,7 @@ def any(a, axis=None, out=None, keepdims=np._NoValue): Input array or object that can be converted to an array. axis : None or int or tuple of ints, optional Axis or axes along which a logical OR reduction is performed. - The default (`axis` = `None`) is to perform a logical OR over all + The default (``axis=None``) is to perform a logical OR over all the dimensions of the input array. `axis` may be negative, in which case it counts from the last to the first axis. @@ -2212,7 +2259,7 @@ def any(a, axis=None, out=None, keepdims=np._NoValue): the same shape as the expected output and its type is preserved (e.g., if it is of type float, then it will remain so, returning 1.0 for True and 0.0 for False, regardless of the type of `a`). - See `doc.ufuncs` (Section "Output arguments") for details. + See `ufuncs-output-type` for more details. keepdims : bool, optional If this is set to True, the axes which are reduced are left @@ -2285,7 +2332,7 @@ def all(a, axis=None, out=None, keepdims=np._NoValue): Input array or object that can be converted to an array. axis : None or int or tuple of ints, optional Axis or axes along which a logical AND reduction is performed. - The default (`axis` = `None`) is to perform a logical AND over all + The default (``axis=None``) is to perform a logical AND over all the dimensions of the input array. `axis` may be negative, in which case it counts from the last to the first axis. @@ -2297,8 +2344,8 @@ def all(a, axis=None, out=None, keepdims=np._NoValue): Alternate output array in which to place the result. It must have the same shape as the expected output and its type is preserved (e.g., if ``dtype(out)`` is float, the result - will consist of 0.0's and 1.0's). See `doc.ufuncs` (Section - "Output arguments") for more details. + will consist of 0.0's and 1.0's). See `ufuncs-output-type` for more + details. keepdims : bool, optional If this is set to True, the axes which are reduced are left @@ -2376,8 +2423,8 @@ def cumsum(a, axis=None, dtype=None, out=None): out : ndarray, optional Alternative output array in which to place the result. It must have the same shape and buffer length as the expected output - but the type will be cast if necessary. See `doc.ufuncs` - (Section "Output arguments") for more details. + but the type will be cast if necessary. See `ufuncs-output-type` for + more details. Returns ------- @@ -2522,7 +2569,7 @@ def amax(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue, out : ndarray, optional Alternative output array in which to place the result. Must be of the same shape and buffer length as the expected output. - See `doc.ufuncs` (Section "Output arguments") for more details. + See `ufuncs-output-type` for more details. keepdims : bool, optional If this is set to True, the axes which are reduced are left @@ -2647,7 +2694,7 @@ def amin(a, axis=None, out=None, keepdims=np._NoValue, initial=np._NoValue, out : ndarray, optional Alternative output array in which to place the result. Must be of the same shape and buffer length as the expected output. - See `doc.ufuncs` (Section "Output arguments") for more details. + See `ufuncs-output-type` for more details. keepdims : bool, optional If this is set to True, the axes which are reduced are left @@ -2778,6 +2825,10 @@ def alen(a): 7 """ + # NumPy 1.18.0, 2019-08-02 + warnings.warn( + "`np.alen` is deprecated, use `len` instead", + DeprecationWarning, stacklevel=2) try: return len(a) except TypeError: @@ -2850,7 +2901,7 @@ def prod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue, See Also -------- ndarray.prod : equivalent method - numpy.doc.ufuncs : Section "Output arguments" + ufuncs-output-type Notes ----- @@ -2946,7 +2997,7 @@ def cumprod(a, axis=None, dtype=None, out=None): See Also -------- - numpy.doc.ufuncs : Section "Output arguments" + ufuncs-output-type Notes ----- @@ -3092,8 +3143,8 @@ def around(a, decimals=0, out=None): out : ndarray, optional Alternative output array in which to place the result. It must have the same shape as the expected output, but the type of the output - values will be cast if necessary. See `doc.ufuncs` (Section - "Output arguments") for details. + values will be cast if necessary. See `ufuncs-output-type` for more + details. Returns ------- @@ -3116,10 +3167,37 @@ def around(a, decimals=0, out=None): ----- For values exactly halfway between rounded decimal values, NumPy rounds to the nearest even value. Thus 1.5 and 2.5 round to 2.0, - -0.5 and 0.5 round to 0.0, etc. Results may also be surprising due - to the inexact representation of decimal fractions in the IEEE - floating point standard [1]_ and errors introduced when scaling - by powers of ten. + -0.5 and 0.5 round to 0.0, etc. + + ``np.around`` uses a fast but sometimes inexact algorithm to round + floating-point datatypes. For positive `decimals` it is equivalent to + ``np.true_divide(np.rint(a * 10**decimals), 10**decimals)``, which has + error due to the inexact representation of decimal fractions in the IEEE + floating point standard [1]_ and errors introduced when scaling by powers + of ten. For instance, note the extra "1" in the following: + + >>> np.round(56294995342131.5, 3) + 56294995342131.51 + + If your goal is to print such values with a fixed number of decimals, it is + preferable to use numpy's float printing routines to limit the number of + printed decimals: + + >>> np.format_float_positional(56294995342131.5, precision=3) + '56294995342131.5' + + The float printing routines use an accurate but much more computationally + demanding algorithm to compute the number of digits after the decimal + point. + + Alternatively, Python's builtin `round` function uses a more accurate + but slower algorithm for 64-bit floating point values: + + >>> round(56294995342131.5, 3) + 56294995342131.5 + >>> np.round(16.055, 2), round(16.055, 2) # equals 16.0549999999999997 + (16.06, 16.05) + References ---------- @@ -3180,7 +3258,7 @@ def mean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): Alternate output array in which to place the result. The default is ``None``; if provided, it must have the same shape as the expected output, but the type will be cast if necessary. - See `doc.ufuncs` for details. + See `ufuncs-output-type` for more details. keepdims : bool, optional If this is set to True, the axes which are reduced are left @@ -3315,7 +3393,7 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): See Also -------- var, mean, nanmean, nanstd, nanvar - numpy.doc.ufuncs : Section "Output arguments" + ufuncs-output-type Notes ----- @@ -3410,7 +3488,7 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): instead of a single axis or all the axes as before. dtype : data-type, optional Type to use in computing the variance. For arrays of integer type - the default is `float32`; for arrays of float types it is the same as + the default is `float64`; for arrays of float types it is the same as the array type. out : ndarray, optional Alternate output array in which to place the result. It must have @@ -3440,7 +3518,7 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): See Also -------- std, mean, nanmean, nanstd, nanvar - numpy.doc.ufuncs : Section "Output arguments" + ufuncs-output-type Notes ----- @@ -3569,30 +3647,3 @@ def alltrue(*args, **kwargs): numpy.all : Equivalent function; see for details. """ return all(*args, **kwargs) - - -@array_function_dispatch(_ndim_dispatcher) -def rank(a): - """ - Return the number of dimensions of an array. - - .. note:: - This function is deprecated in NumPy 1.9 to avoid confusion with - `numpy.linalg.matrix_rank`. The ``ndim`` attribute or function - should be used instead. - - See Also - -------- - ndim : equivalent non-deprecated function - - Notes - ----- - In the old Numeric package, `rank` was the term used for the number of - dimensions, but in NumPy `ndim` is used instead. - """ - # 2014-04-12, 1.9 - warnings.warn( - "`rank` is deprecated; use the `ndim` attribute or function instead. " - "To find the rank of a matrix see `numpy.linalg.matrix_rank`.", - VisibleDeprecationWarning, stacklevel=3) - return ndim(a) diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index c1067299d..538ac8b84 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -3,6 +3,7 @@ from __future__ import division, absolute_import, print_function import functools import warnings import operator +import types from . import numeric as _nx from .numeric import (result_type, NaN, shares_memory, MAY_SHARE_BOUNDS, @@ -17,18 +18,6 @@ array_function_dispatch = functools.partial( overrides.array_function_dispatch, module='numpy') -def _index_deprecate(i, stacklevel=2): - try: - i = operator.index(i) - except TypeError: - msg = ("object of type {} cannot be safely interpreted as " - "an integer.".format(type(i))) - i = int(i) - stacklevel += 1 - warnings.warn(msg, DeprecationWarning, stacklevel=stacklevel) - return i - - def _linspace_dispatcher(start, stop, num=None, endpoint=None, retstep=None, dtype=None, axis=None): return (start, stop) @@ -124,8 +113,13 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, >>> plt.show() """ - # 2016-02-25, 1.12 - num = _index_deprecate(num) + try: + num = operator.index(num) + except TypeError: + raise TypeError( + "object of type {} cannot be safely interpreted as an integer." + .format(type(num))) + if num < 0: raise ValueError("Number of samples, %s, must be non-negative." % num) div = (num - 1) if endpoint else num @@ -145,7 +139,7 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, # from overriding what class is produced, and thus prevents, e.g. use of Quantities, # see gh-7142. Hence, we multiply in place only for standard scalar types. _mult_inplace = _nx.isscalar(delta) - if num > 1: + if div > 0: step = delta / div if _nx.any(step == 0): # Special handling for denormal numbers, gh-5437 @@ -160,7 +154,8 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, else: y = y * step else: - # 0 and 1 item long sequences have an undefined step + # sequences with 0 items or 1 item with endpoint=True (i.e. div <= 0) + # have an undefined step step = NaN # Multiply with delta to allow possible override of output class. y = y * delta @@ -430,28 +425,66 @@ def geomspace(start, stop, num=50, endpoint=True, dtype=None, axis=0): return result.astype(dtype, copy=False) -#always succeed -def _add_docstring(obj, doc): +def _needs_add_docstring(obj): + """ + Returns true if the only way to set the docstring of `obj` from python is + via add_docstring. + + This function errs on the side of being overly conservative. + """ + Py_TPFLAGS_HEAPTYPE = 1 << 9 + + if isinstance(obj, (types.FunctionType, types.MethodType, property)): + return False + + if isinstance(obj, type) and obj.__flags__ & Py_TPFLAGS_HEAPTYPE: + return False + + return True + + +def _add_docstring(obj, doc, warn_on_python): + if warn_on_python and not _needs_add_docstring(obj): + warnings.warn( + "add_newdoc was used on a pure-python object {}. " + "Prefer to attach it directly to the source." + .format(obj), + UserWarning, + stacklevel=3) try: add_docstring(obj, doc) except Exception: pass -def add_newdoc(place, obj, doc): +def add_newdoc(place, obj, doc, warn_on_python=True): """ - Adds documentation to obj which is in module place. + Add documentation to an existing object, typically one defined in C - If doc is a string add it to obj as a docstring + The purpose is to allow easier editing of the docstrings without requiring + a re-compile. This exists primarily for internal use within numpy itself. - If doc is a tuple, then the first element is interpreted as - an attribute of obj and the second as the docstring - (method, docstring) - - If doc is a list, then each element of the list should be a - sequence of length two --> [(method1, docstring1), - (method2, docstring2), ...] + Parameters + ---------- + place : str + The absolute name of the module to import from + obj : str + The name of the object to add documentation to, typically a class or + function name + doc : {str, Tuple[str, str], List[Tuple[str, str]]} + If a string, the documentation to apply to `obj` + + If a tuple, then the first element is interpreted as an attribute of + `obj` and the second as the docstring to apply - ``(method, docstring)`` + + If a list, then each element of the list should be a tuple of length + two - ``[(method1, docstring1), (method2, docstring2), ...]`` + warn_on_python : bool + If True, the default, emit `UserWarning` if this is used to attach + documentation to a pure-python object. + Notes + ----- This routine never raises an error if the docstring can't be written, but will raise an error if the object being documented does not exist. @@ -459,12 +492,23 @@ def add_newdoc(place, obj, doc): in new-style classes or built-in functions. Because this routine never raises an error the caller must check manually that the docstrings were changed. + + Since this function grabs the ``char *`` from a c-level str object and puts + it into the ``tp_doc`` slot of the type of `obj`, it violates a number of + C-API best-practices, by: + + - modifying a `PyTypeObject` after calling `PyType_Ready` + - calling `Py_INCREF` on the str and losing the reference, so the str + will never be released + + If possible it should be avoided. """ new = getattr(__import__(place, globals(), {}, [obj]), obj) if isinstance(doc, str): - _add_docstring(new, doc.strip()) + _add_docstring(new, doc.strip(), warn_on_python) elif isinstance(doc, tuple): - _add_docstring(getattr(new, doc[0]), doc[1].strip()) + attr, docstring = doc + _add_docstring(getattr(new, attr), docstring.strip(), warn_on_python) elif isinstance(doc, list): - for val in doc: - _add_docstring(getattr(new, val[0]), val[1].strip()) + for attr, docstring in doc: + _add_docstring(getattr(new, attr), docstring.strip(), warn_on_python) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 1221aeece..ad98d562b 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1095,7 +1095,8 @@ typedef struct PyArrayIterObject_tag PyArrayIterObject; * type of the function which translates a set of coordinates to a * pointer to the data */ -typedef char* (*npy_iter_get_dataptr_t)(PyArrayIterObject* iter, npy_intp*); +typedef char* (*npy_iter_get_dataptr_t)( + PyArrayIterObject* iter, const npy_intp*); struct PyArrayIterObject_tag { PyObject_HEAD @@ -1695,7 +1696,8 @@ PyArray_CLEARFLAGS(PyArrayObject *arr, int flags) #define PyDataType_ISOBJECT(obj) PyTypeNum_ISOBJECT(((PyArray_Descr*)(obj))->type_num) #define PyDataType_HASFIELDS(obj) (((PyArray_Descr *)(obj))->names != NULL) #define PyDataType_HASSUBARRAY(dtype) ((dtype)->subarray != NULL) -#define PyDataType_ISUNSIZED(dtype) ((dtype)->elsize == 0) +#define PyDataType_ISUNSIZED(dtype) ((dtype)->elsize == 0 && \ + !PyDataType_HASFIELDS(dtype)) #define PyDataType_MAKEUNSIZED(dtype) ((dtype)->elsize = 0) #define PyArray_ISBOOL(obj) PyTypeNum_ISBOOL(PyArray_TYPE(obj)) diff --git a/numpy/core/include/numpy/npy_common.h b/numpy/core/include/numpy/npy_common.h index 108c0a202..27b83f7b5 100644 --- a/numpy/core/include/numpy/npy_common.h +++ b/numpy/core/include/numpy/npy_common.h @@ -44,10 +44,14 @@ #else #define NPY_GCC_TARGET_AVX #endif + +#if defined HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS +#define HAVE_ATTRIBUTE_TARGET_FMA +#define NPY_GCC_TARGET_FMA __attribute__((target("avx2,fma"))) +#endif + #if defined HAVE_ATTRIBUTE_TARGET_AVX2 && defined HAVE_LINK_AVX2 #define NPY_GCC_TARGET_AVX2 __attribute__((target("avx2"))) -#elif defined HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS -#define NPY_GCC_TARGET_AVX2 __attribute__((target("avx2"))) #else #define NPY_GCC_TARGET_AVX2 #endif diff --git a/numpy/core/include/numpy/npy_math.h b/numpy/core/include/numpy/npy_math.h index 6a78ff3c2..69e690f28 100644 --- a/numpy/core/include/numpy/npy_math.h +++ b/numpy/core/include/numpy/npy_math.h @@ -144,7 +144,22 @@ NPY_INLINE static float __npy_nzerof(void) #define NPY_COEFF_Q3_LOGf 9.864942958519418960339e-01f #define NPY_COEFF_Q4_LOGf 1.546476374983906719538e-01f #define NPY_COEFF_Q5_LOGf 5.875095403124574342950e-03f - +/* + * Constants used in vector implementation of sinf/cosf(x) + */ +#define NPY_TWO_O_PIf 0x1.45f306p-1f +#define NPY_CODY_WAITE_PI_O_2_HIGHf -0x1.921fb0p+00f +#define NPY_CODY_WAITE_PI_O_2_MEDf -0x1.5110b4p-22f +#define NPY_CODY_WAITE_PI_O_2_LOWf -0x1.846988p-48f +#define NPY_COEFF_INVF0_COSINEf 0x1.000000p+00f +#define NPY_COEFF_INVF2_COSINEf -0x1.000000p-01f +#define NPY_COEFF_INVF4_COSINEf 0x1.55553cp-05f +#define NPY_COEFF_INVF6_COSINEf -0x1.6c06dcp-10f +#define NPY_COEFF_INVF8_COSINEf 0x1.98e616p-16f +#define NPY_COEFF_INVF3_SINEf -0x1.555556p-03f +#define NPY_COEFF_INVF5_SINEf 0x1.11119ap-07f +#define NPY_COEFF_INVF7_SINEf -0x1.a06bbap-13f +#define NPY_COEFF_INVF9_SINEf 0x1.7d3bbcp-19f /* * Integer functions. */ @@ -162,6 +177,37 @@ NPY_INPLACE npy_long npy_lcml(npy_long a, npy_long b); NPY_INPLACE npy_longlong npy_gcdll(npy_longlong a, npy_longlong b); NPY_INPLACE npy_longlong npy_lcmll(npy_longlong a, npy_longlong b); +NPY_INPLACE npy_ubyte npy_rshiftuhh(npy_ubyte a, npy_ubyte b); +NPY_INPLACE npy_ubyte npy_lshiftuhh(npy_ubyte a, npy_ubyte b); +NPY_INPLACE npy_ushort npy_rshiftuh(npy_ushort a, npy_ushort b); +NPY_INPLACE npy_ushort npy_lshiftuh(npy_ushort a, npy_ushort b); +NPY_INPLACE npy_uint npy_rshiftu(npy_uint a, npy_uint b); +NPY_INPLACE npy_uint npy_lshiftu(npy_uint a, npy_uint b); +NPY_INPLACE npy_ulong npy_rshiftul(npy_ulong a, npy_ulong b); +NPY_INPLACE npy_ulong npy_lshiftul(npy_ulong a, npy_ulong b); +NPY_INPLACE npy_ulonglong npy_rshiftull(npy_ulonglong a, npy_ulonglong b); +NPY_INPLACE npy_ulonglong npy_lshiftull(npy_ulonglong a, npy_ulonglong b); + +NPY_INPLACE npy_byte npy_rshifthh(npy_byte a, npy_byte b); +NPY_INPLACE npy_byte npy_lshifthh(npy_byte a, npy_byte b); +NPY_INPLACE npy_short npy_rshifth(npy_short a, npy_short b); +NPY_INPLACE npy_short npy_lshifth(npy_short a, npy_short b); +NPY_INPLACE npy_int npy_rshift(npy_int a, npy_int b); +NPY_INPLACE npy_int npy_lshift(npy_int a, npy_int b); +NPY_INPLACE npy_long npy_rshiftl(npy_long a, npy_long b); +NPY_INPLACE npy_long npy_lshiftl(npy_long a, npy_long b); +NPY_INPLACE npy_longlong npy_rshiftll(npy_longlong a, npy_longlong b); +NPY_INPLACE npy_longlong npy_lshiftll(npy_longlong a, npy_longlong b); + +/* + * avx function has a common API for both sin & cos. This enum is used to + * distinguish between the two + */ +typedef enum { + npy_compute_sin, + npy_compute_cos +} NPY_TRIG_OP; + /* * C99 double math funcs */ diff --git a/numpy/core/include/numpy/numpyconfig.h b/numpy/core/include/numpy/numpyconfig.h index ab198f36b..4bca82f9f 100644 --- a/numpy/core/include/numpy/numpyconfig.h +++ b/numpy/core/include/numpy/numpyconfig.h @@ -37,5 +37,8 @@ #define NPY_1_13_API_VERSION 0x00000008 #define NPY_1_14_API_VERSION 0x00000008 #define NPY_1_15_API_VERSION 0x00000008 +#define NPY_1_16_API_VERSION 0x00000008 +#define NPY_1_17_API_VERSION 0x00000008 +#define NPY_1_18_API_VERSION 0x00000008 #endif diff --git a/numpy/random/src/bitgen.h b/numpy/core/include/numpy/random/bitgen.h index 0adaaf2ee..83c2858dd 100644 --- a/numpy/random/src/bitgen.h +++ b/numpy/core/include/numpy/random/bitgen.h @@ -6,7 +6,7 @@ #include <stdbool.h> #include <stdint.h> -/* Must match the declaration in numpy/random/common.pxd */ +/* Must match the declaration in numpy/random/<any>.pxd */ typedef struct bitgen { void *state; diff --git a/numpy/random/src/distributions/distributions.h b/numpy/core/include/numpy/random/distributions.h index b778968d7..c474c4d14 100644 --- a/numpy/random/src/distributions/distributions.h +++ b/numpy/core/include/numpy/random/distributions.h @@ -1,15 +1,14 @@ #ifndef _RANDOMDGEN__DISTRIBUTIONS_H_ #define _RANDOMDGEN__DISTRIBUTIONS_H_ -#pragma once +#include "Python.h" +#include "numpy/npy_common.h" #include <stddef.h> #include <stdbool.h> #include <stdint.h> -#include "Python.h" -#include "numpy/npy_common.h" #include "numpy/npy_math.h" -#include "src/bitgen.h" +#include "numpy/random/bitgen.h" /* * RAND_INT_TYPE is used to share integer generators with RandomState which @@ -25,7 +24,7 @@ #define RAND_INT_MAX INT64_MAX #endif -#ifdef DLL_EXPORT +#ifdef _MSC_VER #define DECLDIR __declspec(dllexport) #else #define DECLDIR extern @@ -60,28 +59,10 @@ typedef struct s_binomial_t { double p4; } binomial_t; -/* Inline generators for internal use */ -static NPY_INLINE uint32_t next_uint32(bitgen_t *bitgen_state) { - return bitgen_state->next_uint32(bitgen_state->state); -} - -static NPY_INLINE uint64_t next_uint64(bitgen_t *bitgen_state) { - return bitgen_state->next_uint64(bitgen_state->state); -} - -static NPY_INLINE float next_float(bitgen_t *bitgen_state) { - return (next_uint32(bitgen_state) >> 9) * (1.0f / 8388608.0f); -} - -static NPY_INLINE double next_double(bitgen_t *bitgen_state) { - return bitgen_state->next_double(bitgen_state->state); -} - -DECLDIR double loggam(double x); - -DECLDIR float random_float(bitgen_t *bitgen_state); -DECLDIR double random_double(bitgen_t *bitgen_state); -DECLDIR void random_double_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out); +DECLDIR float random_standard_uniform_f(bitgen_t *bitgen_state); +DECLDIR double random_standard_uniform(bitgen_t *bitgen_state); +DECLDIR void random_standard_uniform_fill(bitgen_t *, npy_intp, double *); +DECLDIR void random_standard_uniform_fill_f(bitgen_t *, npy_intp, float *); DECLDIR int64_t random_positive_int64(bitgen_t *bitgen_state); DECLDIR int32_t random_positive_int32(bitgen_t *bitgen_state); @@ -89,37 +70,23 @@ DECLDIR int64_t random_positive_int(bitgen_t *bitgen_state); DECLDIR uint64_t random_uint(bitgen_t *bitgen_state); DECLDIR double random_standard_exponential(bitgen_t *bitgen_state); -DECLDIR void random_standard_exponential_fill(bitgen_t *bitgen_state, npy_intp cnt, - double *out); DECLDIR float random_standard_exponential_f(bitgen_t *bitgen_state); -DECLDIR double random_standard_exponential_zig(bitgen_t *bitgen_state); -DECLDIR void random_standard_exponential_zig_fill(bitgen_t *bitgen_state, - npy_intp cnt, double *out); -DECLDIR float random_standard_exponential_zig_f(bitgen_t *bitgen_state); - -/* -DECLDIR double random_gauss(bitgen_t *bitgen_state); -DECLDIR float random_gauss_f(bitgen_t *bitgen_state); -*/ -DECLDIR double random_gauss_zig(bitgen_t *bitgen_state); -DECLDIR float random_gauss_zig_f(bitgen_t *bitgen_state); -DECLDIR void random_gauss_zig_fill(bitgen_t *bitgen_state, npy_intp cnt, - double *out); - -/* +DECLDIR void random_standard_exponential_fill(bitgen_t *, npy_intp, double *); +DECLDIR void random_standard_exponential_fill_f(bitgen_t *, npy_intp, float *); +DECLDIR void random_standard_exponential_inv_fill(bitgen_t *, npy_intp, double *); +DECLDIR void random_standard_exponential_inv_fill_f(bitgen_t *, npy_intp, float *); + +DECLDIR double random_standard_normal(bitgen_t *bitgen_state); +DECLDIR float random_standard_normal_f(bitgen_t *bitgen_state); +DECLDIR void random_standard_normal_fill(bitgen_t *, npy_intp, double *); +DECLDIR void random_standard_normal_fill_f(bitgen_t *, npy_intp, float *); DECLDIR double random_standard_gamma(bitgen_t *bitgen_state, double shape); DECLDIR float random_standard_gamma_f(bitgen_t *bitgen_state, float shape); -*/ -DECLDIR double random_standard_gamma_zig(bitgen_t *bitgen_state, double shape); -DECLDIR float random_standard_gamma_zig_f(bitgen_t *bitgen_state, float shape); -/* DECLDIR double random_normal(bitgen_t *bitgen_state, double loc, double scale); -*/ -DECLDIR double random_normal_zig(bitgen_t *bitgen_state, double loc, double scale); DECLDIR double random_gamma(bitgen_t *bitgen_state, double shape, double scale); -DECLDIR float random_gamma_float(bitgen_t *bitgen_state, float shape, float scale); +DECLDIR float random_gamma_f(bitgen_t *bitgen_state, float shape, float scale); DECLDIR double random_exponential(bitgen_t *bitgen_state, double scale); DECLDIR double random_uniform(bitgen_t *bitgen_state, double lower, double range); @@ -147,27 +114,16 @@ DECLDIR double random_triangular(bitgen_t *bitgen_state, double left, double mod DECLDIR RAND_INT_TYPE random_poisson(bitgen_t *bitgen_state, double lam); DECLDIR RAND_INT_TYPE random_negative_binomial(bitgen_t *bitgen_state, double n, - double p); - -DECLDIR RAND_INT_TYPE random_binomial_btpe(bitgen_t *bitgen_state, - RAND_INT_TYPE n, - double p, - binomial_t *binomial); -DECLDIR RAND_INT_TYPE random_binomial_inversion(bitgen_t *bitgen_state, - RAND_INT_TYPE n, - double p, - binomial_t *binomial); + double p); + DECLDIR int64_t random_binomial(bitgen_t *bitgen_state, double p, int64_t n, binomial_t *binomial); DECLDIR RAND_INT_TYPE random_logseries(bitgen_t *bitgen_state, double p); -DECLDIR RAND_INT_TYPE random_geometric_search(bitgen_t *bitgen_state, double p); -DECLDIR RAND_INT_TYPE random_geometric_inversion(bitgen_t *bitgen_state, double p); DECLDIR RAND_INT_TYPE random_geometric(bitgen_t *bitgen_state, double p); DECLDIR RAND_INT_TYPE random_zipf(bitgen_t *bitgen_state, double a); DECLDIR int64_t random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad, int64_t sample); - DECLDIR uint64_t random_interval(bitgen_t *bitgen_state, uint64_t max); /* Generate random uint64 numbers in closed interval [off, off + rng]. */ @@ -212,4 +168,33 @@ DECLDIR void random_bounded_bool_fill(bitgen_t *bitgen_state, npy_bool off, DECLDIR void random_multinomial(bitgen_t *bitgen_state, RAND_INT_TYPE n, RAND_INT_TYPE *mnix, double *pix, npy_intp d, binomial_t *binomial); +/* multivariate hypergeometric, "count" method */ +DECLDIR int random_multivariate_hypergeometric_count(bitgen_t *bitgen_state, + int64_t total, + size_t num_colors, int64_t *colors, + int64_t nsample, + size_t num_variates, int64_t *variates); + +/* multivariate hypergeometric, "marginals" method */ +DECLDIR void random_multivariate_hypergeometric_marginals(bitgen_t *bitgen_state, + int64_t total, + size_t num_colors, int64_t *colors, + int64_t nsample, + size_t num_variates, int64_t *variates); + +/* Common to legacy-distributions.c and distributions.c but not exported */ + +RAND_INT_TYPE random_binomial_btpe(bitgen_t *bitgen_state, + RAND_INT_TYPE n, + double p, + binomial_t *binomial); +RAND_INT_TYPE random_binomial_inversion(bitgen_t *bitgen_state, + RAND_INT_TYPE n, + double p, + binomial_t *binomial); +double random_loggam(double x); +static NPY_INLINE double next_double(bitgen_t *bitgen_state) { + return bitgen_state->next_double(bitgen_state->state); +} + #endif diff --git a/numpy/core/include/numpy/ufuncobject.h b/numpy/core/include/numpy/ufuncobject.h index 15dcdf010..5ff4a0041 100644 --- a/numpy/core/include/numpy/ufuncobject.h +++ b/numpy/core/include/numpy/ufuncobject.h @@ -340,14 +340,6 @@ typedef struct _loop1d_info { #define UFUNC_PYVALS_NAME "UFUNC_PYVALS" -#define UFUNC_CHECK_ERROR(arg) \ - do {if ((((arg)->obj & UFUNC_OBJ_NEEDS_API) && PyErr_Occurred()) || \ - ((arg)->errormask && \ - PyUFunc_checkfperr((arg)->errormask, \ - (arg)->errobj, \ - &(arg)->first))) \ - goto fail;} while (0) - /* * THESE MACROS ARE DEPRECATED. * Use npy_set_floatstatus_* in the npymath library. @@ -357,10 +349,6 @@ typedef struct _loop1d_info { #define UFUNC_FPE_UNDERFLOW NPY_FPE_UNDERFLOW #define UFUNC_FPE_INVALID NPY_FPE_INVALID -#define UFUNC_CHECK_STATUS(ret) \ - { \ - ret = npy_clear_floatstatus(); \ - } #define generate_divbyzero_error() npy_set_floatstatus_divbyzero() #define generate_overflow_error() npy_set_floatstatus_overflow() diff --git a/numpy/core/info.py b/numpy/core/info.py deleted file mode 100644 index c6f7bbcf2..000000000 --- a/numpy/core/info.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Defines a multi-dimensional array and useful procedures for Numerical computation. - -Functions - -- array - NumPy Array construction -- zeros - Return an array of all zeros -- empty - Return an uninitialized array -- shape - Return shape of sequence or array -- rank - Return number of dimensions -- size - Return number of elements in entire array or a - certain dimension -- fromstring - Construct array from (byte) string -- take - Select sub-arrays using sequence of indices -- put - Set sub-arrays using sequence of 1-D indices -- putmask - Set portion of arrays using a mask -- reshape - Return array with new shape -- repeat - Repeat elements of array -- choose - Construct new array from indexed array tuple -- correlate - Correlate two 1-d arrays -- searchsorted - Search for element in 1-d array -- sum - Total sum over a specified dimension -- average - Average, possibly weighted, over axis or array. -- cumsum - Cumulative sum over a specified dimension -- product - Total product over a specified dimension -- cumproduct - Cumulative product over a specified dimension -- alltrue - Logical and over an entire axis -- sometrue - Logical or over an entire axis -- allclose - Tests if sequences are essentially equal - -More Functions: - -- arange - Return regularly spaced array -- asarray - Guarantee NumPy array -- convolve - Convolve two 1-d arrays -- swapaxes - Exchange axes -- concatenate - Join arrays together -- transpose - Permute axes -- sort - Sort elements of array -- argsort - Indices of sorted array -- argmax - Index of largest value -- argmin - Index of smallest value -- inner - Innerproduct of two arrays -- dot - Dot product (matrix multiplication) -- outer - Outerproduct of two arrays -- resize - Return array with arbitrary new shape -- indices - Tuple of indices -- fromfunction - Construct array from universal function -- diagonal - Return diagonal array -- trace - Trace of array -- dump - Dump array to file object (pickle) -- dumps - Return pickled string representing data -- load - Return array stored in file object -- loads - Return array from pickled string -- ravel - Return array as 1-D -- nonzero - Indices of nonzero elements for 1-D array -- shape - Shape of array -- where - Construct array from binary result -- compress - Elements of array where condition is true -- clip - Clip array between two values -- ones - Array of all ones -- identity - 2-D identity array (matrix) - -(Universal) Math Functions - - add logical_or exp - subtract logical_xor log - multiply logical_not log10 - divide maximum sin - divide_safe minimum sinh - conjugate bitwise_and sqrt - power bitwise_or tan - absolute bitwise_xor tanh - negative invert ceil - greater left_shift fabs - greater_equal right_shift floor - less arccos arctan2 - less_equal arcsin fmod - equal arctan hypot - not_equal cos around - logical_and cosh sign - arccosh arcsinh arctanh - -""" -from __future__ import division, absolute_import, print_function - -depends = ['testing'] -global_symbols = ['*'] diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index ea2ef900e..1e011e2e7 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -26,6 +26,7 @@ if sys.version_info[0] < 3: from . import overrides from . import umath +from . import shape_base from .overrides import set_module from .umath import (multiply, invert, sin, PINF, NAN) from . import numerictypes @@ -48,14 +49,6 @@ array_function_dispatch = functools.partial( overrides.array_function_dispatch, module='numpy') -def loads(*args, **kwargs): - # NumPy 1.15.0, 2017-12-10 - warnings.warn( - "np.core.numeric.loads is deprecated, use pickle.loads instead", - DeprecationWarning, stacklevel=2) - return pickle.loads(*args, **kwargs) - - __all__ = [ 'newaxis', 'ndarray', 'flatiter', 'nditer', 'nested_iters', 'ufunc', 'arange', 'array', 'zeros', 'count_nonzero', 'empty', 'broadcast', 'dtype', @@ -66,7 +59,7 @@ __all__ = [ 'correlate', 'convolve', 'inner', 'dot', 'outer', 'vdot', 'roll', 'rollaxis', 'moveaxis', 'cross', 'tensordot', 'little_endian', 'fromiter', 'array_equal', 'array_equiv', 'indices', 'fromfunction', - 'isclose', 'load', 'loads', 'isscalar', 'binary_repr', 'base_repr', 'ones', + 'isclose', 'isscalar', 'binary_repr', 'base_repr', 'ones', 'identity', 'allclose', 'compare_chararrays', 'putmask', 'flatnonzero', 'Inf', 'inf', 'infty', 'Infinity', 'nan', 'NaN', 'False_', 'True_', 'bitwise_not', 'CLIP', 'RAISE', 'WRAP', 'MAXDIMS', @@ -299,7 +292,7 @@ def full(shape, fill_value, dtype=None, order='C'): fill_value : scalar Fill value. dtype : data-type, optional - The desired data-type for the array The default, `None`, means + The desired data-type for the array The default, None, means `np.array(fill_value).dtype`. order : {'C', 'F'}, optional Whether to store multidimensional data in C- or Fortran-contiguous @@ -530,7 +523,7 @@ def isfortran(a): C-ordered arrays evaluate as False even if they are also FORTRAN-ordered. - >>> np.isfortran(np.array([1, 2], order='FORTRAN')) + >>> np.isfortran(np.array([1, 2], order='F')) False """ @@ -553,8 +546,10 @@ def argwhere(a): Returns ------- - index_array : ndarray + index_array : (N, a.ndim) ndarray Indices of elements that are non-zero. Indices are grouped by element. + This array will have shape ``(N, a.ndim)`` where ``N`` is the number of + non-zero items. See Also -------- @@ -562,7 +557,8 @@ def argwhere(a): Notes ----- - ``np.argwhere(a)`` is the same as ``np.transpose(np.nonzero(a))``. + ``np.argwhere(a)`` is almost the same as ``np.transpose(np.nonzero(a))``, + but produces a result of the correct shape for a 0D array. The output of ``argwhere`` is not suitable for indexing arrays. For this purpose use ``nonzero(a)`` instead. @@ -580,6 +576,11 @@ def argwhere(a): [1, 2]]) """ + # nonzero does not behave well on 0d, so promote to 1d + if np.ndim(a) == 0: + a = shape_base.atleast_1d(a) + # then remove the added dimension + return argwhere(a)[:,:0] return transpose(nonzero(a)) @@ -937,7 +938,7 @@ def tensordot(a, b, axes=2): Returns ------- output : ndarray - The tensor dot product of the input. + The tensor dot product of the input. See Also -------- @@ -959,6 +960,9 @@ def tensordot(a, b, axes=2): two sequences of the same length, with the first axis to sum over given first in both sequences, the second axis second, and so forth. + The shape of the result consists of the non-contracted axes of the + first tensor, followed by the non-contracted axes of the second. + Examples -------- A "traditional" example: @@ -1780,19 +1784,19 @@ def _frombuffer(buf, dtype, shape, order): @set_module('numpy') -def isscalar(num): +def isscalar(element): """ - Returns True if the type of `num` is a scalar type. + Returns True if the type of `element` is a scalar type. Parameters ---------- - num : any + element : any Input argument, can be of any type and shape. Returns ------- val : bool - True if `num` is a scalar type, False if it is not. + True if `element` is a scalar type, False if it is not. See Also -------- @@ -1800,10 +1804,14 @@ def isscalar(num): Notes ----- - In almost all cases ``np.ndim(x) == 0`` should be used instead of this - function, as that will also return true for 0d arrays. This is how - numpy overloads functions in the style of the ``dx`` arguments to `gradient` - and the ``bins`` argument to `histogram`. Some key differences: + If you need a stricter way to identify a *numerical* scalar, use + ``isinstance(x, numbers.Number)``, as that returns ``False`` for most + non-numerical elements such as strings. + + In most cases ``np.ndim(x) == 0`` should be used instead of this function, + as that will also return true for 0d arrays. This is how numpy overloads + functions in the style of the ``dx`` arguments to `gradient` and the ``bins`` + argument to `histogram`. Some key differences: +--------------------------------------+---------------+-------------------+ | x |``isscalar(x)``|``np.ndim(x) == 0``| @@ -1851,9 +1859,9 @@ def isscalar(num): True """ - return (isinstance(num, generic) - or type(num) in ScalarType - or isinstance(num, numbers.Number)) + return (isinstance(element, generic) + or type(element) in ScalarType + or isinstance(element, numbers.Number)) @set_module('numpy') @@ -1935,6 +1943,10 @@ def binary_repr(num, width=None): "will raise an error in the future.", DeprecationWarning, stacklevel=3) + # Ensure that num is a Python integer to avoid overflow or unwanted + # casts to floating point. + num = operator.index(num) + if num == 0: return '0' * (width or 1) @@ -2024,30 +2036,6 @@ def base_repr(number, base=2, padding=0): return ''.join(reversed(res or '0')) -def load(file): - """ - Wrapper around cPickle.load which accepts either a file-like object or - a filename. - - Note that the NumPy binary format is not based on pickle/cPickle anymore. - For details on the preferred way of loading and saving files, see `load` - and `save`. - - See Also - -------- - load, save - - """ - # NumPy 1.15.0, 2017-12-10 - warnings.warn( - "np.core.numeric.load is deprecated, use pickle.load instead", - DeprecationWarning, stacklevel=2) - if isinstance(file, type("")): - with open(file, "rb") as file_pointer: - return pickle.load(file_pointer) - return pickle.load(file) - - # These are all essentially abbreviations # These might wind up in a special abbreviations module @@ -2110,9 +2098,9 @@ def allclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False): `atol` are added together to compare against the absolute difference between `a` and `b`. - If either array contains one or more NaNs, False is returned. - Infs are treated as equal if they are in the same place and of the same - sign in both arrays. + NaNs are treated as equal if they are in the same place and if + ``equal_nan=True``. Infs are treated as equal if they are in the same + place and of the same sign in both arrays. Parameters ---------- diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index ab1ff65a4..761c7087c 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -485,7 +485,7 @@ def sctype2char(sctype): Examples -------- - >>> for sctype in [np.int32, np.double, np.complex, np.string_, np.ndarray]: + >>> for sctype in [np.int32, np.double, np.complex_, np.string_, np.ndarray]: ... print(np.sctype2char(sctype)) l # may vary d diff --git a/numpy/core/overrides.py b/numpy/core/overrides.py index 04a5a995f..55c7bd1ea 100644 --- a/numpy/core/overrides.py +++ b/numpy/core/overrides.py @@ -109,6 +109,18 @@ def set_module(module): return decorator + +# Call textwrap.dedent here instead of in the function so as to avoid +# calling dedent multiple times on the same text +_wrapped_func_source = textwrap.dedent(""" + @functools.wraps(implementation) + def {name}(*args, **kwargs): + relevant_args = dispatcher(*args, **kwargs) + return implement_array_function( + implementation, {name}, relevant_args, args, kwargs) + """) + + def array_function_dispatch(dispatcher, module=None, verify=True, docs_from_dispatcher=False): """Decorator for adding dispatch with the __array_function__ protocol. @@ -163,13 +175,7 @@ def array_function_dispatch(dispatcher, module=None, verify=True, # more interpettable name. Otherwise, the original function does not # show up at all in many cases, e.g., if it's written in C or if the # dispatcher gets an invalid keyword argument. - source = textwrap.dedent(""" - @functools.wraps(implementation) - def {name}(*args, **kwargs): - relevant_args = dispatcher(*args, **kwargs) - return implement_array_function( - implementation, {name}, relevant_args, args, kwargs) - """).format(name=implementation.__name__) + source = _wrapped_func_source.format(name=implementation.__name__) source_object = compile( source, filename='<__array_function__ internals>', mode='exec') diff --git a/numpy/core/records.py b/numpy/core/records.py index 8a5fee541..a1cad9075 100644 --- a/numpy/core/records.py +++ b/numpy/core/records.py @@ -163,7 +163,7 @@ class format_parser(object): self._createdescr(byteorder) self.dtype = self._descr - def _parseFormats(self, formats, aligned=0): + def _parseFormats(self, formats, aligned=False): """ Parse the field formats """ if formats is None: diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 0a2a8233e..974ec4628 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -394,7 +394,7 @@ def visibility_define(config): def configuration(parent_package='',top_path=None): from numpy.distutils.misc_util import Configuration, dot_join - from numpy.distutils.system_info import get_info + from numpy.distutils.system_info import get_info, dict_append config = Configuration('core', parent_package, top_path) local_dir = config.local_path @@ -463,8 +463,14 @@ def configuration(parent_package='',top_path=None): rep = check_long_double_representation(config_cmd) moredefs.append(('HAVE_LDOUBLE_%s' % rep, 1)) + if check_for_right_shift_internal_compiler_error(config_cmd): + moredefs.append('NPY_DO_NOT_OPTIMIZE_LONG_right_shift') + moredefs.append('NPY_DO_NOT_OPTIMIZE_ULONG_right_shift') + moredefs.append('NPY_DO_NOT_OPTIMIZE_LONGLONG_right_shift') + moredefs.append('NPY_DO_NOT_OPTIMIZE_ULONGLONG_right_shift') + # Py3K check - if sys.version_info[0] == 3: + if sys.version_info[0] >= 3: moredefs.append(('NPY_PY3K', 1)) # Generate the config.h file from moredefs @@ -491,10 +497,10 @@ def configuration(parent_package='',top_path=None): #endif """)) - print('File:', target) + log.info('File: %s' % target) with open(target) as target_f: - print(target_f.read()) - print('EOF') + log.info(target_f.read()) + log.info('EOF') else: mathlibs = [] with open(target) as target_f: @@ -581,10 +587,10 @@ def configuration(parent_package='',top_path=None): """)) # Dump the numpyconfig.h header to stdout - print('File: %s' % target) + log.info('File: %s' % target) with open(target) as target_f: - print(target_f.read()) - print('EOF') + log.info(target_f.read()) + log.info('EOF') config.add_data_files((header_dir, target)) return target @@ -633,23 +639,6 @@ def configuration(parent_package='',top_path=None): ] ####################################################################### - # dummy module # - ####################################################################### - - # npymath needs the config.h and numpyconfig.h files to be generated, but - # build_clib cannot handle generate_config_h and generate_numpyconfig_h - # (don't ask). Because clib are generated before extensions, we have to - # explicitly add an extension which has generate_config_h and - # generate_numpyconfig_h as sources *before* adding npymath. - - config.add_extension('_dummy', - sources=[join('src', 'dummymodule.c'), - generate_config_h, - generate_numpyconfig_h, - generate_numpy_api] - ) - - ####################################################################### # npymath library # ####################################################################### @@ -666,6 +655,9 @@ def configuration(parent_package='',top_path=None): # compiler does not work). st = config_cmd.try_link('int main(void) { return 0;}') if not st: + # rerun the failing command in verbose mode + config_cmd.compiler.verbose = True + config_cmd.try_link('int main(void) { return 0;}') raise RuntimeError("Broken toolchain: cannot link a simple C program") mlibs = check_mathlib(config_cmd) @@ -761,8 +753,14 @@ def configuration(parent_package='',top_path=None): join('src', 'common', 'numpyos.c'), ] - blas_info = get_info('blas_opt', 0) - if blas_info and ('HAVE_CBLAS', None) in blas_info.get('define_macros', []): + if os.environ.get('NPY_USE_BLAS_ILP64', "0") != "0": + blas_info = get_info('blas_ilp64_opt', 2) + else: + blas_info = get_info('blas_opt', 0) + + have_blas = blas_info and ('HAVE_CBLAS', None) in blas_info.get('define_macros', []) + + if have_blas: extra_info = blas_info # These files are also in MANIFEST.in so that they are always in # the source distribution independently of HAVE_CBLAS. diff --git a/numpy/core/setup_common.py b/numpy/core/setup_common.py index 80706c99f..6356f08ba 100644 --- a/numpy/core/setup_common.py +++ b/numpy/core/setup_common.py @@ -5,6 +5,7 @@ import sys import warnings import copy import binascii +import textwrap from numpy.distutils.misc_util import mingw32 @@ -14,7 +15,7 @@ from numpy.distutils.misc_util import mingw32 #------------------- # How to change C_API_VERSION ? # - increase C_API_VERSION value -# - record the hash for the new C API with the script cversions.py +# - record the hash for the new C API with the cversions.py script # and add the hash to cversions.txt # The hash values are used to remind developers when the C API number was not # updated - generates a MismatchCAPIWarning warning which is turned into an @@ -88,14 +89,13 @@ def check_api_version(apiversion, codegen_dir): # codegen_dir have been updated without the API version being # updated. Any modification in those .txt files should be reflected # in the api and eventually abi versions. - # To compute the checksum of the current API, use - # code_generators/cversions.py script + # To compute the checksum of the current API, use numpy/core/cversions.py if not curapi_hash == api_hash: msg = ("API mismatch detected, the C API version " "numbers have to be updated. Current C api version is %d, " - "with checksum %s, but recorded checksum for C API version %d in " - "codegen_dir/cversions.txt is %s. If functions were added in the " - "C API, you have to update C_API_VERSION in %s." + "with checksum %s, but recorded checksum for C API version %d " + "in core/codegen_dir/cversions.txt is %s. If functions were " + "added in the C API, you have to update C_API_VERSION in %s." ) warnings.warn(msg % (apiversion, curapi_hash, apiversion, api_hash, __file__), @@ -179,9 +179,10 @@ OPTIONAL_FUNCTION_ATTRIBUTES = [('__attribute__((optimize("unroll-loops")))', # gcc 4.8.4 support attributes but not with intrisics # tested via "#include<%s> int %s %s(void *){code; return 0;};" % (header, attribute, name, code) # function name will be converted to HAVE_<upper-case-name> preprocessor macro -OPTIONAL_FUNCTION_ATTRIBUTES_WITH_INTRINSICS = [('__attribute__((target("avx2")))', +OPTIONAL_FUNCTION_ATTRIBUTES_WITH_INTRINSICS = [('__attribute__((target("avx2,fma")))', 'attribute_target_avx2_with_intrinsics', - '__m256 temp = _mm256_set1_ps(1.0)', + '__m256 temp = _mm256_set1_ps(1.0); temp = \ + _mm256_fmadd_ps(temp, temp, temp)', 'immintrin.h'), ('__attribute__((target("avx512f")))', 'attribute_target_avx512f_with_intrinsics', @@ -416,3 +417,41 @@ def long_double_representation(lines): else: # We never detected the after_sequence raise ValueError("Could not lock sequences (%s)" % saw) + + +def check_for_right_shift_internal_compiler_error(cmd): + """ + On our arm CI, this fails with an internal compilation error + + The failure looks like the following, and can be reproduced on ARM64 GCC 5.4: + + <source>: In function 'right_shift': + <source>:4:20: internal compiler error: in expand_shift_1, at expmed.c:2349 + ip1[i] = ip1[i] >> in2; + ^ + Please submit a full bug report, + with preprocessed source if appropriate. + See <http://gcc.gnu.org/bugs.html> for instructions. + Compiler returned: 1 + + This function returns True if this compiler bug is present, and we need to + turn off optimization for the function + """ + cmd._check_compiler() + has_optimize = cmd.try_compile(textwrap.dedent("""\ + __attribute__((optimize("O3"))) void right_shift() {} + """), None, None) + if not has_optimize: + return False + + no_err = cmd.try_compile(textwrap.dedent("""\ + typedef long the_type; /* fails also for unsigned and long long */ + __attribute__((optimize("O3"))) void right_shift(the_type in2, the_type *ip1, int n) { + for (int i = 0; i < n; i++) { + if (in2 < (the_type)sizeof(the_type) * 8) { + ip1[i] = ip1[i] >> in2; + } + } + } + """), None, None) + return not no_err diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index 710f64827..31b1c20b9 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -9,8 +9,9 @@ import warnings from . import numeric as _nx from . import overrides -from .numeric import array, asanyarray, newaxis +from ._asarray import array, asanyarray from .multiarray import normalize_axis_index +from . import fromnumeric as _from_nx array_function_dispatch = functools.partial( @@ -123,7 +124,7 @@ def atleast_2d(*arys): if ary.ndim == 0: result = ary.reshape(1, 1) elif ary.ndim == 1: - result = ary[newaxis, :] + result = ary[_nx.newaxis, :] else: result = ary res.append(result) @@ -193,9 +194,9 @@ def atleast_3d(*arys): if ary.ndim == 0: result = ary.reshape(1, 1, 1) elif ary.ndim == 1: - result = ary[newaxis, :, newaxis] + result = ary[_nx.newaxis, :, _nx.newaxis] elif ary.ndim == 2: - result = ary[:, :, newaxis] + result = ary[:, :, _nx.newaxis] else: result = ary res.append(result) @@ -435,9 +436,9 @@ def stack(arrays, axis=0, out=None): # Internal functions to eliminate the overhead of repeated dispatch in one of # the two possible paths inside np.block. # Use getattr to protect against __array_function__ being disabled. -_size = getattr(_nx.size, '__wrapped__', _nx.size) -_ndim = getattr(_nx.ndim, '__wrapped__', _nx.ndim) -_concatenate = getattr(_nx.concatenate, '__wrapped__', _nx.concatenate) +_size = getattr(_from_nx.size, '__wrapped__', _from_nx.size) +_ndim = getattr(_from_nx.ndim, '__wrapped__', _from_nx.ndim) +_concatenate = getattr(_from_nx.concatenate, '__wrapped__', _from_nx.concatenate) def _block_format_index(index): @@ -471,7 +472,7 @@ def _block_check_depths_match(arrays, parent_index=[]): first_index : list of int The full index of an element from the bottom of the nesting in `arrays`. If any element at the bottom is an empty list, this will - refer to it, and the last index along the empty axis will be `None`. + refer to it, and the last index along the empty axis will be None. max_arr_ndim : int The maximum of the ndims of the arrays nested in `arrays`. final_size: int @@ -574,7 +575,7 @@ def _concatenate_shapes(shapes, axis): that was computed deeper in the recursion. These are returned as tuples to ensure that they can quickly be added - to existing slice tuple without creating a new tuple everytime. + to existing slice tuple without creating a new tuple every time. """ # Cache a result that will be reused. diff --git a/numpy/core/src/common/binop_override.h b/numpy/core/src/common/binop_override.h index 47df63e38..c5e7ab808 100644 --- a/numpy/core/src/common/binop_override.h +++ b/numpy/core/src/common/binop_override.h @@ -129,11 +129,14 @@ binop_should_defer(PyObject *self, PyObject *other, int inplace) * check whether __array_ufunc__ equals None. */ attr = PyArray_LookupSpecial(other, "__array_ufunc__"); - if (attr) { + if (attr != NULL) { defer = !inplace && (attr == Py_None); Py_DECREF(attr); return defer; } + else if (PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } /* * Otherwise, we need to check for the legacy __array_priority__. But if * other.__class__ is a subtype of self.__class__, then it's already had diff --git a/numpy/core/src/common/cblasfuncs.c b/numpy/core/src/common/cblasfuncs.c index 39572fed4..e78587de0 100644 --- a/numpy/core/src/common/cblasfuncs.c +++ b/numpy/core/src/common/cblasfuncs.c @@ -24,28 +24,28 @@ static const float oneF[2] = {1.0, 0.0}, zeroF[2] = {0.0, 0.0}; static void gemm(int typenum, enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE transA, enum CBLAS_TRANSPOSE transB, - int m, int n, int k, - PyArrayObject *A, int lda, PyArrayObject *B, int ldb, PyArrayObject *R) + npy_intp m, npy_intp n, npy_intp k, + PyArrayObject *A, npy_intp lda, PyArrayObject *B, npy_intp ldb, PyArrayObject *R) { const void *Adata = PyArray_DATA(A), *Bdata = PyArray_DATA(B); void *Rdata = PyArray_DATA(R); - int ldc = PyArray_DIM(R, 1) > 1 ? PyArray_DIM(R, 1) : 1; + npy_intp ldc = PyArray_DIM(R, 1) > 1 ? PyArray_DIM(R, 1) : 1; switch (typenum) { case NPY_DOUBLE: - cblas_dgemm(order, transA, transB, m, n, k, 1., + CBLAS_FUNC(cblas_dgemm)(order, transA, transB, m, n, k, 1., Adata, lda, Bdata, ldb, 0., Rdata, ldc); break; case NPY_FLOAT: - cblas_sgemm(order, transA, transB, m, n, k, 1.f, + CBLAS_FUNC(cblas_sgemm)(order, transA, transB, m, n, k, 1.f, Adata, lda, Bdata, ldb, 0.f, Rdata, ldc); break; case NPY_CDOUBLE: - cblas_zgemm(order, transA, transB, m, n, k, oneD, + CBLAS_FUNC(cblas_zgemm)(order, transA, transB, m, n, k, oneD, Adata, lda, Bdata, ldb, zeroD, Rdata, ldc); break; case NPY_CFLOAT: - cblas_cgemm(order, transA, transB, m, n, k, oneF, + CBLAS_FUNC(cblas_cgemm)(order, transA, transB, m, n, k, oneF, Adata, lda, Bdata, ldb, zeroF, Rdata, ldc); break; } @@ -57,29 +57,29 @@ gemm(int typenum, enum CBLAS_ORDER order, */ static void gemv(int typenum, enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE trans, - PyArrayObject *A, int lda, PyArrayObject *X, int incX, + PyArrayObject *A, npy_intp lda, PyArrayObject *X, npy_intp incX, PyArrayObject *R) { const void *Adata = PyArray_DATA(A), *Xdata = PyArray_DATA(X); void *Rdata = PyArray_DATA(R); - int m = PyArray_DIM(A, 0), n = PyArray_DIM(A, 1); + npy_intp m = PyArray_DIM(A, 0), n = PyArray_DIM(A, 1); switch (typenum) { case NPY_DOUBLE: - cblas_dgemv(order, trans, m, n, 1., Adata, lda, Xdata, incX, + CBLAS_FUNC(cblas_dgemv)(order, trans, m, n, 1., Adata, lda, Xdata, incX, 0., Rdata, 1); break; case NPY_FLOAT: - cblas_sgemv(order, trans, m, n, 1.f, Adata, lda, Xdata, incX, + CBLAS_FUNC(cblas_sgemv)(order, trans, m, n, 1.f, Adata, lda, Xdata, incX, 0.f, Rdata, 1); break; case NPY_CDOUBLE: - cblas_zgemv(order, trans, m, n, oneD, Adata, lda, Xdata, incX, + CBLAS_FUNC(cblas_zgemv)(order, trans, m, n, oneD, Adata, lda, Xdata, incX, zeroD, Rdata, 1); break; case NPY_CFLOAT: - cblas_cgemv(order, trans, m, n, oneF, Adata, lda, Xdata, incX, + CBLAS_FUNC(cblas_cgemv)(order, trans, m, n, oneF, Adata, lda, Xdata, incX, zeroF, Rdata, 1); break; } @@ -91,19 +91,19 @@ gemv(int typenum, enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE trans, */ static void syrk(int typenum, enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE trans, - int n, int k, - PyArrayObject *A, int lda, PyArrayObject *R) + npy_intp n, npy_intp k, + PyArrayObject *A, npy_intp lda, PyArrayObject *R) { const void *Adata = PyArray_DATA(A); void *Rdata = PyArray_DATA(R); - int ldc = PyArray_DIM(R, 1) > 1 ? PyArray_DIM(R, 1) : 1; + npy_intp ldc = PyArray_DIM(R, 1) > 1 ? PyArray_DIM(R, 1) : 1; npy_intp i; npy_intp j; switch (typenum) { case NPY_DOUBLE: - cblas_dsyrk(order, CblasUpper, trans, n, k, 1., + CBLAS_FUNC(cblas_dsyrk)(order, CblasUpper, trans, n, k, 1., Adata, lda, 0., Rdata, ldc); for (i = 0; i < n; i++) { @@ -114,7 +114,7 @@ syrk(int typenum, enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE trans, } break; case NPY_FLOAT: - cblas_ssyrk(order, CblasUpper, trans, n, k, 1.f, + CBLAS_FUNC(cblas_ssyrk)(order, CblasUpper, trans, n, k, 1.f, Adata, lda, 0.f, Rdata, ldc); for (i = 0; i < n; i++) { @@ -125,7 +125,7 @@ syrk(int typenum, enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE trans, } break; case NPY_CDOUBLE: - cblas_zsyrk(order, CblasUpper, trans, n, k, oneD, + CBLAS_FUNC(cblas_zsyrk)(order, CblasUpper, trans, n, k, oneD, Adata, lda, zeroD, Rdata, ldc); for (i = 0; i < n; i++) { @@ -136,7 +136,7 @@ syrk(int typenum, enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE trans, } break; case NPY_CFLOAT: - cblas_csyrk(order, CblasUpper, trans, n, k, oneF, + CBLAS_FUNC(cblas_csyrk)(order, CblasUpper, trans, n, k, oneF, Adata, lda, zeroF, Rdata, ldc); for (i = 0; i < n; i++) { @@ -222,7 +222,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, PyArrayObject *out) { PyArrayObject *result = NULL, *out_buf = NULL; - int j, lda, ldb; + npy_intp j, lda, ldb; npy_intp l; int nd; npy_intp ap1stride = 0; @@ -385,14 +385,15 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, *((double *)PyArray_DATA(ap1)); } else if (ap1shape != _matrix) { - cblas_daxpy(l, + CBLAS_FUNC(cblas_daxpy)(l, *((double *)PyArray_DATA(ap2)), (double *)PyArray_DATA(ap1), ap1stride/sizeof(double), (double *)PyArray_DATA(out_buf), 1); } else { - int maxind, oind, i, a1s, outs; + int maxind, oind; + npy_intp i, a1s, outs; char *ptr, *optr; double val; @@ -405,7 +406,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, a1s = PyArray_STRIDE(ap1, maxind) / sizeof(double); outs = PyArray_STRIDE(out_buf, maxind) / sizeof(double); for (i = 0; i < PyArray_DIM(ap1, oind); i++) { - cblas_daxpy(l, val, (double *)ptr, a1s, + CBLAS_FUNC(cblas_daxpy)(l, val, (double *)ptr, a1s, (double *)optr, outs); ptr += PyArray_STRIDE(ap1, oind); optr += PyArray_STRIDE(out_buf, oind); @@ -423,14 +424,15 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, res->imag = ptr1->real * ptr2->imag + ptr1->imag * ptr2->real; } else if (ap1shape != _matrix) { - cblas_zaxpy(l, + CBLAS_FUNC(cblas_zaxpy)(l, (double *)PyArray_DATA(ap2), (double *)PyArray_DATA(ap1), ap1stride/sizeof(npy_cdouble), (double *)PyArray_DATA(out_buf), 1); } else { - int maxind, oind, i, a1s, outs; + int maxind, oind; + npy_intp i, a1s, outs; char *ptr, *optr; double *pval; @@ -443,7 +445,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, a1s = PyArray_STRIDE(ap1, maxind) / sizeof(npy_cdouble); outs = PyArray_STRIDE(out_buf, maxind) / sizeof(npy_cdouble); for (i = 0; i < PyArray_DIM(ap1, oind); i++) { - cblas_zaxpy(l, pval, (double *)ptr, a1s, + CBLAS_FUNC(cblas_zaxpy)(l, pval, (double *)ptr, a1s, (double *)optr, outs); ptr += PyArray_STRIDE(ap1, oind); optr += PyArray_STRIDE(out_buf, oind); @@ -456,14 +458,15 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, *((float *)PyArray_DATA(ap1)); } else if (ap1shape != _matrix) { - cblas_saxpy(l, + CBLAS_FUNC(cblas_saxpy)(l, *((float *)PyArray_DATA(ap2)), (float *)PyArray_DATA(ap1), ap1stride/sizeof(float), (float *)PyArray_DATA(out_buf), 1); } else { - int maxind, oind, i, a1s, outs; + int maxind, oind; + npy_intp i, a1s, outs; char *ptr, *optr; float val; @@ -476,7 +479,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, a1s = PyArray_STRIDE(ap1, maxind) / sizeof(float); outs = PyArray_STRIDE(out_buf, maxind) / sizeof(float); for (i = 0; i < PyArray_DIM(ap1, oind); i++) { - cblas_saxpy(l, val, (float *)ptr, a1s, + CBLAS_FUNC(cblas_saxpy)(l, val, (float *)ptr, a1s, (float *)optr, outs); ptr += PyArray_STRIDE(ap1, oind); optr += PyArray_STRIDE(out_buf, oind); @@ -494,14 +497,15 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, res->imag = ptr1->real * ptr2->imag + ptr1->imag * ptr2->real; } else if (ap1shape != _matrix) { - cblas_caxpy(l, + CBLAS_FUNC(cblas_caxpy)(l, (float *)PyArray_DATA(ap2), (float *)PyArray_DATA(ap1), ap1stride/sizeof(npy_cfloat), (float *)PyArray_DATA(out_buf), 1); } else { - int maxind, oind, i, a1s, outs; + int maxind, oind; + npy_intp i, a1s, outs; char *ptr, *optr; float *pval; @@ -514,7 +518,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, a1s = PyArray_STRIDE(ap1, maxind) / sizeof(npy_cfloat); outs = PyArray_STRIDE(out_buf, maxind) / sizeof(npy_cfloat); for (i = 0; i < PyArray_DIM(ap1, oind); i++) { - cblas_caxpy(l, pval, (float *)ptr, a1s, + CBLAS_FUNC(cblas_caxpy)(l, pval, (float *)ptr, a1s, (float *)optr, outs); ptr += PyArray_STRIDE(ap1, oind); optr += PyArray_STRIDE(out_buf, oind); @@ -537,7 +541,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, /* Matrix vector multiplication -- Level 2 BLAS */ /* lda must be MAX(M,1) */ enum CBLAS_ORDER Order; - int ap2s; + npy_intp ap2s; if (!PyArray_ISONESEGMENT(ap1)) { PyObject *new; @@ -564,7 +568,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, else if (ap1shape != _matrix && ap2shape == _matrix) { /* Vector matrix multiplication -- Level 2 BLAS */ enum CBLAS_ORDER Order; - int ap1s; + npy_intp ap1s; if (!PyArray_ISONESEGMENT(ap2)) { PyObject *new; @@ -601,7 +605,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, */ enum CBLAS_ORDER Order; enum CBLAS_TRANSPOSE Trans1, Trans2; - int M, N, L; + npy_intp M, N, L; /* Optimization possible: */ /* diff --git a/numpy/core/src/common/get_attr_string.h b/numpy/core/src/common/get_attr_string.h index d458d9550..d3401aea6 100644 --- a/numpy/core/src/common/get_attr_string.h +++ b/numpy/core/src/common/get_attr_string.h @@ -40,18 +40,14 @@ _is_basic_python_type(PyTypeObject *tp) } /* - * Stripped down version of PyObject_GetAttrString, - * avoids lookups for None, tuple, and List objects, - * and doesn't create a PyErr since this code ignores it. + * Stripped down version of PyObject_GetAttrString(obj, name) that does not + * raise PyExc_AttributeError. * - * This can be much faster then PyObject_GetAttrString where - * exceptions are not used by caller. + * This allows it to avoid creating then discarding exception objects when + * performing lookups on objects without any attributes. * - * 'obj' is the object to search for attribute. - * - * 'name' is the attribute to search for. - * - * Returns attribute value on success, NULL on failure. + * Returns attribute value on success, NULL without an exception set if + * there is no such attribute, and NULL with an exception on failure. */ static NPY_INLINE PyObject * maybe_get_attr(PyObject *obj, char *name) @@ -62,7 +58,7 @@ maybe_get_attr(PyObject *obj, char *name) /* Attribute referenced by (char *)name */ if (tp->tp_getattr != NULL) { res = (*tp->tp_getattr)(obj, name); - if (res == NULL) { + if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); } } @@ -78,7 +74,7 @@ maybe_get_attr(PyObject *obj, char *name) } res = (*tp->tp_getattro)(obj, w); Py_DECREF(w); - if (res == NULL) { + if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); } } diff --git a/numpy/core/src/common/npy_cblas.h b/numpy/core/src/common/npy_cblas.h index a083f3bcc..97308238a 100644 --- a/numpy/core/src/common/npy_cblas.h +++ b/numpy/core/src/common/npy_cblas.h @@ -17,565 +17,47 @@ extern "C" /* * Enumerated and derived types */ -#define CBLAS_INDEX size_t /* this may vary between platforms */ - enum CBLAS_ORDER {CblasRowMajor=101, CblasColMajor=102}; enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113}; enum CBLAS_UPLO {CblasUpper=121, CblasLower=122}; enum CBLAS_DIAG {CblasNonUnit=131, CblasUnit=132}; enum CBLAS_SIDE {CblasLeft=141, CblasRight=142}; -/* - * =========================================================================== - * Prototypes for level 1 BLAS functions (complex are recast as routines) - * =========================================================================== - */ -float cblas_sdsdot(const int N, const float alpha, const float *X, - const int incX, const float *Y, const int incY); -double cblas_dsdot(const int N, const float *X, const int incX, const float *Y, - const int incY); -float cblas_sdot(const int N, const float *X, const int incX, - const float *Y, const int incY); -double cblas_ddot(const int N, const double *X, const int incX, - const double *Y, const int incY); - -/* - * Functions having prefixes Z and C only - */ -void cblas_cdotu_sub(const int N, const void *X, const int incX, - const void *Y, const int incY, void *dotu); -void cblas_cdotc_sub(const int N, const void *X, const int incX, - const void *Y, const int incY, void *dotc); - -void cblas_zdotu_sub(const int N, const void *X, const int incX, - const void *Y, const int incY, void *dotu); -void cblas_zdotc_sub(const int N, const void *X, const int incX, - const void *Y, const int incY, void *dotc); - - -/* - * Functions having prefixes S D SC DZ - */ -float cblas_snrm2(const int N, const float *X, const int incX); -float cblas_sasum(const int N, const float *X, const int incX); - -double cblas_dnrm2(const int N, const double *X, const int incX); -double cblas_dasum(const int N, const double *X, const int incX); - -float cblas_scnrm2(const int N, const void *X, const int incX); -float cblas_scasum(const int N, const void *X, const int incX); - -double cblas_dznrm2(const int N, const void *X, const int incX); -double cblas_dzasum(const int N, const void *X, const int incX); - - -/* - * Functions having standard 4 prefixes (S D C Z) - */ -CBLAS_INDEX cblas_isamax(const int N, const float *X, const int incX); -CBLAS_INDEX cblas_idamax(const int N, const double *X, const int incX); -CBLAS_INDEX cblas_icamax(const int N, const void *X, const int incX); -CBLAS_INDEX cblas_izamax(const int N, const void *X, const int incX); - -/* - * =========================================================================== - * Prototypes for level 1 BLAS routines - * =========================================================================== - */ - -/* - * Routines with standard 4 prefixes (s, d, c, z) - */ -void cblas_sswap(const int N, float *X, const int incX, - float *Y, const int incY); -void cblas_scopy(const int N, const float *X, const int incX, - float *Y, const int incY); -void cblas_saxpy(const int N, const float alpha, const float *X, - const int incX, float *Y, const int incY); - -void cblas_dswap(const int N, double *X, const int incX, - double *Y, const int incY); -void cblas_dcopy(const int N, const double *X, const int incX, - double *Y, const int incY); -void cblas_daxpy(const int N, const double alpha, const double *X, - const int incX, double *Y, const int incY); - -void cblas_cswap(const int N, void *X, const int incX, - void *Y, const int incY); -void cblas_ccopy(const int N, const void *X, const int incX, - void *Y, const int incY); -void cblas_caxpy(const int N, const void *alpha, const void *X, - const int incX, void *Y, const int incY); - -void cblas_zswap(const int N, void *X, const int incX, - void *Y, const int incY); -void cblas_zcopy(const int N, const void *X, const int incX, - void *Y, const int incY); -void cblas_zaxpy(const int N, const void *alpha, const void *X, - const int incX, void *Y, const int incY); - - -/* - * Routines with S and D prefix only - */ -void cblas_srotg(float *a, float *b, float *c, float *s); -void cblas_srotmg(float *d1, float *d2, float *b1, const float b2, float *P); -void cblas_srot(const int N, float *X, const int incX, - float *Y, const int incY, const float c, const float s); -void cblas_srotm(const int N, float *X, const int incX, - float *Y, const int incY, const float *P); - -void cblas_drotg(double *a, double *b, double *c, double *s); -void cblas_drotmg(double *d1, double *d2, double *b1, const double b2, double *P); -void cblas_drot(const int N, double *X, const int incX, - double *Y, const int incY, const double c, const double s); -void cblas_drotm(const int N, double *X, const int incX, - double *Y, const int incY, const double *P); - - -/* - * Routines with S D C Z CS and ZD prefixes - */ -void cblas_sscal(const int N, const float alpha, float *X, const int incX); -void cblas_dscal(const int N, const double alpha, double *X, const int incX); -void cblas_cscal(const int N, const void *alpha, void *X, const int incX); -void cblas_zscal(const int N, const void *alpha, void *X, const int incX); -void cblas_csscal(const int N, const float alpha, void *X, const int incX); -void cblas_zdscal(const int N, const double alpha, void *X, const int incX); - -/* - * =========================================================================== - * Prototypes for level 2 BLAS - * =========================================================================== - */ - -/* - * Routines with standard 4 prefixes (S, D, C, Z) - */ -void cblas_sgemv(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const int M, const int N, - const float alpha, const float *A, const int lda, - const float *X, const int incX, const float beta, - float *Y, const int incY); -void cblas_sgbmv(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const int M, const int N, - const int KL, const int KU, const float alpha, - const float *A, const int lda, const float *X, - const int incX, const float beta, float *Y, const int incY); -void cblas_strmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const float *A, const int lda, - float *X, const int incX); -void cblas_stbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const int K, const float *A, const int lda, - float *X, const int incX); -void cblas_stpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const float *Ap, float *X, const int incX); -void cblas_strsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const float *A, const int lda, float *X, - const int incX); -void cblas_stbsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const int K, const float *A, const int lda, - float *X, const int incX); -void cblas_stpsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const float *Ap, float *X, const int incX); - -void cblas_dgemv(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const int M, const int N, - const double alpha, const double *A, const int lda, - const double *X, const int incX, const double beta, - double *Y, const int incY); -void cblas_dgbmv(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const int M, const int N, - const int KL, const int KU, const double alpha, - const double *A, const int lda, const double *X, - const int incX, const double beta, double *Y, const int incY); -void cblas_dtrmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const double *A, const int lda, - double *X, const int incX); -void cblas_dtbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const int K, const double *A, const int lda, - double *X, const int incX); -void cblas_dtpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const double *Ap, double *X, const int incX); -void cblas_dtrsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const double *A, const int lda, double *X, - const int incX); -void cblas_dtbsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const int K, const double *A, const int lda, - double *X, const int incX); -void cblas_dtpsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const double *Ap, double *X, const int incX); - -void cblas_cgemv(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const int M, const int N, - const void *alpha, const void *A, const int lda, - const void *X, const int incX, const void *beta, - void *Y, const int incY); -void cblas_cgbmv(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const int M, const int N, - const int KL, const int KU, const void *alpha, - const void *A, const int lda, const void *X, - const int incX, const void *beta, void *Y, const int incY); -void cblas_ctrmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const void *A, const int lda, - void *X, const int incX); -void cblas_ctbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const int K, const void *A, const int lda, - void *X, const int incX); -void cblas_ctpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const void *Ap, void *X, const int incX); -void cblas_ctrsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const void *A, const int lda, void *X, - const int incX); -void cblas_ctbsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const int K, const void *A, const int lda, - void *X, const int incX); -void cblas_ctpsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const void *Ap, void *X, const int incX); - -void cblas_zgemv(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const int M, const int N, - const void *alpha, const void *A, const int lda, - const void *X, const int incX, const void *beta, - void *Y, const int incY); -void cblas_zgbmv(const enum CBLAS_ORDER order, - const enum CBLAS_TRANSPOSE TransA, const int M, const int N, - const int KL, const int KU, const void *alpha, - const void *A, const int lda, const void *X, - const int incX, const void *beta, void *Y, const int incY); -void cblas_ztrmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const void *A, const int lda, - void *X, const int incX); -void cblas_ztbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const int K, const void *A, const int lda, - void *X, const int incX); -void cblas_ztpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const void *Ap, void *X, const int incX); -void cblas_ztrsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const void *A, const int lda, void *X, - const int incX); -void cblas_ztbsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const int K, const void *A, const int lda, - void *X, const int incX); -void cblas_ztpsv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, - const int N, const void *Ap, void *X, const int incX); - - -/* - * Routines with S and D prefixes only - */ -void cblas_ssymv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const float alpha, const float *A, - const int lda, const float *X, const int incX, - const float beta, float *Y, const int incY); -void cblas_ssbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const int K, const float alpha, const float *A, - const int lda, const float *X, const int incX, - const float beta, float *Y, const int incY); -void cblas_sspmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const float alpha, const float *Ap, - const float *X, const int incX, - const float beta, float *Y, const int incY); -void cblas_sger(const enum CBLAS_ORDER order, const int M, const int N, - const float alpha, const float *X, const int incX, - const float *Y, const int incY, float *A, const int lda); -void cblas_ssyr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const float alpha, const float *X, - const int incX, float *A, const int lda); -void cblas_sspr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const float alpha, const float *X, - const int incX, float *Ap); -void cblas_ssyr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const float alpha, const float *X, - const int incX, const float *Y, const int incY, float *A, - const int lda); -void cblas_sspr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const float alpha, const float *X, - const int incX, const float *Y, const int incY, float *A); - -void cblas_dsymv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const double alpha, const double *A, - const int lda, const double *X, const int incX, - const double beta, double *Y, const int incY); -void cblas_dsbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const int K, const double alpha, const double *A, - const int lda, const double *X, const int incX, - const double beta, double *Y, const int incY); -void cblas_dspmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const double alpha, const double *Ap, - const double *X, const int incX, - const double beta, double *Y, const int incY); -void cblas_dger(const enum CBLAS_ORDER order, const int M, const int N, - const double alpha, const double *X, const int incX, - const double *Y, const int incY, double *A, const int lda); -void cblas_dsyr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const double alpha, const double *X, - const int incX, double *A, const int lda); -void cblas_dspr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const double alpha, const double *X, - const int incX, double *Ap); -void cblas_dsyr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const double alpha, const double *X, - const int incX, const double *Y, const int incY, double *A, - const int lda); -void cblas_dspr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const double alpha, const double *X, - const int incX, const double *Y, const int incY, double *A); - - -/* - * Routines with C and Z prefixes only - */ -void cblas_chemv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const void *alpha, const void *A, - const int lda, const void *X, const int incX, - const void *beta, void *Y, const int incY); -void cblas_chbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const int K, const void *alpha, const void *A, - const int lda, const void *X, const int incX, - const void *beta, void *Y, const int incY); -void cblas_chpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const void *alpha, const void *Ap, - const void *X, const int incX, - const void *beta, void *Y, const int incY); -void cblas_cgeru(const enum CBLAS_ORDER order, const int M, const int N, - const void *alpha, const void *X, const int incX, - const void *Y, const int incY, void *A, const int lda); -void cblas_cgerc(const enum CBLAS_ORDER order, const int M, const int N, - const void *alpha, const void *X, const int incX, - const void *Y, const int incY, void *A, const int lda); -void cblas_cher(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const float alpha, const void *X, const int incX, - void *A, const int lda); -void cblas_chpr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const float alpha, const void *X, - const int incX, void *A); -void cblas_cher2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, - const void *alpha, const void *X, const int incX, - const void *Y, const int incY, void *A, const int lda); -void cblas_chpr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, - const void *alpha, const void *X, const int incX, - const void *Y, const int incY, void *Ap); - -void cblas_zhemv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const void *alpha, const void *A, - const int lda, const void *X, const int incX, - const void *beta, void *Y, const int incY); -void cblas_zhbmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const int K, const void *alpha, const void *A, - const int lda, const void *X, const int incX, - const void *beta, void *Y, const int incY); -void cblas_zhpmv(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const void *alpha, const void *Ap, - const void *X, const int incX, - const void *beta, void *Y, const int incY); -void cblas_zgeru(const enum CBLAS_ORDER order, const int M, const int N, - const void *alpha, const void *X, const int incX, - const void *Y, const int incY, void *A, const int lda); -void cblas_zgerc(const enum CBLAS_ORDER order, const int M, const int N, - const void *alpha, const void *X, const int incX, - const void *Y, const int incY, void *A, const int lda); -void cblas_zher(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const double alpha, const void *X, const int incX, - void *A, const int lda); -void cblas_zhpr(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, - const int N, const double alpha, const void *X, - const int incX, void *A); -void cblas_zher2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, - const void *alpha, const void *X, const int incX, - const void *Y, const int incY, void *A, const int lda); -void cblas_zhpr2(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const int N, - const void *alpha, const void *X, const int incX, - const void *Y, const int incY, void *Ap); +#define CBLAS_INDEX size_t /* this may vary between platforms */ -/* - * =========================================================================== - * Prototypes for level 3 BLAS - * =========================================================================== - */ +#ifdef NO_APPEND_FORTRAN +#define BLAS_FORTRAN_SUFFIX +#else +#define BLAS_FORTRAN_SUFFIX _ +#endif -/* - * Routines with standard 4 prefixes (S, D, C, Z) - */ -void cblas_sgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_TRANSPOSE TransB, const int M, const int N, - const int K, const float alpha, const float *A, - const int lda, const float *B, const int ldb, - const float beta, float *C, const int ldc); -void cblas_ssymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const int M, const int N, - const float alpha, const float *A, const int lda, - const float *B, const int ldb, const float beta, - float *C, const int ldc); -void cblas_ssyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const float alpha, const float *A, const int lda, - const float beta, float *C, const int ldc); -void cblas_ssyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const float alpha, const float *A, const int lda, - const float *B, const int ldb, const float beta, - float *C, const int ldc); -void cblas_strmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_DIAG Diag, const int M, const int N, - const float alpha, const float *A, const int lda, - float *B, const int ldb); -void cblas_strsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_DIAG Diag, const int M, const int N, - const float alpha, const float *A, const int lda, - float *B, const int ldb); +#ifndef BLAS_SYMBOL_PREFIX +#define BLAS_SYMBOL_PREFIX +#endif -void cblas_dgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_TRANSPOSE TransB, const int M, const int N, - const int K, const double alpha, const double *A, - const int lda, const double *B, const int ldb, - const double beta, double *C, const int ldc); -void cblas_dsymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const int M, const int N, - const double alpha, const double *A, const int lda, - const double *B, const int ldb, const double beta, - double *C, const int ldc); -void cblas_dsyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const double alpha, const double *A, const int lda, - const double beta, double *C, const int ldc); -void cblas_dsyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const double alpha, const double *A, const int lda, - const double *B, const int ldb, const double beta, - double *C, const int ldc); -void cblas_dtrmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_DIAG Diag, const int M, const int N, - const double alpha, const double *A, const int lda, - double *B, const int ldb); -void cblas_dtrsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_DIAG Diag, const int M, const int N, - const double alpha, const double *A, const int lda, - double *B, const int ldb); +#ifndef BLAS_SYMBOL_SUFFIX +#define BLAS_SYMBOL_SUFFIX +#endif -void cblas_cgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_TRANSPOSE TransB, const int M, const int N, - const int K, const void *alpha, const void *A, - const int lda, const void *B, const int ldb, - const void *beta, void *C, const int ldc); -void cblas_csymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const int M, const int N, - const void *alpha, const void *A, const int lda, - const void *B, const int ldb, const void *beta, - void *C, const int ldc); -void cblas_csyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const void *alpha, const void *A, const int lda, - const void *beta, void *C, const int ldc); -void cblas_csyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const void *alpha, const void *A, const int lda, - const void *B, const int ldb, const void *beta, - void *C, const int ldc); -void cblas_ctrmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_DIAG Diag, const int M, const int N, - const void *alpha, const void *A, const int lda, - void *B, const int ldb); -void cblas_ctrsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_DIAG Diag, const int M, const int N, - const void *alpha, const void *A, const int lda, - void *B, const int ldb); +#define BLAS_FUNC_CONCAT(name,prefix,suffix,suffix2) prefix ## name ## suffix ## suffix2 +#define BLAS_FUNC_EXPAND(name,prefix,suffix,suffix2) BLAS_FUNC_CONCAT(name,prefix,suffix,suffix2) -void cblas_zgemm(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_TRANSPOSE TransB, const int M, const int N, - const int K, const void *alpha, const void *A, - const int lda, const void *B, const int ldb, - const void *beta, void *C, const int ldc); -void cblas_zsymm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const int M, const int N, - const void *alpha, const void *A, const int lda, - const void *B, const int ldb, const void *beta, - void *C, const int ldc); -void cblas_zsyrk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const void *alpha, const void *A, const int lda, - const void *beta, void *C, const int ldc); -void cblas_zsyr2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const void *alpha, const void *A, const int lda, - const void *B, const int ldb, const void *beta, - void *C, const int ldc); -void cblas_ztrmm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_DIAG Diag, const int M, const int N, - const void *alpha, const void *A, const int lda, - void *B, const int ldb); -void cblas_ztrsm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, - const enum CBLAS_DIAG Diag, const int M, const int N, - const void *alpha, const void *A, const int lda, - void *B, const int ldb); +#define CBLAS_FUNC(name) BLAS_FUNC_EXPAND(name,BLAS_SYMBOL_PREFIX,,BLAS_SYMBOL_SUFFIX) +#define BLAS_FUNC(name) BLAS_FUNC_EXPAND(name,BLAS_SYMBOL_PREFIX,BLAS_FORTRAN_SUFFIX,BLAS_SYMBOL_SUFFIX) +#ifdef HAVE_BLAS_ILP64 +#define CBLAS_INT npy_int64 +#else +#define CBLAS_INT int +#endif -/* - * Routines with prefixes C and Z only - */ -void cblas_chemm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const int M, const int N, - const void *alpha, const void *A, const int lda, - const void *B, const int ldb, const void *beta, - void *C, const int ldc); -void cblas_cherk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const float alpha, const void *A, const int lda, - const float beta, void *C, const int ldc); -void cblas_cher2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const void *alpha, const void *A, const int lda, - const void *B, const int ldb, const float beta, - void *C, const int ldc); +#define BLASNAME(name) CBLAS_FUNC(name) +#define BLASINT CBLAS_INT -void cblas_zhemm(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, - const enum CBLAS_UPLO Uplo, const int M, const int N, - const void *alpha, const void *A, const int lda, - const void *B, const int ldb, const void *beta, - void *C, const int ldc); -void cblas_zherk(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const double alpha, const void *A, const int lda, - const double beta, void *C, const int ldc); -void cblas_zher2k(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, - const enum CBLAS_TRANSPOSE Trans, const int N, const int K, - const void *alpha, const void *A, const int lda, - const void *B, const int ldb, const double beta, - void *C, const int ldc); +#include "npy_cblas_base.h" -void cblas_xerbla(int p, const char *rout, const char *form, ...); +#undef BLASINT +#undef BLASNAME #ifdef __cplusplus } diff --git a/numpy/core/src/common/npy_cblas_base.h b/numpy/core/src/common/npy_cblas_base.h new file mode 100644 index 000000000..792b6f09e --- /dev/null +++ b/numpy/core/src/common/npy_cblas_base.h @@ -0,0 +1,557 @@ +/* + * This header provides numpy a consistent interface to CBLAS code. It is needed + * because not all providers of cblas provide cblas.h. For instance, MKL provides + * mkl_cblas.h and also typedefs the CBLAS_XXX enums. + */ + +/* + * =========================================================================== + * Prototypes for level 1 BLAS functions (complex are recast as routines) + * =========================================================================== + */ +float BLASNAME(cblas_sdsdot)(const BLASINT N, const float alpha, const float *X, + const BLASINT incX, const float *Y, const BLASINT incY); +double BLASNAME(cblas_dsdot)(const BLASINT N, const float *X, const BLASINT incX, const float *Y, + const BLASINT incY); +float BLASNAME(cblas_sdot)(const BLASINT N, const float *X, const BLASINT incX, + const float *Y, const BLASINT incY); +double BLASNAME(cblas_ddot)(const BLASINT N, const double *X, const BLASINT incX, + const double *Y, const BLASINT incY); + +/* + * Functions having prefixes Z and C only + */ +void BLASNAME(cblas_cdotu_sub)(const BLASINT N, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *dotu); +void BLASNAME(cblas_cdotc_sub)(const BLASINT N, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *dotc); + +void BLASNAME(cblas_zdotu_sub)(const BLASINT N, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *dotu); +void BLASNAME(cblas_zdotc_sub)(const BLASINT N, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *dotc); + + +/* + * Functions having prefixes S D SC DZ + */ +float BLASNAME(cblas_snrm2)(const BLASINT N, const float *X, const BLASINT incX); +float BLASNAME(cblas_sasum)(const BLASINT N, const float *X, const BLASINT incX); + +double BLASNAME(cblas_dnrm2)(const BLASINT N, const double *X, const BLASINT incX); +double BLASNAME(cblas_dasum)(const BLASINT N, const double *X, const BLASINT incX); + +float BLASNAME(cblas_scnrm2)(const BLASINT N, const void *X, const BLASINT incX); +float BLASNAME(cblas_scasum)(const BLASINT N, const void *X, const BLASINT incX); + +double BLASNAME(cblas_dznrm2)(const BLASINT N, const void *X, const BLASINT incX); +double BLASNAME(cblas_dzasum)(const BLASINT N, const void *X, const BLASINT incX); + + +/* + * Functions having standard 4 prefixes (S D C Z) + */ +CBLAS_INDEX BLASNAME(cblas_isamax)(const BLASINT N, const float *X, const BLASINT incX); +CBLAS_INDEX BLASNAME(cblas_idamax)(const BLASINT N, const double *X, const BLASINT incX); +CBLAS_INDEX BLASNAME(cblas_icamax)(const BLASINT N, const void *X, const BLASINT incX); +CBLAS_INDEX BLASNAME(cblas_izamax)(const BLASINT N, const void *X, const BLASINT incX); + +/* + * =========================================================================== + * Prototypes for level 1 BLAS routines + * =========================================================================== + */ + +/* + * Routines with standard 4 prefixes (s, d, c, z) + */ +void BLASNAME(cblas_sswap)(const BLASINT N, float *X, const BLASINT incX, + float *Y, const BLASINT incY); +void BLASNAME(cblas_scopy)(const BLASINT N, const float *X, const BLASINT incX, + float *Y, const BLASINT incY); +void BLASNAME(cblas_saxpy)(const BLASINT N, const float alpha, const float *X, + const BLASINT incX, float *Y, const BLASINT incY); + +void BLASNAME(cblas_dswap)(const BLASINT N, double *X, const BLASINT incX, + double *Y, const BLASINT incY); +void BLASNAME(cblas_dcopy)(const BLASINT N, const double *X, const BLASINT incX, + double *Y, const BLASINT incY); +void BLASNAME(cblas_daxpy)(const BLASINT N, const double alpha, const double *X, + const BLASINT incX, double *Y, const BLASINT incY); + +void BLASNAME(cblas_cswap)(const BLASINT N, void *X, const BLASINT incX, + void *Y, const BLASINT incY); +void BLASNAME(cblas_ccopy)(const BLASINT N, const void *X, const BLASINT incX, + void *Y, const BLASINT incY); +void BLASNAME(cblas_caxpy)(const BLASINT N, const void *alpha, const void *X, + const BLASINT incX, void *Y, const BLASINT incY); + +void BLASNAME(cblas_zswap)(const BLASINT N, void *X, const BLASINT incX, + void *Y, const BLASINT incY); +void BLASNAME(cblas_zcopy)(const BLASINT N, const void *X, const BLASINT incX, + void *Y, const BLASINT incY); +void BLASNAME(cblas_zaxpy)(const BLASINT N, const void *alpha, const void *X, + const BLASINT incX, void *Y, const BLASINT incY); + + +/* + * Routines with S and D prefix only + */ +void BLASNAME(cblas_srotg)(float *a, float *b, float *c, float *s); +void BLASNAME(cblas_srotmg)(float *d1, float *d2, float *b1, const float b2, float *P); +void BLASNAME(cblas_srot)(const BLASINT N, float *X, const BLASINT incX, + float *Y, const BLASINT incY, const float c, const float s); +void BLASNAME(cblas_srotm)(const BLASINT N, float *X, const BLASINT incX, + float *Y, const BLASINT incY, const float *P); + +void BLASNAME(cblas_drotg)(double *a, double *b, double *c, double *s); +void BLASNAME(cblas_drotmg)(double *d1, double *d2, double *b1, const double b2, double *P); +void BLASNAME(cblas_drot)(const BLASINT N, double *X, const BLASINT incX, + double *Y, const BLASINT incY, const double c, const double s); +void BLASNAME(cblas_drotm)(const BLASINT N, double *X, const BLASINT incX, + double *Y, const BLASINT incY, const double *P); + + +/* + * Routines with S D C Z CS and ZD prefixes + */ +void BLASNAME(cblas_sscal)(const BLASINT N, const float alpha, float *X, const BLASINT incX); +void BLASNAME(cblas_dscal)(const BLASINT N, const double alpha, double *X, const BLASINT incX); +void BLASNAME(cblas_cscal)(const BLASINT N, const void *alpha, void *X, const BLASINT incX); +void BLASNAME(cblas_zscal)(const BLASINT N, const void *alpha, void *X, const BLASINT incX); +void BLASNAME(cblas_csscal)(const BLASINT N, const float alpha, void *X, const BLASINT incX); +void BLASNAME(cblas_zdscal)(const BLASINT N, const double alpha, void *X, const BLASINT incX); + +/* + * =========================================================================== + * Prototypes for level 2 BLAS + * =========================================================================== + */ + +/* + * Routines with standard 4 prefixes (S, D, C, Z) + */ +void BLASNAME(cblas_sgemv)(const enum CBLAS_ORDER order, + const enum CBLAS_TRANSPOSE TransA, const BLASINT M, const BLASINT N, + const float alpha, const float *A, const BLASINT lda, + const float *X, const BLASINT incX, const float beta, + float *Y, const BLASINT incY); +void BLASNAME(cblas_sgbmv)(const enum CBLAS_ORDER order, + const enum CBLAS_TRANSPOSE TransA, const BLASINT M, const BLASINT N, + const BLASINT KL, const BLASINT KU, const float alpha, + const float *A, const BLASINT lda, const float *X, + const BLASINT incX, const float beta, float *Y, const BLASINT incY); +void BLASNAME(cblas_strmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const float *A, const BLASINT lda, + float *X, const BLASINT incX); +void BLASNAME(cblas_stbmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const BLASINT K, const float *A, const BLASINT lda, + float *X, const BLASINT incX); +void BLASNAME(cblas_stpmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const float *Ap, float *X, const BLASINT incX); +void BLASNAME(cblas_strsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const float *A, const BLASINT lda, float *X, + const BLASINT incX); +void BLASNAME(cblas_stbsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const BLASINT K, const float *A, const BLASINT lda, + float *X, const BLASINT incX); +void BLASNAME(cblas_stpsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const float *Ap, float *X, const BLASINT incX); + +void BLASNAME(cblas_dgemv)(const enum CBLAS_ORDER order, + const enum CBLAS_TRANSPOSE TransA, const BLASINT M, const BLASINT N, + const double alpha, const double *A, const BLASINT lda, + const double *X, const BLASINT incX, const double beta, + double *Y, const BLASINT incY); +void BLASNAME(cblas_dgbmv)(const enum CBLAS_ORDER order, + const enum CBLAS_TRANSPOSE TransA, const BLASINT M, const BLASINT N, + const BLASINT KL, const BLASINT KU, const double alpha, + const double *A, const BLASINT lda, const double *X, + const BLASINT incX, const double beta, double *Y, const BLASINT incY); +void BLASNAME(cblas_dtrmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const double *A, const BLASINT lda, + double *X, const BLASINT incX); +void BLASNAME(cblas_dtbmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const BLASINT K, const double *A, const BLASINT lda, + double *X, const BLASINT incX); +void BLASNAME(cblas_dtpmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const double *Ap, double *X, const BLASINT incX); +void BLASNAME(cblas_dtrsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const double *A, const BLASINT lda, double *X, + const BLASINT incX); +void BLASNAME(cblas_dtbsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const BLASINT K, const double *A, const BLASINT lda, + double *X, const BLASINT incX); +void BLASNAME(cblas_dtpsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const double *Ap, double *X, const BLASINT incX); + +void BLASNAME(cblas_cgemv)(const enum CBLAS_ORDER order, + const enum CBLAS_TRANSPOSE TransA, const BLASINT M, const BLASINT N, + const void *alpha, const void *A, const BLASINT lda, + const void *X, const BLASINT incX, const void *beta, + void *Y, const BLASINT incY); +void BLASNAME(cblas_cgbmv)(const enum CBLAS_ORDER order, + const enum CBLAS_TRANSPOSE TransA, const BLASINT M, const BLASINT N, + const BLASINT KL, const BLASINT KU, const void *alpha, + const void *A, const BLASINT lda, const void *X, + const BLASINT incX, const void *beta, void *Y, const BLASINT incY); +void BLASNAME(cblas_ctrmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const void *A, const BLASINT lda, + void *X, const BLASINT incX); +void BLASNAME(cblas_ctbmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const BLASINT K, const void *A, const BLASINT lda, + void *X, const BLASINT incX); +void BLASNAME(cblas_ctpmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const void *Ap, void *X, const BLASINT incX); +void BLASNAME(cblas_ctrsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const void *A, const BLASINT lda, void *X, + const BLASINT incX); +void BLASNAME(cblas_ctbsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const BLASINT K, const void *A, const BLASINT lda, + void *X, const BLASINT incX); +void BLASNAME(cblas_ctpsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const void *Ap, void *X, const BLASINT incX); + +void BLASNAME(cblas_zgemv)(const enum CBLAS_ORDER order, + const enum CBLAS_TRANSPOSE TransA, const BLASINT M, const BLASINT N, + const void *alpha, const void *A, const BLASINT lda, + const void *X, const BLASINT incX, const void *beta, + void *Y, const BLASINT incY); +void BLASNAME(cblas_zgbmv)(const enum CBLAS_ORDER order, + const enum CBLAS_TRANSPOSE TransA, const BLASINT M, const BLASINT N, + const BLASINT KL, const BLASINT KU, const void *alpha, + const void *A, const BLASINT lda, const void *X, + const BLASINT incX, const void *beta, void *Y, const BLASINT incY); +void BLASNAME(cblas_ztrmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const void *A, const BLASINT lda, + void *X, const BLASINT incX); +void BLASNAME(cblas_ztbmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const BLASINT K, const void *A, const BLASINT lda, + void *X, const BLASINT incX); +void BLASNAME(cblas_ztpmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const void *Ap, void *X, const BLASINT incX); +void BLASNAME(cblas_ztrsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const void *A, const BLASINT lda, void *X, + const BLASINT incX); +void BLASNAME(cblas_ztbsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const BLASINT K, const void *A, const BLASINT lda, + void *X, const BLASINT incX); +void BLASNAME(cblas_ztpsv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE TransA, const enum CBLAS_DIAG Diag, + const BLASINT N, const void *Ap, void *X, const BLASINT incX); + + +/* + * Routines with S and D prefixes only + */ +void BLASNAME(cblas_ssymv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const float alpha, const float *A, + const BLASINT lda, const float *X, const BLASINT incX, + const float beta, float *Y, const BLASINT incY); +void BLASNAME(cblas_ssbmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const BLASINT K, const float alpha, const float *A, + const BLASINT lda, const float *X, const BLASINT incX, + const float beta, float *Y, const BLASINT incY); +void BLASNAME(cblas_sspmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const float alpha, const float *Ap, + const float *X, const BLASINT incX, + const float beta, float *Y, const BLASINT incY); +void BLASNAME(cblas_sger)(const enum CBLAS_ORDER order, const BLASINT M, const BLASINT N, + const float alpha, const float *X, const BLASINT incX, + const float *Y, const BLASINT incY, float *A, const BLASINT lda); +void BLASNAME(cblas_ssyr)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const float alpha, const float *X, + const BLASINT incX, float *A, const BLASINT lda); +void BLASNAME(cblas_sspr)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const float alpha, const float *X, + const BLASINT incX, float *Ap); +void BLASNAME(cblas_ssyr2)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const float alpha, const float *X, + const BLASINT incX, const float *Y, const BLASINT incY, float *A, + const BLASINT lda); +void BLASNAME(cblas_sspr2)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const float alpha, const float *X, + const BLASINT incX, const float *Y, const BLASINT incY, float *A); + +void BLASNAME(cblas_dsymv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const double alpha, const double *A, + const BLASINT lda, const double *X, const BLASINT incX, + const double beta, double *Y, const BLASINT incY); +void BLASNAME(cblas_dsbmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const BLASINT K, const double alpha, const double *A, + const BLASINT lda, const double *X, const BLASINT incX, + const double beta, double *Y, const BLASINT incY); +void BLASNAME(cblas_dspmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const double alpha, const double *Ap, + const double *X, const BLASINT incX, + const double beta, double *Y, const BLASINT incY); +void BLASNAME(cblas_dger)(const enum CBLAS_ORDER order, const BLASINT M, const BLASINT N, + const double alpha, const double *X, const BLASINT incX, + const double *Y, const BLASINT incY, double *A, const BLASINT lda); +void BLASNAME(cblas_dsyr)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const double alpha, const double *X, + const BLASINT incX, double *A, const BLASINT lda); +void BLASNAME(cblas_dspr)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const double alpha, const double *X, + const BLASINT incX, double *Ap); +void BLASNAME(cblas_dsyr2)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const double alpha, const double *X, + const BLASINT incX, const double *Y, const BLASINT incY, double *A, + const BLASINT lda); +void BLASNAME(cblas_dspr2)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const double alpha, const double *X, + const BLASINT incX, const double *Y, const BLASINT incY, double *A); + + +/* + * Routines with C and Z prefixes only + */ +void BLASNAME(cblas_chemv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const void *alpha, const void *A, + const BLASINT lda, const void *X, const BLASINT incX, + const void *beta, void *Y, const BLASINT incY); +void BLASNAME(cblas_chbmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const BLASINT K, const void *alpha, const void *A, + const BLASINT lda, const void *X, const BLASINT incX, + const void *beta, void *Y, const BLASINT incY); +void BLASNAME(cblas_chpmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const void *alpha, const void *Ap, + const void *X, const BLASINT incX, + const void *beta, void *Y, const BLASINT incY); +void BLASNAME(cblas_cgeru)(const enum CBLAS_ORDER order, const BLASINT M, const BLASINT N, + const void *alpha, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *A, const BLASINT lda); +void BLASNAME(cblas_cgerc)(const enum CBLAS_ORDER order, const BLASINT M, const BLASINT N, + const void *alpha, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *A, const BLASINT lda); +void BLASNAME(cblas_cher)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const float alpha, const void *X, const BLASINT incX, + void *A, const BLASINT lda); +void BLASNAME(cblas_chpr)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const float alpha, const void *X, + const BLASINT incX, void *A); +void BLASNAME(cblas_cher2)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const BLASINT N, + const void *alpha, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *A, const BLASINT lda); +void BLASNAME(cblas_chpr2)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const BLASINT N, + const void *alpha, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *Ap); + +void BLASNAME(cblas_zhemv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const void *alpha, const void *A, + const BLASINT lda, const void *X, const BLASINT incX, + const void *beta, void *Y, const BLASINT incY); +void BLASNAME(cblas_zhbmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const BLASINT K, const void *alpha, const void *A, + const BLASINT lda, const void *X, const BLASINT incX, + const void *beta, void *Y, const BLASINT incY); +void BLASNAME(cblas_zhpmv)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const void *alpha, const void *Ap, + const void *X, const BLASINT incX, + const void *beta, void *Y, const BLASINT incY); +void BLASNAME(cblas_zgeru)(const enum CBLAS_ORDER order, const BLASINT M, const BLASINT N, + const void *alpha, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *A, const BLASINT lda); +void BLASNAME(cblas_zgerc)(const enum CBLAS_ORDER order, const BLASINT M, const BLASINT N, + const void *alpha, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *A, const BLASINT lda); +void BLASNAME(cblas_zher)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const double alpha, const void *X, const BLASINT incX, + void *A, const BLASINT lda); +void BLASNAME(cblas_zhpr)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, + const BLASINT N, const double alpha, const void *X, + const BLASINT incX, void *A); +void BLASNAME(cblas_zher2)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const BLASINT N, + const void *alpha, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *A, const BLASINT lda); +void BLASNAME(cblas_zhpr2)(const enum CBLAS_ORDER order, const enum CBLAS_UPLO Uplo, const BLASINT N, + const void *alpha, const void *X, const BLASINT incX, + const void *Y, const BLASINT incY, void *Ap); + +/* + * =========================================================================== + * Prototypes for level 3 BLAS + * =========================================================================== + */ + +/* + * Routines with standard 4 prefixes (S, D, C, Z) + */ +void BLASNAME(cblas_sgemm)(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_TRANSPOSE TransB, const BLASINT M, const BLASINT N, + const BLASINT K, const float alpha, const float *A, + const BLASINT lda, const float *B, const BLASINT ldb, + const float beta, float *C, const BLASINT ldc); +void BLASNAME(cblas_ssymm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const BLASINT M, const BLASINT N, + const float alpha, const float *A, const BLASINT lda, + const float *B, const BLASINT ldb, const float beta, + float *C, const BLASINT ldc); +void BLASNAME(cblas_ssyrk)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const float alpha, const float *A, const BLASINT lda, + const float beta, float *C, const BLASINT ldc); +void BLASNAME(cblas_ssyr2k)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const float alpha, const float *A, const BLASINT lda, + const float *B, const BLASINT ldb, const float beta, + float *C, const BLASINT ldc); +void BLASNAME(cblas_strmm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const BLASINT M, const BLASINT N, + const float alpha, const float *A, const BLASINT lda, + float *B, const BLASINT ldb); +void BLASNAME(cblas_strsm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const BLASINT M, const BLASINT N, + const float alpha, const float *A, const BLASINT lda, + float *B, const BLASINT ldb); + +void BLASNAME(cblas_dgemm)(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_TRANSPOSE TransB, const BLASINT M, const BLASINT N, + const BLASINT K, const double alpha, const double *A, + const BLASINT lda, const double *B, const BLASINT ldb, + const double beta, double *C, const BLASINT ldc); +void BLASNAME(cblas_dsymm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const BLASINT M, const BLASINT N, + const double alpha, const double *A, const BLASINT lda, + const double *B, const BLASINT ldb, const double beta, + double *C, const BLASINT ldc); +void BLASNAME(cblas_dsyrk)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const double alpha, const double *A, const BLASINT lda, + const double beta, double *C, const BLASINT ldc); +void BLASNAME(cblas_dsyr2k)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const double alpha, const double *A, const BLASINT lda, + const double *B, const BLASINT ldb, const double beta, + double *C, const BLASINT ldc); +void BLASNAME(cblas_dtrmm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const BLASINT M, const BLASINT N, + const double alpha, const double *A, const BLASINT lda, + double *B, const BLASINT ldb); +void BLASNAME(cblas_dtrsm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const BLASINT M, const BLASINT N, + const double alpha, const double *A, const BLASINT lda, + double *B, const BLASINT ldb); + +void BLASNAME(cblas_cgemm)(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_TRANSPOSE TransB, const BLASINT M, const BLASINT N, + const BLASINT K, const void *alpha, const void *A, + const BLASINT lda, const void *B, const BLASINT ldb, + const void *beta, void *C, const BLASINT ldc); +void BLASNAME(cblas_csymm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const BLASINT M, const BLASINT N, + const void *alpha, const void *A, const BLASINT lda, + const void *B, const BLASINT ldb, const void *beta, + void *C, const BLASINT ldc); +void BLASNAME(cblas_csyrk)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const void *alpha, const void *A, const BLASINT lda, + const void *beta, void *C, const BLASINT ldc); +void BLASNAME(cblas_csyr2k)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const void *alpha, const void *A, const BLASINT lda, + const void *B, const BLASINT ldb, const void *beta, + void *C, const BLASINT ldc); +void BLASNAME(cblas_ctrmm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const BLASINT M, const BLASINT N, + const void *alpha, const void *A, const BLASINT lda, + void *B, const BLASINT ldb); +void BLASNAME(cblas_ctrsm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const BLASINT M, const BLASINT N, + const void *alpha, const void *A, const BLASINT lda, + void *B, const BLASINT ldb); + +void BLASNAME(cblas_zgemm)(const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_TRANSPOSE TransB, const BLASINT M, const BLASINT N, + const BLASINT K, const void *alpha, const void *A, + const BLASINT lda, const void *B, const BLASINT ldb, + const void *beta, void *C, const BLASINT ldc); +void BLASNAME(cblas_zsymm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const BLASINT M, const BLASINT N, + const void *alpha, const void *A, const BLASINT lda, + const void *B, const BLASINT ldb, const void *beta, + void *C, const BLASINT ldc); +void BLASNAME(cblas_zsyrk)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const void *alpha, const void *A, const BLASINT lda, + const void *beta, void *C, const BLASINT ldc); +void BLASNAME(cblas_zsyr2k)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const void *alpha, const void *A, const BLASINT lda, + const void *B, const BLASINT ldb, const void *beta, + void *C, const BLASINT ldc); +void BLASNAME(cblas_ztrmm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const BLASINT M, const BLASINT N, + const void *alpha, const void *A, const BLASINT lda, + void *B, const BLASINT ldb); +void BLASNAME(cblas_ztrsm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_DIAG Diag, const BLASINT M, const BLASINT N, + const void *alpha, const void *A, const BLASINT lda, + void *B, const BLASINT ldb); + + +/* + * Routines with prefixes C and Z only + */ +void BLASNAME(cblas_chemm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const BLASINT M, const BLASINT N, + const void *alpha, const void *A, const BLASINT lda, + const void *B, const BLASINT ldb, const void *beta, + void *C, const BLASINT ldc); +void BLASNAME(cblas_cherk)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const float alpha, const void *A, const BLASINT lda, + const float beta, void *C, const BLASINT ldc); +void BLASNAME(cblas_cher2k)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const void *alpha, const void *A, const BLASINT lda, + const void *B, const BLASINT ldb, const float beta, + void *C, const BLASINT ldc); + +void BLASNAME(cblas_zhemm)(const enum CBLAS_ORDER Order, const enum CBLAS_SIDE Side, + const enum CBLAS_UPLO Uplo, const BLASINT M, const BLASINT N, + const void *alpha, const void *A, const BLASINT lda, + const void *B, const BLASINT ldb, const void *beta, + void *C, const BLASINT ldc); +void BLASNAME(cblas_zherk)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const double alpha, const void *A, const BLASINT lda, + const double beta, void *C, const BLASINT ldc); +void BLASNAME(cblas_zher2k)(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const enum CBLAS_TRANSPOSE Trans, const BLASINT N, const BLASINT K, + const void *alpha, const void *A, const BLASINT lda, + const void *B, const BLASINT ldb, const double beta, + void *C, const BLASINT ldc); + +void BLASNAME(cblas_xerbla)(BLASINT p, const char *rout, const char *form, ...); diff --git a/numpy/core/src/common/npy_partition.h.src b/numpy/core/src/common/npy_partition.h.src index a22cf911c..97dc2536b 100644 --- a/numpy/core/src/common/npy_partition.h.src +++ b/numpy/core/src/common/npy_partition.h.src @@ -113,9 +113,6 @@ get_argpartition_func(int type, NPY_SELECTKIND which) npy_intp i; npy_intp ntypes = ARRAY_SIZE(_part_map); - if (which >= NPY_NSELECTS) { - return NULL; - } for (i = 0; i < ntypes; i++) { if (type == _part_map[i].typenum) { return _part_map[i].argpart[which]; diff --git a/numpy/core/src/common/python_xerbla.c b/numpy/core/src/common/python_xerbla.c index bdf0b9058..fe2f718b2 100644 --- a/numpy/core/src/common/python_xerbla.c +++ b/numpy/core/src/common/python_xerbla.c @@ -1,10 +1,6 @@ #include "Python.h" - -/* - * From f2c.h, this should be safe unless fortran is set to use 64 - * bit integers. We don't seem to have any good way to detect that. - */ -typedef int integer; +#include "numpy/npy_common.h" +#include "npy_cblas.h" /* From the original manpage: @@ -23,7 +19,7 @@ typedef int integer; info: Number of the invalid parameter. */ -int xerbla_(char *srname, integer *info) +CBLAS_INT BLAS_FUNC(xerbla)(char *srname, CBLAS_INT *info) { static const char format[] = "On entry to %.*s" \ " parameter number %d had an illegal value"; @@ -41,7 +37,7 @@ int xerbla_(char *srname, integer *info) #ifdef WITH_THREAD save = PyGILState_Ensure(); #endif - PyOS_snprintf(buf, sizeof(buf), format, len, srname, *info); + PyOS_snprintf(buf, sizeof(buf), format, len, srname, (int)*info); PyErr_SetString(PyExc_ValueError, buf); #ifdef WITH_THREAD PyGILState_Release(save); diff --git a/numpy/core/src/common/ufunc_override.c b/numpy/core/src/common/ufunc_override.c index 89f08a9cb..3f699bcdd 100644 --- a/numpy/core/src/common/ufunc_override.c +++ b/numpy/core/src/common/ufunc_override.c @@ -36,6 +36,9 @@ PyUFuncOverride_GetNonDefaultArrayUfunc(PyObject *obj) */ cls_array_ufunc = PyArray_LookupSpecial(obj, "__array_ufunc__"); if (cls_array_ufunc == NULL) { + if (PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } return NULL; } /* Ignore if the same as ndarray.__array_ufunc__ */ diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index 9061c0518..fa2efb428 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -593,6 +593,25 @@ fail: return NULL; } +/* + * Helper to test fromstring of 0 terminated strings, as the C-API supports + * the -1 length identifier. + */ +static PyObject * +fromstring_null_term_c_api(PyObject *dummy, PyObject *byte_obj) +{ + char *string; + PyArray_Descr *descr; + + string = PyBytes_AsString(byte_obj); + if (string == NULL) { + return NULL; + } + descr = PyArray_DescrNewFromType(NPY_FLOAT64); + return PyArray_FromString(string, -1, descr, -1, " "); +} + + /* check no elison for avoided increfs */ static PyObject * incref_elide(PyObject *dummy, PyObject *args) @@ -656,6 +675,43 @@ npy_updateifcopy_deprecation(PyObject* NPY_UNUSED(self), PyObject* args) Py_RETURN_NONE; } +/* used to test PyArray_As1D usage emits not implemented error */ +static PyObject* +npy_pyarrayas1d_deprecation(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) +{ + PyObject *op = Py_BuildValue("i", 42); + PyObject *result = op; + int dim = 4; + double arg[2] = {1, 2}; + int temp = PyArray_As1D(&result, (char **)&arg, &dim, NPY_DOUBLE); + if (temp < 0) { + Py_DECREF(op); + return NULL; + } + /* op != result */ + Py_DECREF(op); + return result; +} + +/* used to test PyArray_As2D usage emits not implemented error */ +static PyObject* +npy_pyarrayas2d_deprecation(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) +{ + PyObject *op = Py_BuildValue("i", 42); + PyObject *result = op; + int dim1 = 4; + int dim2 = 6; + double arg[2][2] = {{1, 2}, {3, 4}}; + int temp = PyArray_As2D(&result, (char ***)&arg, &dim1, &dim2, NPY_DOUBLE); + if (temp < 0) { + Py_DECREF(op); + return NULL; + } + /* op != result */ + Py_DECREF(op); + return result; +} + /* used to create array with WRITEBACKIFCOPY flag */ static PyObject* npy_create_writebackifcopy(PyObject* NPY_UNUSED(self), PyObject* args) @@ -857,6 +913,7 @@ static PyObject* get_c_wrapping_array(PyObject* NPY_UNUSED(self), PyObject* arg) { int writeable, flags; + PyArray_Descr *descr; npy_intp zero = 0; writeable = PyObject_IsTrue(arg); @@ -866,7 +923,8 @@ get_c_wrapping_array(PyObject* NPY_UNUSED(self), PyObject* arg) flags = writeable ? NPY_ARRAY_WRITEABLE : 0; /* Create an empty array (which points to a random place) */ - return PyArray_NewFromDescr(&PyArray_Type, PyArray_DescrFromType(NPY_INTP), + descr = PyArray_DescrNewFromType(NPY_INTP); + return PyArray_NewFromDescr(&PyArray_Type, descr, 1, &zero, NULL, &zero, flags, NULL); } @@ -928,6 +986,7 @@ test_as_c_array(PyObject *NPY_UNUSED(self), PyObject *args) num_dims = PyArray_NDIM(array_obj); descr = PyArray_DESCR(array_obj); + Py_INCREF(descr); /* PyArray_AsCArray steals a reference to this */ switch (num_dims) { case 1: @@ -970,6 +1029,7 @@ test_as_c_array(PyObject *NPY_UNUSED(self), PyObject *args) PyArray_Free((PyObject *) array_obj, (void *) array3); break; default: + Py_DECREF(descr); PyErr_SetString(PyExc_ValueError, "array.ndim not in [1, 3]"); return NULL; } @@ -1263,7 +1323,9 @@ pylong_from_int128(npy_extint128_t value) } Py_DECREF(val); + Py_DECREF(val_64); val = tmp; + val_64 = NULL; tmp = PyLong_FromUnsignedLongLong(value.lo); if (tmp == NULL) { @@ -1923,6 +1985,9 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"test_inplace_increment", inplace_increment, METH_VARARGS, NULL}, + {"fromstring_null_term_c_api", + fromstring_null_term_c_api, + METH_O, NULL}, {"incref_elide", incref_elide, METH_VARARGS, NULL}, @@ -1935,6 +2000,12 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"npy_updateifcopy_deprecation", npy_updateifcopy_deprecation, METH_O, NULL}, + {"npy_pyarrayas1d_deprecation", + npy_pyarrayas1d_deprecation, + METH_NOARGS, NULL}, + {"npy_pyarrayas2d_deprecation", + npy_pyarrayas2d_deprecation, + METH_NOARGS, NULL}, {"npy_create_writebackifcopy", npy_create_writebackifcopy, METH_O, NULL}, diff --git a/numpy/core/src/multiarray/arrayfunction_override.c b/numpy/core/src/multiarray/arrayfunction_override.c index 62e597764..9ea8efdd9 100644 --- a/numpy/core/src/multiarray/arrayfunction_override.c +++ b/numpy/core/src/multiarray/arrayfunction_override.c @@ -26,6 +26,7 @@ static PyObject * get_array_function(PyObject *obj) { static PyObject *ndarray_array_function = NULL; + PyObject *array_function; if (ndarray_array_function == NULL) { ndarray_array_function = get_ndarray_array_function(); @@ -37,7 +38,12 @@ get_array_function(PyObject *obj) return ndarray_array_function; } - return PyArray_LookupSpecial(obj, "__array_function__"); + array_function = PyArray_LookupSpecial(obj, "__array_function__"); + if (array_function == NULL && PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } + + return array_function; } diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index ebcb9b0b0..a5cebfbd8 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -462,7 +462,7 @@ WARN_IN_DEALLOC(PyObject* warning, const char * msg) { PyErr_WriteUnraisable(Py_None); } } -}; +} /* array object functions */ @@ -483,10 +483,11 @@ array_dealloc(PyArrayObject *self) char const * msg = "WRITEBACKIFCOPY detected in array_dealloc. " " Required call to PyArray_ResolveWritebackIfCopy or " "PyArray_DiscardWritebackIfCopy is missing."; - Py_INCREF(self); /* hold on to self in next call since if - * refcount == 0 it will recurse back into - *array_dealloc - */ + /* + * prevent reaching 0 twice and thus recursing into dealloc. + * Increasing sys.gettotalrefcount, but path should not be taken. + */ + Py_INCREF(self); WARN_IN_DEALLOC(PyExc_RuntimeWarning, msg); retval = PyArray_ResolveWritebackIfCopy(self); if (retval < 0) @@ -500,10 +501,11 @@ array_dealloc(PyArrayObject *self) char const * msg = "UPDATEIFCOPY detected in array_dealloc. " " Required call to PyArray_ResolveWritebackIfCopy or " "PyArray_DiscardWritebackIfCopy is missing"; - Py_INCREF(self); /* hold on to self in next call since if - * refcount == 0 it will recurse back into - *array_dealloc - */ + /* + * prevent reaching 0 twice and thus recursing into dealloc. + * Increasing sys.gettotalrefcount, but path should not be taken. + */ + Py_INCREF(self); /* 2017-Nov-10 1.14 */ WARN_IN_DEALLOC(PyExc_DeprecationWarning, msg); retval = PyArray_ResolveWritebackIfCopy(self); @@ -523,12 +525,7 @@ array_dealloc(PyArrayObject *self) if ((fa->flags & NPY_ARRAY_OWNDATA) && fa->data) { /* Free internal references if an Object array */ if (PyDataType_FLAGCHK(fa->descr, NPY_ITEM_REFCOUNT)) { - Py_INCREF(self); /*hold on to self */ PyArray_XDECREF(self); - /* - * Don't need to DECREF -- because we are deleting - * self already... - */ } npy_free_cache(fa->data, PyArray_NBYTES(self)); } @@ -560,7 +557,7 @@ PyArray_DebugPrint(PyArrayObject *obj) printf(" ndim : %d\n", fobj->nd); printf(" shape :"); for (i = 0; i < fobj->nd; ++i) { - printf(" %d", (int)fobj->dimensions[i]); + printf(" %" NPY_INTP_FMT, fobj->dimensions[i]); } printf("\n"); @@ -570,7 +567,7 @@ PyArray_DebugPrint(PyArrayObject *obj) printf(" data : %p\n", fobj->data); printf(" strides:"); for (i = 0; i < fobj->nd; ++i) { - printf(" %d", (int)fobj->strides[i]); + printf(" %" NPY_INTP_FMT, fobj->strides[i]); } printf("\n"); @@ -610,7 +607,7 @@ PyArray_DebugPrint(PyArrayObject *obj) * TO BE REMOVED - NOT USED INTERNALLY. */ NPY_NO_EXPORT void -PyArray_SetDatetimeParseFunction(PyObject *op) +PyArray_SetDatetimeParseFunction(PyObject *NPY_UNUSED(op)) { } @@ -633,7 +630,7 @@ PyArray_CompareUCS4(npy_ucs4 *s1, npy_ucs4 *s2, size_t len) /*NUMPY_API */ NPY_NO_EXPORT int -PyArray_CompareString(char *s1, char *s2, size_t len) +PyArray_CompareString(const char *s1, const char *s2, size_t len) { const unsigned char *c1 = (unsigned char *)s1; const unsigned char *c2 = (unsigned char *)s2; @@ -1203,15 +1200,28 @@ _void_compare(PyArrayObject *self, PyArrayObject *other, int cmp_op) } } if (res == NULL && !PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, "No fields found."); + /* these dtypes had no fields. Use a MultiIter to broadcast them + * to an output array, and fill with True (for EQ)*/ + PyArrayMultiIterObject *mit = (PyArrayMultiIterObject *) + PyArray_MultiIterNew(2, self, other); + if (mit == NULL) { + return NULL; + } + + res = PyArray_NewFromDescr(&PyArray_Type, + PyArray_DescrFromType(NPY_BOOL), + mit->nd, mit->dimensions, + NULL, NULL, 0, NULL); + Py_DECREF(mit); + if (res) { + PyArray_FILLWBYTE((PyArrayObject *)res, + cmp_op == Py_EQ ? 1 : 0); + } } return res; } else { - /* - * compare as a string. Assumes self and - * other have same descr->type - */ + /* compare as a string. Assumes self and other have same descr->type */ return _strings_richcompare(self, other, cmp_op, 0); } } diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index ef51112d6..9e108e3e1 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -221,9 +221,7 @@ static int if (PySequence_NoString_Check(op)) { PyErr_SetString(PyExc_ValueError, "setting an array element with a sequence."); - Py_DECREF(type); - Py_XDECREF(value); - Py_XDECREF(traceback); + npy_PyErr_ChainExceptionsCause(type, value, traceback); } else { PyErr_Restore(type, value, traceback); @@ -1083,6 +1081,7 @@ TIMEDELTA_setitem(PyObject *op, void *ov, void *vap) * npy_long, npy_ulong, npy_longlong, npy_ulonglong, * npy_float, npy_double, npy_longdouble, * npy_datetime, npy_timedelta# + * #supports_nat = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# */ /**begin repeat1 @@ -1094,6 +1093,7 @@ TIMEDELTA_setitem(PyObject *op, void *ov, void *vap) * npy_long, npy_ulong, npy_longlong, npy_ulonglong, * npy_float, npy_double, npy_longdouble, * npy_datetime, npy_timedelta# + * #floatingpoint = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0# */ static void @FROMTYPE@_to_@TOTYPE@(void *input, void *output, npy_intp n, @@ -1103,7 +1103,15 @@ static void @totype@ *op = output; while (n--) { - *op++ = (@totype@)*ip++; + @fromtype@ f = *ip++; + @totype@ t = (@totype@)f; +#if @supports_nat@ && @floatingpoint@ + /* Avoid undefined behaviour for NaN -> NaT */ + if (npy_isnan(f)) { + t = (@totype@)NPY_DATETIME_NAT; + } +#endif + *op++ = t; } } /**end repeat1**/ @@ -1121,7 +1129,15 @@ static void @totype@ *op = output; while (n--) { - *op++ = (@totype@)*ip; + @fromtype@ f = *ip; + @totype@ t = (@totype@)f; +#if @supports_nat@ + /* Avoid undefined behaviour for NaN -> NaT */ + if (npy_isnan(f)) { + t = (@totype@)NPY_DATETIME_NAT; + } +#endif + *op++ = t; ip += 2; } } @@ -1759,7 +1775,58 @@ BOOL_scan(FILE *fp, npy_bool *ip, void *NPY_UNUSED(ignore), } /**begin repeat - * #fname = CFLOAT, CDOUBLE, CLONGDOUBLE, + * #fname = CFLOAT, CDOUBLE# + * #type = npy_cfloat, npy_cdouble# + */ +static int +@fname@_scan(FILE *fp, @type@ *ip, void *NPY_UNUSED(ignore), + PyArray_Descr *NPY_UNUSED(ignored)) +{ + double result; + int ret_real, ret_imag; + + ret_real = NumPyOS_ascii_ftolf(fp, &result); + @type@ output; + // Peek next character + char next = getc(fp); + if ((next == '+') || (next == '-')) { + // Imaginary component specified + output.real = result; + // Revert peek and read imaginary component + ungetc(next, fp); + ret_imag = NumPyOS_ascii_ftolf(fp, &result); + // Peak next character + next = getc(fp); + if ((ret_imag == 1) && (next == 'j')) { + // If read is successful and the immediate following char is j + output.imag = result; + } + else { + output.imag = 0; + // Push an invalid char to trigger the not everything is read error + ungetc('a', fp); + } + } + else if (next == 'j') { + // Real component not specified + output.real = 0; + output.imag = result; + } + else { + // Imaginary component not specified + output.real = result; + output.imag = 0.; + // Next character is not + / - / j. Revert peek. + ungetc(next, fp); + } + *(@type@ *)ip = output; + return ret_real; +} +/**end repeat**/ + + +/**begin repeat + * #fname = CLONGDOUBLE, * OBJECT, STRING, UNICODE, VOID, * DATETIME, TIMEDELTA# */ @@ -1851,7 +1918,60 @@ BOOL_fromstr(char *str, void *ip, char **endptr, } /**begin repeat - * #fname = CFLOAT, CDOUBLE, CLONGDOUBLE, + * #fname = CFLOAT, CDOUBLE# + * #type = npy_cfloat, npy_cdouble# + */ +static int +@fname@_fromstr(char *str, void *ip, char **endptr, + PyArray_Descr *NPY_UNUSED(ignore)) +{ + double result; + + result = NumPyOS_ascii_strtod(str, endptr); + @type@ output; + + if (endptr && ((*endptr[0] == '+') || (*endptr[0] == '-'))) { + // Imaginary component specified + output.real = result; + // Reading imaginary component + char **prev = endptr; + str = *endptr; + result = NumPyOS_ascii_strtod(str, endptr); + if (endptr && *endptr[0] == 'j') { + // Read is successful if the immediate following char is j + output.imag = result; + // Skip j + ++*endptr; + } + else { + /* + * Set endptr to previous char to trigger the not everything is + * read error + */ + endptr = prev; + output.imag = 0; + } + } + else if (endptr && *endptr[0] == 'j') { + // Real component not specified + output.real = 0; + output.imag = result; + // Skip j + ++*endptr; + } + else { + // Imaginary component not specified + output.real = result; + output.imag = 0.; + } + *(@type@ *)ip = output; + return 0; +} +/**end repeat**/ + + +/**begin repeat + * #fname = CLONGDOUBLE, * OBJECT, STRING, UNICODE, VOID# */ @@ -3080,6 +3200,7 @@ BOOL_argmax(npy_bool *ip, npy_intp n, npy_intp *max_ind, * #le = _LESS_THAN_OR_EQUAL*10, npy_half_le, _LESS_THAN_OR_EQUAL*8# * #iscomplex = 0*14, 1*3, 0*2# * #incr = ip++*14, ip+=2*3, ip++*2# + * #isdatetime = 0*17, 1*2# */ static int @fname@_argmax(@type@ *ip, npy_intp n, npy_intp *max_ind, @@ -3105,6 +3226,12 @@ static int return 0; } #endif +#if @isdatetime@ + if (mp == NPY_DATETIME_NAT) { + /* NaT encountered, it's maximal */ + return 0; + } +#endif for (i = 1; i < n; i++) { @incr@; @@ -3124,6 +3251,13 @@ static int } } #else +#if @isdatetime@ + if (*ip == NPY_DATETIME_NAT) { + /* NaT encountered, it's maximal */ + *max_ind = i; + break; + } +#endif if (!@le@(*ip, mp)) { /* negated, for correct nan handling */ mp = *ip; *max_ind = i; @@ -3160,16 +3294,19 @@ BOOL_argmin(npy_bool *ip, npy_intp n, npy_intp *min_ind, * #fname = BYTE, UBYTE, SHORT, USHORT, INT, UINT, * LONG, ULONG, LONGLONG, ULONGLONG, * HALF, FLOAT, DOUBLE, LONGDOUBLE, - * CFLOAT, CDOUBLE, CLONGDOUBLE# + * CFLOAT, CDOUBLE, CLONGDOUBLE, + * DATETIME, TIMEDELTA# * #type = npy_byte, npy_ubyte, npy_short, npy_ushort, npy_int, npy_uint, * npy_long, npy_ulong, npy_longlong, npy_ulonglong, * npy_half, npy_float, npy_double, npy_longdouble, - * npy_float, npy_double, npy_longdouble# - * #isfloat = 0*10, 1*7# - * #isnan = nop*10, npy_half_isnan, npy_isnan*6# - * #le = _LESS_THAN_OR_EQUAL*10, npy_half_le, _LESS_THAN_OR_EQUAL*6# - * #iscomplex = 0*14, 1*3# - * #incr = ip++*14, ip+=2*3# + * npy_float, npy_double, npy_longdouble, + * npy_datetime, npy_timedelta# + * #isfloat = 0*10, 1*7, 0*2# + * #isnan = nop*10, npy_half_isnan, npy_isnan*6, nop*2# + * #le = _LESS_THAN_OR_EQUAL*10, npy_half_le, _LESS_THAN_OR_EQUAL*8# + * #iscomplex = 0*14, 1*3, 0*2# + * #incr = ip++*14, ip+=2*3, ip++*2# + * #isdatetime = 0*17, 1*2# */ static int @fname@_argmin(@type@ *ip, npy_intp n, npy_intp *min_ind, @@ -3195,6 +3332,12 @@ static int return 0; } #endif +#if @isdatetime@ + if (mp == NPY_DATETIME_NAT) { + /* NaT encountered, it's minimal */ + return 0; + } +#endif for (i = 1; i < n; i++) { @incr@; @@ -3214,6 +3357,13 @@ static int } } #else +#if @isdatetime@ + if (*ip == NPY_DATETIME_NAT) { + /* NaT encountered, it's minimal */ + *min_ind = i; + break; + } +#endif if (!@le@(mp, *ip)) { /* negated, for correct nan handling */ mp = *ip; *min_ind = i; @@ -3233,43 +3383,6 @@ static int #undef _LESS_THAN_OR_EQUAL -/**begin repeat - * - * #fname = DATETIME, TIMEDELTA# - * #type = npy_datetime, npy_timedelta# - */ -static int -@fname@_argmin(@type@ *ip, npy_intp n, npy_intp *min_ind, - PyArrayObject *NPY_UNUSED(aip)) -{ - /* NPY_DATETIME_NAT is smaller than every other value, we skip - * it for consistency with min(). - */ - npy_intp i; - @type@ mp = NPY_DATETIME_NAT; - - i = 0; - while (i < n && mp == NPY_DATETIME_NAT) { - mp = ip[i]; - i++; - } - if (i == n) { - /* All NaTs: return 0 */ - *min_ind = 0; - return 0; - } - *min_ind = i - 1; - for (; i < n; i++) { - if (mp > ip[i] && ip[i] != NPY_DATETIME_NAT) { - mp = ip[i]; - *min_ind = i; - } - } - return 0; -} - -/**end repeat**/ - static int OBJECT_argmax(PyObject **ip, npy_intp n, npy_intp *max_ind, PyArrayObject *NPY_UNUSED(aip)) @@ -3422,17 +3535,17 @@ NPY_NO_EXPORT void npy_intp n, void *NPY_UNUSED(ignore)) { #if defined(HAVE_CBLAS) - int is1b = blas_stride(is1, sizeof(@type@)); - int is2b = blas_stride(is2, sizeof(@type@)); + CBLAS_INT is1b = blas_stride(is1, sizeof(@type@)); + CBLAS_INT is2b = blas_stride(is2, sizeof(@type@)); if (is1b && is2b) { double sum = 0.; /* double for stability */ while (n > 0) { - int chunk = n < NPY_CBLAS_CHUNK ? n : NPY_CBLAS_CHUNK; + CBLAS_INT chunk = n < NPY_CBLAS_CHUNK ? n : NPY_CBLAS_CHUNK; - sum += cblas_@prefix@dot(chunk, + sum += CBLAS_FUNC(cblas_@prefix@dot)(chunk, (@type@ *) ip1, is1b, (@type@ *) ip2, is2b); /* use char strides here */ @@ -3471,17 +3584,17 @@ NPY_NO_EXPORT void char *op, npy_intp n, void *NPY_UNUSED(ignore)) { #if defined(HAVE_CBLAS) - int is1b = blas_stride(is1, sizeof(@ctype@)); - int is2b = blas_stride(is2, sizeof(@ctype@)); + CBLAS_INT is1b = blas_stride(is1, sizeof(@ctype@)); + CBLAS_INT is2b = blas_stride(is2, sizeof(@ctype@)); if (is1b && is2b) { double sum[2] = {0., 0.}; /* double for stability */ while (n > 0) { - int chunk = n < NPY_CBLAS_CHUNK ? n : NPY_CBLAS_CHUNK; + CBLAS_INT chunk = n < NPY_CBLAS_CHUNK ? n : NPY_CBLAS_CHUNK; @type@ tmp[2]; - cblas_@prefix@dotu_sub((int)n, ip1, is1b, ip2, is2b, tmp); + CBLAS_FUNC(cblas_@prefix@dotu_sub)((CBLAS_INT)n, ip1, is1b, ip2, is2b, tmp); sum[0] += (double)tmp[0]; sum[1] += (double)tmp[1]; /* use char strides here */ diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 6b8cac9a7..c991f7428 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -147,7 +147,6 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, if (dtype == NULL) { goto fail; } - Py_INCREF(dtype); goto promote_types; } /* Check if it's a NumPy scalar */ @@ -214,6 +213,10 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, int itemsize; PyObject *temp; + /* dtype is not used in this (string discovery) branch */ + Py_DECREF(dtype); + dtype = NULL; + if (string_type == NPY_STRING) { if ((temp = PyObject_Str(obj)) == NULL) { goto fail; @@ -364,6 +367,10 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, } Py_DECREF(ip); } + else if (PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } + /* The array struct interface */ ip = PyArray_LookupSpecial_OnInstance(obj, "__array_struct__"); @@ -386,6 +393,9 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, } Py_DECREF(ip); } + else if (PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } /* The old buffer interface */ #if !defined(NPY_PY3K) @@ -416,6 +426,9 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, goto fail; } } + else if (PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } /* * If we reached the maximum recursion depth without hitting one diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index 487d530a1..7eee9ddc5 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -303,7 +303,11 @@ blas_stride(npy_intp stride, unsigned itemsize) */ if (stride > 0 && npy_is_aligned((void *)stride, itemsize)) { stride /= itemsize; +#ifndef HAVE_BLAS_ILP64 if (stride <= INT_MAX) { +#else + if (stride <= NPY_MAX_INT64) { +#endif return stride; } } @@ -314,7 +318,11 @@ blas_stride(npy_intp stride, unsigned itemsize) * Define a chunksize for CBLAS. CBLAS counts in integers. */ #if NPY_MAX_INTP > INT_MAX -# define NPY_CBLAS_CHUNK (INT_MAX / 2 + 1) +# ifndef HAVE_BLAS_ILP64 +# define NPY_CBLAS_CHUNK (INT_MAX / 2 + 1) +# else +# define NPY_CBLAS_CHUNK (NPY_MAX_INT64 / 2 + 1) +# endif #else # define NPY_CBLAS_CHUNK NPY_MAX_INTP #endif diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index c38067681..055d3e60f 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -942,6 +942,20 @@ ravel_multi_index_loop(int ravel_ndim, npy_intp *ravel_dims, char invalid; npy_intp j, m; + /* + * Check for 0-dimensional axes unless there is nothing to do. + * An empty array/shape cannot be indexed at all. + */ + if (count != 0) { + for (i = 0; i < ravel_ndim; ++i) { + if (ravel_dims[i] == 0) { + PyErr_SetString(PyExc_ValueError, + "cannot unravel if shape has zero entries (is empty)."); + return NPY_FAIL; + } + } + } + NPY_BEGIN_ALLOW_THREADS; invalid = 0; while (count--) { diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 52cb58726..ca126b4b1 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -406,7 +406,6 @@ PyArray_SortkindConverter(PyObject *obj, NPY_SORTKIND *sortkind) } *sortkind = NPY_QUICKSORT; - str = PyBytes_AsString(obj); if (!str) { @@ -551,10 +550,9 @@ PyArray_OrderConverter(PyObject *object, NPY_ORDER *val) int ret; tmp = PyUnicode_AsASCIIString(object); if (tmp == NULL) { - PyErr_SetString(PyExc_ValueError, "Invalid unicode string passed in " - "for the array ordering. " - "Please pass in 'C', 'F', 'A' " - "or 'K' instead"); + PyErr_SetString(PyExc_ValueError, + "Invalid unicode string passed in for the array ordering. " + "Please pass in 'C', 'F', 'A' or 'K' instead"); return NPY_FAIL; } ret = PyArray_OrderConverter(tmp, val); @@ -562,38 +560,18 @@ PyArray_OrderConverter(PyObject *object, NPY_ORDER *val) return ret; } else if (!PyBytes_Check(object) || PyBytes_GET_SIZE(object) < 1) { - /* 2015-12-14, 1.11 */ - int ret = DEPRECATE("Non-string object detected for " - "the array ordering. Please pass " - "in 'C', 'F', 'A', or 'K' instead"); - - if (ret < 0) { - return -1; - } - - if (PyObject_IsTrue(object)) { - *val = NPY_FORTRANORDER; - } - else { - *val = NPY_CORDER; - } - if (PyErr_Occurred()) { - return NPY_FAIL; - } - return NPY_SUCCEED; + PyErr_SetString(PyExc_ValueError, + "Non-string object detected for the array ordering. " + "Please pass in 'C', 'F', 'A', or 'K' instead"); + return NPY_FAIL; } else { str = PyBytes_AS_STRING(object); if (strlen(str) != 1) { - /* 2015-12-14, 1.11 */ - int ret = DEPRECATE("Non length-one string passed " - "in for the array ordering. " - "Please pass in 'C', 'F', 'A', " - "or 'K' instead"); - - if (ret < 0) { - return -1; - } + PyErr_SetString(PyExc_ValueError, + "Non-string object detected for the array ordering. " + "Please pass in 'C', 'F', 'A', or 'K' instead"); + return NPY_FAIL; } if (str[0] == 'C' || str[0] == 'c') { @@ -689,8 +667,8 @@ PyArray_ConvertClipmodeSequence(PyObject *object, NPY_CLIPMODE *modes, int n) if (object && (PyTuple_Check(object) || PyList_Check(object))) { if (PySequence_Size(object) != n) { PyErr_Format(PyExc_ValueError, - "list of clipmodes has wrong length (%d instead of %d)", - (int)PySequence_Size(object), n); + "list of clipmodes has wrong length (%zd instead of %d)", + PySequence_Size(object), n); return NPY_FAIL; } diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index 025c66013..4326448dc 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -877,7 +877,13 @@ PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, from_order = dtype_kind_to_ordering(from->kind); to_order = dtype_kind_to_ordering(to->kind); - return from_order != -1 && from_order <= to_order; + if (to->kind == 'm') { + /* both types being timedelta is already handled before. */ + int integer_order = dtype_kind_to_ordering('i'); + return (from_order != -1) && (from_order <= integer_order); + } + + return (from_order != -1) && (from_order <= to_order); } else { return 0; diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 0897feaf4..7276add75 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -40,9 +40,31 @@ * regards to the handling of text representations. */ +/* + * Scanning function for next element parsing and seperator skipping. + * These functions return: + * - 0 to indicate more data to read + * - -1 when reading stopped at the end of the string/file + * - -2 when reading stopped before the end was reached. + * + * The dtype specific parsing functions may set the python error state + * (they have to get the GIL first) additionally. + */ typedef int (*next_element)(void **, void *, PyArray_Descr *, void *); typedef int (*skip_separator)(void **, const char *, void *); + +static npy_bool +string_is_fully_read(char const* start, char const* end) { + if (end == NULL) { + return *start == '\0'; /* null terminated */ + } + else { + return start >= end; /* fixed length */ + } +} + + static int fromstr_next_element(char **s, void *dptr, PyArray_Descr *dtype, const char *end) @@ -50,19 +72,23 @@ fromstr_next_element(char **s, void *dptr, PyArray_Descr *dtype, char *e = *s; int r = dtype->f->fromstr(*s, dptr, &e, dtype); /* - * fromstr always returns 0 for basic dtypes - * s points to the end of the parsed string - * if an error occurs s is not changed + * fromstr always returns 0 for basic dtypes; s points to the end of the + * parsed string. If s is not changed an error occurred or the end was + * reached. */ - if (*s == e) { - /* Nothing read */ - return -1; + if (*s == e || r < 0) { + /* Nothing read, could be end of string or an error (or both) */ + if (string_is_fully_read(*s, end)) { + return -1; + } + return -2; } *s = e; if (end != NULL && *s > end) { + /* Stop the iteration if we read far enough */ return -1; } - return r; + return 0; } static int @@ -75,9 +101,13 @@ fromfile_next_element(FILE **fp, void *dptr, PyArray_Descr *dtype, if (r == 1) { return 0; } - else { + else if (r == EOF) { return -1; } + else { + /* unable to read more, but EOF not reached indicating an error. */ + return -2; + } } /* @@ -143,9 +173,10 @@ fromstr_skip_separator(char **s, const char *sep, const char *end) { char *string = *s; int result = 0; + while (1) { char c = *string; - if (c == '\0' || (end != NULL && string >= end)) { + if (string_is_fully_read(string, end)) { result = -1; break; } @@ -488,8 +519,8 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, */ if (slen != PyArray_DIMS(a)[dim] && slen != 1) { PyErr_Format(PyExc_ValueError, - "cannot copy sequence with size %d to array axis " - "with dimension %d", (int)slen, (int)PyArray_DIMS(a)[dim]); + "cannot copy sequence with size %zd to array axis " + "with dimension %" NPY_INTP_FMT, slen, PyArray_DIMS(a)[dim]); goto fail; } @@ -796,6 +827,10 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, return 0; } } + else if (PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } + /* obj has the __array_interface__ interface */ e = PyArray_LookupSpecial_OnInstance(obj, "__array_interface__"); @@ -825,6 +860,9 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, return 0; } } + else if (PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } seq = PySequence_Fast(obj, "Could not convert object to sequence"); if (seq == NULL) { @@ -911,6 +949,39 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, return 0; } +static PyObject * +raise_memory_error(int nd, npy_intp *dims, PyArray_Descr *descr) +{ + static PyObject *exc_type = NULL; + + npy_cache_import( + "numpy.core._exceptions", "_ArrayMemoryError", + &exc_type); + if (exc_type == NULL) { + goto fail; + } + + PyObject *shape = PyArray_IntTupleFromIntp(nd, dims); + if (shape == NULL) { + goto fail; + } + + /* produce an error object */ + PyObject *exc_value = PyTuple_Pack(2, shape, (PyObject *)descr); + Py_DECREF(shape); + if (exc_value == NULL){ + goto fail; + } + PyErr_SetObject(exc_type, exc_value); + Py_DECREF(exc_value); + return NULL; + +fail: + /* we couldn't raise the formatted exception for some reason */ + PyErr_WriteUnraisable(NULL); + return PyErr_NoMemory(); +} + /* * Generic new array creation routine. * Internal variant with calloc argument for PyArray_Zeros. @@ -1088,30 +1159,7 @@ PyArray_NewFromDescr_int( data = npy_alloc_cache(nbytes); } if (data == NULL) { - static PyObject *exc_type = NULL; - - npy_cache_import( - "numpy.core._exceptions", "_ArrayMemoryError", - &exc_type); - if (exc_type == NULL) { - return NULL; - } - - PyObject *shape = PyArray_IntTupleFromIntp(fa->nd,fa->dimensions); - if (shape == NULL) { - return NULL; - } - - /* produce an error object */ - PyObject *exc_value = PyTuple_Pack(2, shape, descr); - Py_DECREF(shape); - if (exc_value == NULL){ - return NULL; - } - PyErr_SetObject(exc_type, exc_value); - Py_DECREF(exc_value); - return NULL; - + return raise_memory_error(fa->nd, fa->dimensions, descr); } fa->flags |= NPY_ARRAY_OWNDATA; @@ -1873,6 +1921,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, if (arr == NULL) { if ((flags & NPY_ARRAY_WRITEBACKIFCOPY) || (flags & NPY_ARRAY_UPDATEIFCOPY)) { + Py_DECREF(dtype); Py_XDECREF(newtype); PyErr_SetString(PyExc_TypeError, "WRITEBACKIFCOPY used for non-array input."); @@ -2249,7 +2298,11 @@ PyArray_FromStructInterface(PyObject *input) attr = PyArray_LookupSpecial_OnInstance(input, "__array_struct__"); if (attr == NULL) { - return Py_NotImplemented; + if (PyErr_Occurred()) { + return NULL; + } else { + return Py_NotImplemented; + } } if (!NpyCapsule_Check(attr)) { goto fail; @@ -2361,6 +2414,9 @@ PyArray_FromInterface(PyObject *origin) iface = PyArray_LookupSpecial_OnInstance(origin, "__array_interface__"); if (iface == NULL) { + if (PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } return Py_NotImplemented; } if (!PyDict_Check(iface)) { @@ -2614,6 +2670,9 @@ PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode, PyObject *context) array_meth = PyArray_LookupSpecial_OnInstance(op, "__array__"); if (array_meth == NULL) { + if (PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } return Py_NotImplemented; } if (context == NULL) { @@ -2682,61 +2741,30 @@ PyArray_DescrFromObject(PyObject *op, PyArray_Descr *mintype) /* They all zero-out the memory as previously done */ /* steals reference to descr -- and enforces native byteorder on it.*/ + /*NUMPY_API - Like FromDimsAndData but uses the Descr structure instead of typecode - as input. + Deprecated, use PyArray_NewFromDescr instead. */ NPY_NO_EXPORT PyObject * -PyArray_FromDimsAndDataAndDescr(int nd, int *d, +PyArray_FromDimsAndDataAndDescr(int NPY_UNUSED(nd), int *NPY_UNUSED(d), PyArray_Descr *descr, - char *data) + char *NPY_UNUSED(data)) { - PyObject *ret; - int i; - npy_intp newd[NPY_MAXDIMS]; - char msg[] = "PyArray_FromDimsAndDataAndDescr: use PyArray_NewFromDescr."; - - if (DEPRECATE(msg) < 0) { - /* 2009-04-30, 1.5 */ - return NULL; - } - if (!PyArray_ISNBO(descr->byteorder)) - descr->byteorder = '='; - for (i = 0; i < nd; i++) { - newd[i] = (npy_intp) d[i]; - } - ret = PyArray_NewFromDescr(&PyArray_Type, descr, - nd, newd, - NULL, data, - (data ? NPY_ARRAY_CARRAY : 0), NULL); - return ret; + PyErr_SetString(PyExc_NotImplementedError, + "PyArray_FromDimsAndDataAndDescr: use PyArray_NewFromDescr."); + Py_DECREF(descr); + return NULL; } /*NUMPY_API - Construct an empty array from dimensions and typenum + Deprecated, use PyArray_SimpleNew instead. */ NPY_NO_EXPORT PyObject * -PyArray_FromDims(int nd, int *d, int type) +PyArray_FromDims(int NPY_UNUSED(nd), int *NPY_UNUSED(d), int NPY_UNUSED(type)) { - PyArrayObject *ret; - char msg[] = "PyArray_FromDims: use PyArray_SimpleNew."; - - if (DEPRECATE(msg) < 0) { - /* 2009-04-30, 1.5 */ - return NULL; - } - ret = (PyArrayObject *)PyArray_FromDimsAndDataAndDescr(nd, d, - PyArray_DescrFromType(type), - NULL); - /* - * Old FromDims set memory to zero --- some algorithms - * relied on that. Better keep it the same. If - * Object type, then it's already been set to zero, though. - */ - if (ret && (PyArray_DESCR(ret)->type_num != NPY_OBJECT)) { - memset(PyArray_DATA(ret), 0, PyArray_NBYTES(ret)); - } - return (PyObject *)ret; + PyErr_SetString(PyExc_NotImplementedError, + "PyArray_FromDims: use PyArray_SimpleNew."); + return NULL; } /* end old calls */ @@ -2823,8 +2851,8 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) src_size = PyArray_SIZE(src); if (dst_size != src_size) { PyErr_Format(PyExc_ValueError, - "cannot copy from array of size %d into an array " - "of size %d", (int)src_size, (int)dst_size); + "cannot copy from array of size %" NPY_INTP_FMT " into an array " + "of size %" NPY_INTP_FMT, src_size, dst_size); return -1; } @@ -3503,11 +3531,13 @@ PyArray_ArangeObj(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr return NULL; } +/* This array creation function steals the reference to dtype. */ static PyArrayObject * array_fromfile_binary(FILE *fp, PyArray_Descr *dtype, npy_intp num, size_t *nread) { PyArrayObject *r; npy_off_t start, numbytes; + int elsize; if (num < 0) { int fail = 0; @@ -3534,27 +3564,29 @@ array_fromfile_binary(FILE *fp, PyArray_Descr *dtype, npy_intp num, size_t *nrea } num = numbytes / dtype->elsize; } + /* - * When dtype->subarray is true, PyArray_NewFromDescr will decref dtype - * even on success, so make sure it stays around until exit. + * Array creation may move sub-array dimensions from the dtype to array + * dimensions, so we need to use the original element size when reading. */ - Py_INCREF(dtype); + elsize = dtype->elsize; + r = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &num, NULL, NULL, 0, NULL); if (r == NULL) { - Py_DECREF(dtype); return NULL; } + NPY_BEGIN_ALLOW_THREADS; - *nread = fread(PyArray_DATA(r), dtype->elsize, num, fp); + *nread = fread(PyArray_DATA(r), elsize, num, fp); NPY_END_ALLOW_THREADS; - Py_DECREF(dtype); return r; } /* * Create an array by reading from the given stream, using the passed * next_element and skip_separator functions. + * As typical for array creation functions, it steals the reference to dtype. */ #define FROM_BUFFER_SIZE 4096 static PyArrayObject * @@ -3566,6 +3598,7 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread, npy_intp i; char *dptr, *clean_sep, *tmp; int err = 0; + int stop_reading_flag; /* -1 indicates end reached; -2 a parsing error */ npy_intp thisbuf = 0; npy_intp size; npy_intp bytes, totalbytes; @@ -3573,10 +3606,11 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread, size = (num >= 0) ? num : FROM_BUFFER_SIZE; /* - * When dtype->subarray is true, PyArray_NewFromDescr will decref dtype - * even on success, so make sure it stays around until exit. + * Array creation may move sub-array dimensions from the dtype to array + * dimensions, so we need to use the original dtype when reading. */ Py_INCREF(dtype); + r = (PyArrayObject *) PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &size, NULL, NULL, 0, NULL); @@ -3584,6 +3618,7 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread, Py_DECREF(dtype); return NULL; } + clean_sep = swab_separator(sep); if (clean_sep == NULL) { err = 1; @@ -3593,9 +3628,9 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread, NPY_BEGIN_ALLOW_THREADS; totalbytes = bytes = size * dtype->elsize; dptr = PyArray_DATA(r); - for (i= 0; num < 0 || i < num; i++) { - if (next(&stream, dptr, dtype, stream_data) < 0) { - /* EOF */ + for (i = 0; num < 0 || i < num; i++) { + stop_reading_flag = next(&stream, dptr, dtype, stream_data); + if (stop_reading_flag < 0) { break; } *nread += 1; @@ -3612,7 +3647,12 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread, dptr = tmp + (totalbytes - bytes); thisbuf = 0; } - if (skip_sep(&stream, clean_sep, stream_data) < 0) { + stop_reading_flag = skip_sep(&stream, clean_sep, stream_data); + if (stop_reading_flag < 0) { + if (num == i + 1) { + /* if we read as much as requested sep is optional */ + stop_reading_flag = -1; + } break; } } @@ -3631,8 +3671,24 @@ array_from_text(PyArray_Descr *dtype, npy_intp num, char *sep, size_t *nread, } } NPY_END_ALLOW_THREADS; + free(clean_sep); + if (stop_reading_flag == -2) { + if (PyErr_Occurred()) { + /* If an error is already set (unlikely), do not create new one */ + Py_DECREF(r); + Py_DECREF(dtype); + return NULL; + } + /* 2019-09-12, NumPy 1.18 */ + if (DEPRECATE( + "string or file could not be read to its end due to unmatched " + "data; this will raise a ValueError in the future.") < 0) { + goto fail; + } + } + fail: Py_DECREF(dtype); if (err == 1) { @@ -3651,9 +3707,8 @@ fail: * Given a ``FILE *`` pointer ``fp``, and a ``PyArray_Descr``, return an * array corresponding to the data encoded in that file. * - * If the dtype is NULL, the default array type is used (double). - * If non-null, the reference is stolen and if dtype->subarray is true dtype - * will be decrefed even on success. + * The reference to `dtype` is stolen (it is possible that the passed in + * dtype is not held on to). * * The number of elements to read is given as ``num``; if it is < 0, then * then as many as possible are read. @@ -3701,7 +3756,6 @@ PyArray_FromFile(FILE *fp, PyArray_Descr *dtype, npy_intp num, char *sep) (skip_separator) fromfile_skip_separator, NULL); } if (ret == NULL) { - Py_DECREF(dtype); return NULL; } if (((npy_intp) nread) < num) { @@ -3791,7 +3845,13 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type, s = (npy_intp)ts - offset; n = (npy_intp)count; itemsize = type->elsize; - if (n < 0 ) { + if (n < 0) { + if (itemsize == 0) { + PyErr_SetString(PyExc_ValueError, + "cannot determine count if itemsize is 0"); + Py_DECREF(type); + return NULL; + } if (s % itemsize != 0) { PyErr_SetString(PyExc_ValueError, "buffer size must be a multiple"\ @@ -3896,6 +3956,11 @@ PyArray_FromString(char *data, npy_intp slen, PyArray_Descr *dtype, return NULL; } } + /* + * NewFromDescr may replace dtype to absorb subarray shape + * into the array, so get size beforehand. + */ + npy_intp size_to_copy = num*dtype->elsize; ret = (PyArrayObject *) PyArray_NewFromDescr(&PyArray_Type, dtype, 1, &num, NULL, NULL, @@ -3903,14 +3968,14 @@ PyArray_FromString(char *data, npy_intp slen, PyArray_Descr *dtype, if (ret == NULL) { return NULL; } - memcpy(PyArray_DATA(ret), data, num*dtype->elsize); + memcpy(PyArray_DATA(ret), data, size_to_copy); } else { /* read from character-based string */ size_t nread = 0; char *end; - if (dtype->f->scanfunc == NULL) { + if (dtype->f->fromstr == NULL) { PyErr_SetString(PyExc_ValueError, "don't know how to read " \ "character strings with that " \ @@ -3984,7 +4049,7 @@ PyArray_FromIter(PyObject *obj, PyArray_Descr *dtype, npy_intp count) } for (i = 0; (i < count || count == -1) && (value = PyIter_Next(iter)); i++) { - if (i >= elcount) { + if (i >= elcount && elsize != 0) { npy_intp nbytes; /* Grow PyArray_DATA(ret): diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 4268b8893..72a3df89c 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -27,6 +27,40 @@ #include "datetime_strings.h" /* + * Computes the python `ret, d = divmod(d, unit)`. + * + * Note that GCC is smart enough at -O2 to eliminate the `if(*d < 0)` branch + * for subsequent calls to this command - it is able to deduce that `*d >= 0`. + */ +static inline +npy_int64 extract_unit_64(npy_int64 *d, npy_int64 unit) { + assert(unit > 0); + npy_int64 div = *d / unit; + npy_int64 mod = *d % unit; + if (mod < 0) { + mod += unit; + div -= 1; + } + assert(mod >= 0); + *d = mod; + return div; +} + +static inline +npy_int32 extract_unit_32(npy_int32 *d, npy_int32 unit) { + assert(unit > 0); + npy_int32 div = *d / unit; + npy_int32 mod = *d % unit; + if (mod < 0) { + mod += unit; + div -= 1; + } + assert(mod >= 0); + *d = mod; + return div; +} + +/* * Imports the PyDateTime functions so we can create these objects. * This is called during module initialization */ @@ -160,17 +194,7 @@ days_to_yearsdays(npy_int64 *days_) npy_int64 year; /* Break down the 400 year cycle to get the year and day within the year */ - if (days >= 0) { - year = 400 * (days / days_per_400years); - days = days % days_per_400years; - } - else { - year = 400 * ((days - (days_per_400years - 1)) / days_per_400years); - days = days % days_per_400years; - if (days < 0) { - days += days_per_400years; - } - } + year = 400 * extract_unit_64(&days, days_per_400years); /* Work out the year/day within the 400 year cycle */ if (days >= 366) { @@ -386,7 +410,8 @@ convert_datetimestruct_to_datetime(PyArray_DatetimeMetaData *meta, * TO BE REMOVED - NOT USED INTERNALLY. */ NPY_NO_EXPORT npy_datetime -PyArray_DatetimeStructToDatetime(NPY_DATETIMEUNIT fr, npy_datetimestruct *d) +PyArray_DatetimeStructToDatetime( + NPY_DATETIMEUNIT NPY_UNUSED(fr), npy_datetimestruct *NPY_UNUSED(d)) { PyErr_SetString(PyExc_RuntimeError, "The NumPy PyArray_DatetimeStructToDatetime function has " @@ -400,7 +425,8 @@ PyArray_DatetimeStructToDatetime(NPY_DATETIMEUNIT fr, npy_datetimestruct *d) * TO BE REMOVED - NOT USED INTERNALLY. */ NPY_NO_EXPORT npy_datetime -PyArray_TimedeltaStructToTimedelta(NPY_DATETIMEUNIT fr, npy_timedeltastruct *d) +PyArray_TimedeltaStructToTimedelta( + NPY_DATETIMEUNIT NPY_UNUSED(fr), npy_timedeltastruct *NPY_UNUSED(d)) { PyErr_SetString(PyExc_RuntimeError, "The NumPy PyArray_TimedeltaStructToTimedelta function has " @@ -409,26 +435,6 @@ PyArray_TimedeltaStructToTimedelta(NPY_DATETIMEUNIT fr, npy_timedeltastruct *d) } /* - * Computes the python `ret, d = divmod(d, unit)`. - * - * Note that GCC is smart enough at -O2 to eliminate the `if(*d < 0)` branch - * for subsequent calls to this command - it is able to deduce that `*d >= 0`. - */ -static inline -npy_int64 extract_unit(npy_datetime *d, npy_datetime unit) { - assert(unit > 0); - npy_int64 div = *d / unit; - npy_int64 mod = *d % unit; - if (mod < 0) { - mod += unit; - div -= 1; - } - assert(mod >= 0); - *d = mod; - return div; -} - -/* * Converts a datetime based on the given metadata into a datetimestruct */ NPY_NO_EXPORT int @@ -436,7 +442,7 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, npy_datetime dt, npy_datetimestruct *out) { - npy_int64 perday; + npy_int64 days; /* Initialize the output to all zeros */ memset(out, 0, sizeof(npy_datetimestruct)); @@ -471,7 +477,7 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, break; case NPY_FR_M: - out->year = 1970 + extract_unit(&dt, 12); + out->year = 1970 + extract_unit_64(&dt, 12); out->month = dt + 1; break; @@ -485,73 +491,67 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, break; case NPY_FR_h: - perday = 24LL; - - set_datetimestruct_days(extract_unit(&dt, perday), out); + days = extract_unit_64(&dt, 24LL); + set_datetimestruct_days(days, out); out->hour = (int)dt; break; case NPY_FR_m: - perday = 24LL * 60; - - set_datetimestruct_days(extract_unit(&dt, perday), out); - out->hour = (int)extract_unit(&dt, 60); - out->min = (int)dt; + days = extract_unit_64(&dt, 60LL*24); + set_datetimestruct_days(days, out); + out->hour = (int)extract_unit_64(&dt, 60LL); + out->min = (int)dt; break; case NPY_FR_s: - perday = 24LL * 60 * 60; - - set_datetimestruct_days(extract_unit(&dt, perday), out); - out->hour = (int)extract_unit(&dt, 60*60); - out->min = (int)extract_unit(&dt, 60); + days = extract_unit_64(&dt, 60LL*60*24); + set_datetimestruct_days(days, out); + out->hour = (int)extract_unit_64(&dt, 60LL*60); + out->min = (int)extract_unit_64(&dt, 60LL); out->sec = (int)dt; break; case NPY_FR_ms: - perday = 24LL * 60 * 60 * 1000; - - set_datetimestruct_days(extract_unit(&dt, perday), out); - out->hour = (int)extract_unit(&dt, 1000LL*60*60); - out->min = (int)extract_unit(&dt, 1000LL*60); - out->sec = (int)extract_unit(&dt, 1000LL); + days = extract_unit_64(&dt, 1000LL*60*60*24); + set_datetimestruct_days(days, out); + out->hour = (int)extract_unit_64(&dt, 1000LL*60*60); + out->min = (int)extract_unit_64(&dt, 1000LL*60); + out->sec = (int)extract_unit_64(&dt, 1000LL); out->us = (int)(dt * 1000); break; case NPY_FR_us: - perday = 24LL * 60LL * 60LL * 1000LL * 1000LL; - set_datetimestruct_days(extract_unit(&dt, perday), out); - out->hour = (int)extract_unit(&dt, 1000LL*1000*60*60); - out->min = (int)extract_unit(&dt, 1000LL*1000*60); - out->sec = (int)extract_unit(&dt, 1000LL*1000); + days = extract_unit_64(&dt, 1000LL*1000*60*60*24); + set_datetimestruct_days(days, out); + out->hour = (int)extract_unit_64(&dt, 1000LL*1000*60*60); + out->min = (int)extract_unit_64(&dt, 1000LL*1000*60); + out->sec = (int)extract_unit_64(&dt, 1000LL*1000); out->us = (int)dt; break; case NPY_FR_ns: - perday = 24LL * 60LL * 60LL * 1000LL * 1000LL * 1000LL; - - set_datetimestruct_days(extract_unit(&dt, perday), out); - out->hour = (int)extract_unit(&dt, 1000LL*1000*1000*60*60); - out->min = (int)extract_unit(&dt, 1000LL*1000*1000*60); - out->sec = (int)extract_unit(&dt, 1000LL*1000*1000); - out->us = (int)extract_unit(&dt, 1000LL); + days = extract_unit_64(&dt, 1000LL*1000*1000*60*60*24); + set_datetimestruct_days(days, out); + out->hour = (int)extract_unit_64(&dt, 1000LL*1000*1000*60*60); + out->min = (int)extract_unit_64(&dt, 1000LL*1000*1000*60); + out->sec = (int)extract_unit_64(&dt, 1000LL*1000*1000); + out->us = (int)extract_unit_64(&dt, 1000LL); out->ps = (int)(dt * 1000); break; case NPY_FR_ps: - perday = 24LL * 60 * 60 * 1000 * 1000 * 1000 * 1000; - - set_datetimestruct_days(extract_unit(&dt, perday), out); - out->hour = (int)extract_unit(&dt, 1000LL*1000*1000*1000*60*60); - out->min = (int)extract_unit(&dt, 1000LL*1000*1000*1000*60); - out->sec = (int)extract_unit(&dt, 1000LL*1000*1000*1000); - out->us = (int)extract_unit(&dt, 1000LL*1000); + days = extract_unit_64(&dt, 1000LL*1000*1000*1000*60*60*24); + set_datetimestruct_days(days, out); + out->hour = (int)extract_unit_64(&dt, 1000LL*1000*1000*1000*60*60); + out->min = (int)extract_unit_64(&dt, 1000LL*1000*1000*1000*60); + out->sec = (int)extract_unit_64(&dt, 1000LL*1000*1000*1000); + out->us = (int)extract_unit_64(&dt, 1000LL*1000); out->ps = (int)(dt); break; case NPY_FR_fs: /* entire range is only +- 2.6 hours */ - out->hour = (int)extract_unit(&dt, 1000LL*1000*1000*1000*1000*60*60); + out->hour = (int)extract_unit_64(&dt, 1000LL*1000*1000*1000*1000*60*60); if (out->hour < 0) { out->year = 1969; out->month = 12; @@ -559,16 +559,16 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, out->hour += 24; assert(out->hour >= 0); } - out->min = (int)extract_unit(&dt, 1000LL*1000*1000*1000*1000*60); - out->sec = (int)extract_unit(&dt, 1000LL*1000*1000*1000*1000); - out->us = (int)extract_unit(&dt, 1000LL*1000*1000); - out->ps = (int)extract_unit(&dt, 1000LL); + out->min = (int)extract_unit_64(&dt, 1000LL*1000*1000*1000*1000*60); + out->sec = (int)extract_unit_64(&dt, 1000LL*1000*1000*1000*1000); + out->us = (int)extract_unit_64(&dt, 1000LL*1000*1000); + out->ps = (int)extract_unit_64(&dt, 1000LL); out->as = (int)(dt * 1000); break; case NPY_FR_as: /* entire range is only +- 9.2 seconds */ - out->sec = (int)extract_unit(&dt, 1000LL*1000*1000*1000*1000*1000); + out->sec = (int)extract_unit_64(&dt, 1000LL*1000*1000*1000*1000*1000); if (out->sec < 0) { out->year = 1969; out->month = 12; @@ -578,8 +578,8 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, out->sec += 60; assert(out->sec >= 0); } - out->us = (int)extract_unit(&dt, 1000LL*1000*1000*1000); - out->ps = (int)extract_unit(&dt, 1000LL*1000); + out->us = (int)extract_unit_64(&dt, 1000LL*1000*1000*1000); + out->ps = (int)extract_unit_64(&dt, 1000LL*1000); out->as = (int)dt; break; @@ -600,8 +600,9 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, * TO BE REMOVED - NOT USED INTERNALLY. */ NPY_NO_EXPORT void -PyArray_DatetimeToDatetimeStruct(npy_datetime val, NPY_DATETIMEUNIT fr, - npy_datetimestruct *result) +PyArray_DatetimeToDatetimeStruct( + npy_datetime NPY_UNUSED(val), NPY_DATETIMEUNIT NPY_UNUSED(fr), + npy_datetimestruct *result) { PyErr_SetString(PyExc_RuntimeError, "The NumPy PyArray_DatetimeToDatetimeStruct function has " @@ -621,8 +622,9 @@ PyArray_DatetimeToDatetimeStruct(npy_datetime val, NPY_DATETIMEUNIT fr, * TO BE REMOVED - NOT USED INTERNALLY. */ NPY_NO_EXPORT void -PyArray_TimedeltaToTimedeltaStruct(npy_timedelta val, NPY_DATETIMEUNIT fr, - npy_timedeltastruct *result) +PyArray_TimedeltaToTimedeltaStruct( + npy_timedelta NPY_UNUSED(val), NPY_DATETIMEUNIT NPY_UNUSED(fr), + npy_timedeltastruct *result) { PyErr_SetString(PyExc_RuntimeError, "The NumPy PyArray_TimedeltaToTimedeltaStruct function has " @@ -756,8 +758,8 @@ parse_datetime_extended_unit_from_string(char *str, Py_ssize_t len, bad_input: if (metastr != NULL) { PyErr_Format(PyExc_TypeError, - "Invalid datetime metadata string \"%s\" at position %d", - metastr, (int)(substr-metastr)); + "Invalid datetime metadata string \"%s\" at position %zd", + metastr, substr-metastr); } else { PyErr_Format(PyExc_TypeError, @@ -818,8 +820,8 @@ parse_datetime_metadata_from_metastr(char *metastr, Py_ssize_t len, bad_input: if (substr != metastr) { PyErr_Format(PyExc_TypeError, - "Invalid datetime metadata string \"%s\" at position %d", - metastr, (int)(substr-metastr)); + "Invalid datetime metadata string \"%s\" at position %zd", + metastr, substr - metastr); } else { PyErr_Format(PyExc_TypeError, @@ -1832,6 +1834,7 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, return -1; } equal_one = PyObject_RichCompareBool(event, one, Py_EQ); + Py_DECREF(one); if (equal_one == -1) { return -1; } @@ -2012,20 +2015,8 @@ add_seconds_to_datetimestruct(npy_datetimestruct *dts, int seconds) int minutes; dts->sec += seconds; - if (dts->sec < 0) { - minutes = dts->sec / 60; - dts->sec = dts->sec % 60; - if (dts->sec < 0) { - --minutes; - dts->sec += 60; - } - add_minutes_to_datetimestruct(dts, minutes); - } - else if (dts->sec >= 60) { - minutes = dts->sec / 60; - dts->sec = dts->sec % 60; - add_minutes_to_datetimestruct(dts, minutes); - } + minutes = extract_unit_32(&dts->sec, 60); + add_minutes_to_datetimestruct(dts, minutes); } /* @@ -2037,28 +2028,13 @@ add_minutes_to_datetimestruct(npy_datetimestruct *dts, int minutes) { int isleap; - /* MINUTES */ dts->min += minutes; - while (dts->min < 0) { - dts->min += 60; - dts->hour--; - } - while (dts->min >= 60) { - dts->min -= 60; - dts->hour++; - } - /* HOURS */ - while (dts->hour < 0) { - dts->hour += 24; - dts->day--; - } - while (dts->hour >= 24) { - dts->hour -= 24; - dts->day++; - } + /* propagate invalid minutes into hour and day changes */ + dts->hour += extract_unit_32(&dts->min, 60); + dts->day += extract_unit_32(&dts->hour, 24); - /* DAYS */ + /* propagate invalid days into month and year changes */ if (dts->day < 1) { dts->month--; if (dts->month < 1) { @@ -2250,6 +2226,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, if (DEPRECATE( "parsing timezone aware datetimes is deprecated; " "this will raise an error in the future") < 0) { + Py_DECREF(tmp); return -1; } @@ -2266,6 +2243,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, * which contains the value we want. */ tmp = PyObject_CallMethod(offset, "total_seconds", ""); + Py_DECREF(offset); if (tmp == NULL) { return -1; } @@ -2295,15 +2273,15 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, invalid_date: PyErr_Format(PyExc_ValueError, - "Invalid date (%d,%d,%d) when converting to NumPy datetime", - (int)out->year, (int)out->month, (int)out->day); + "Invalid date (%" NPY_INT64_FMT ",%" NPY_INT32_FMT ",%" NPY_INT32_FMT ") when converting to NumPy datetime", + out->year, out->month, out->day); return -1; invalid_time: PyErr_Format(PyExc_ValueError, - "Invalid time (%d,%d,%d,%d) when converting " + "Invalid time (%" NPY_INT32_FMT ",%" NPY_INT32_FMT ",%" NPY_INT32_FMT ",%" NPY_INT32_FMT ") when converting " "to NumPy datetime", - (int)out->hour, (int)out->min, (int)out->sec, (int)out->us); + out->hour, out->min, out->sec, out->us); return -1; } @@ -2883,7 +2861,6 @@ convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta) NPY_NO_EXPORT PyObject * convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta) { - PyObject *ret = NULL; npy_timedelta value; int days = 0, seconds = 0, useconds = 0; @@ -2913,54 +2890,47 @@ convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta) /* Convert to days/seconds/useconds */ switch (meta->base) { case NPY_FR_W: - value *= 7; + days = value * 7; break; case NPY_FR_D: + days = value; break; case NPY_FR_h: - seconds = (int)((value % 24) * (60*60)); - value = value / 24; + days = extract_unit_64(&value, 24ULL); + seconds = value*60*60; break; case NPY_FR_m: - seconds = (int)(value % (24*60)) * 60; - value = value / (24*60); + days = extract_unit_64(&value, 60ULL*24); + seconds = value*60; break; case NPY_FR_s: - seconds = (int)(value % (24*60*60)); - value = value / (24*60*60); + days = extract_unit_64(&value, 60ULL*60*24); + seconds = value; break; case NPY_FR_ms: - useconds = (int)(value % 1000) * 1000; - value = value / 1000; - seconds = (int)(value % (24*60*60)); - value = value / (24*60*60); + days = extract_unit_64(&value, 1000ULL*60*60*24); + seconds = extract_unit_64(&value, 1000ULL); + useconds = value*1000; break; case NPY_FR_us: - useconds = (int)(value % (1000*1000)); - value = value / (1000*1000); - seconds = (int)(value % (24*60*60)); - value = value / (24*60*60); + days = extract_unit_64(&value, 1000ULL*1000*60*60*24); + seconds = extract_unit_64(&value, 1000ULL*1000); + useconds = value; break; default: + // unreachable, handled by the `if` above + assert(NPY_FALSE); break; } /* - * 'value' represents days, and seconds/useconds are filled. - * * If it would overflow the datetime.timedelta days, return a raw int */ - if (value < -999999999 || value > 999999999) { + if (days < -999999999 || days > 999999999) { return PyLong_FromLongLong(td); } else { - days = (int)value; - ret = PyDelta_FromDSU(days, seconds, useconds); - if (ret == NULL) { - return NULL; - } + return PyDelta_FromDSU(days, seconds, useconds); } - - return ret; } /* @@ -3125,7 +3095,7 @@ is_any_numpy_datetime_or_timedelta(PyObject *obj) */ NPY_NO_EXPORT int convert_pyobjects_to_datetimes(int count, - PyObject **objs, int *type_nums, + PyObject **objs, const int *type_nums, NPY_CASTING casting, npy_int64 *out_values, PyArray_DatetimeMetaData *inout_meta) @@ -3251,18 +3221,6 @@ NPY_NO_EXPORT PyArrayObject * datetime_arange(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr *dtype) { - PyArray_DatetimeMetaData meta; - /* - * Both datetime and timedelta are stored as int64, so they can - * share value variables. - */ - npy_int64 values[3]; - PyObject *objs[3]; - int type_nums[3]; - - npy_intp i, length; - PyArrayObject *ret; - npy_int64 *ret_data; /* * First normalize the input parameters so there is no Py_None, @@ -3295,6 +3253,8 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, /* Check if the units of the given dtype are generic, in which * case we use the code path that detects the units */ + int type_nums[3]; + PyArray_DatetimeMetaData meta; if (dtype != NULL) { PyArray_DatetimeMetaData *meta_tmp; @@ -3343,6 +3303,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, } /* Set up to convert the objects to a common datetime unit metadata */ + PyObject *objs[3]; objs[0] = start; objs[1] = stop; objs[2] = step; @@ -3363,11 +3324,22 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, type_nums[2] = NPY_TIMEDELTA; } - /* Convert all the arguments */ + /* Convert all the arguments + * + * Both datetime and timedelta are stored as int64, so they can + * share value variables. + */ + npy_int64 values[3]; if (convert_pyobjects_to_datetimes(3, objs, type_nums, NPY_SAME_KIND_CASTING, values, &meta) < 0) { return NULL; } + /* If no start was provided, default to 0 */ + if (start == NULL) { + /* enforced above */ + assert(type_nums[0] == NPY_TIMEDELTA); + values[0] = 0; + } /* If no step was provided, default to 1 */ if (step == NULL) { @@ -3392,6 +3364,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, } /* Calculate the array length */ + npy_intp length; if (values[2] > 0 && values[1] > values[0]) { length = (values[1] - values[0] + (values[2] - 1)) / values[2]; } @@ -3419,19 +3392,20 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, } /* Create the result array */ - ret = (PyArrayObject *)PyArray_NewFromDescr( - &PyArray_Type, dtype, 1, &length, NULL, - NULL, 0, NULL); + PyArrayObject *ret = (PyArrayObject *)PyArray_NewFromDescr( + &PyArray_Type, dtype, 1, &length, NULL, + NULL, 0, NULL); + if (ret == NULL) { return NULL; } if (length > 0) { /* Extract the data pointer */ - ret_data = (npy_int64 *)PyArray_DATA(ret); + npy_int64 *ret_data = (npy_int64 *)PyArray_DATA(ret); /* Create the timedeltas or datetimes */ - for (i = 0; i < length; ++i) { + for (npy_intp i = 0; i < length; ++i) { *ret_data = values[0]; values[0] += values[2]; ret_data++; diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index c04a6c125..cdeb65d0e 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -48,7 +48,7 @@ get_day_of_week(npy_datetime date) */ static int is_holiday(npy_datetime date, - npy_datetime *holidays_begin, npy_datetime *holidays_end) + npy_datetime *holidays_begin, const npy_datetime *holidays_end) { npy_datetime *trial; @@ -88,7 +88,7 @@ is_holiday(npy_datetime date, */ static npy_datetime * find_earliest_holiday_on_or_after(npy_datetime date, - npy_datetime *holidays_begin, npy_datetime *holidays_end) + npy_datetime *holidays_begin, const npy_datetime *holidays_end) { npy_datetime *trial; @@ -127,7 +127,7 @@ find_earliest_holiday_on_or_after(npy_datetime date, */ static npy_datetime * find_earliest_holiday_after(npy_datetime date, - npy_datetime *holidays_begin, npy_datetime *holidays_end) + npy_datetime *holidays_begin, const npy_datetime *holidays_end) { npy_datetime *trial; @@ -159,7 +159,7 @@ static int apply_business_day_roll(npy_datetime date, npy_datetime *out, int *out_day_of_week, NPY_BUSDAY_ROLL roll, - npy_bool *weekmask, + const npy_bool *weekmask, npy_datetime *holidays_begin, npy_datetime *holidays_end) { int day_of_week; @@ -361,7 +361,7 @@ apply_business_day_offset(npy_datetime date, npy_int64 offset, static int apply_business_day_count(npy_datetime date_begin, npy_datetime date_end, npy_int64 *out, - npy_bool *weekmask, int busdays_in_weekmask, + const npy_bool *weekmask, int busdays_in_weekmask, npy_datetime *holidays_begin, npy_datetime *holidays_end) { npy_int64 count, whole_weeks; @@ -722,7 +722,7 @@ finish: */ NPY_NO_EXPORT PyArrayObject * is_business_day(PyArrayObject *dates, PyArrayObject *out, - npy_bool *weekmask, int busdays_in_weekmask, + const npy_bool *weekmask, int busdays_in_weekmask, npy_datetime *holidays_begin, npy_datetime *holidays_end) { PyArray_DatetimeMetaData temp_meta; diff --git a/numpy/core/src/multiarray/datetime_strings.c b/numpy/core/src/multiarray/datetime_strings.c index 95b7bb3dc..dfc01494f 100644 --- a/numpy/core/src/multiarray/datetime_strings.c +++ b/numpy/core/src/multiarray/datetime_strings.c @@ -743,8 +743,8 @@ finish: parse_error: PyErr_Format(PyExc_ValueError, - "Error parsing datetime string \"%s\" at position %d", - str, (int)(substr-str)); + "Error parsing datetime string \"%s\" at position %zd", + str, substr - str); return -1; error: diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index cb4d7964e..d4e18e457 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -102,6 +102,7 @@ _arraydescr_from_dtype_attr(PyObject *obj, PyArray_Descr **newdescr) if (Py_EnterRecursiveCall( " while trying to convert the given data type from its " "`.dtype` attribute.") != 0) { + Py_DECREF(dtypedescr); return 1; } @@ -148,7 +149,7 @@ array_set_typeDict(PyObject *NPY_UNUSED(ignored), PyObject *args) arg == '|' || arg == '=') static int -_check_for_commastring(char *type, Py_ssize_t len) +_check_for_commastring(const char *type, Py_ssize_t len) { Py_ssize_t i; int sqbracket; @@ -1148,8 +1149,8 @@ _convert_from_dict(PyObject *obj, int align) } Py_DECREF(off); if (offset < 0) { - PyErr_Format(PyExc_ValueError, "offset %d cannot be negative", - (int)offset); + PyErr_Format(PyExc_ValueError, "offset %ld cannot be negative", + offset); Py_DECREF(tup); Py_DECREF(ind); goto fail; @@ -1163,10 +1164,10 @@ _convert_from_dict(PyObject *obj, int align) /* If align=True, enforce field alignment */ if (align && offset % newdescr->alignment != 0) { PyErr_Format(PyExc_ValueError, - "offset %d for NumPy dtype with fields is " + "offset %ld for NumPy dtype with fields is " "not divisible by the field alignment %d " "with align=True", - (int)offset, (int)newdescr->alignment); + offset, newdescr->alignment); ret = NPY_FAIL; } else if (offset + newdescr->elsize > totalsize) { @@ -1285,7 +1286,7 @@ _convert_from_dict(PyObject *obj, int align) PyErr_Format(PyExc_ValueError, "NumPy dtype descriptor requires %d bytes, " "cannot override to smaller itemsize of %d", - (int)new->elsize, (int)itemsize); + new->elsize, itemsize); Py_DECREF(new); goto fail; } @@ -1294,7 +1295,7 @@ _convert_from_dict(PyObject *obj, int align) PyErr_Format(PyExc_ValueError, "NumPy dtype descriptor requires alignment of %d bytes, " "which is not divisible into the specified itemsize %d", - (int)new->alignment, (int)itemsize); + new->alignment, itemsize); Py_DECREF(new); goto fail; } @@ -1384,7 +1385,6 @@ NPY_NO_EXPORT int PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) { int check_num = NPY_NOTYPE + 10; - PyObject *item; int elsize = 0; char endian = '='; @@ -1663,16 +1663,22 @@ finish: PyErr_Clear(); /* Now check to see if the object is registered in typeDict */ if (typeDict != NULL) { - item = PyDict_GetItem(typeDict, obj); + PyObject *item = NULL; #if defined(NPY_PY3K) - if (!item && PyBytes_Check(obj)) { + if (PyBytes_Check(obj)) { PyObject *tmp; tmp = PyUnicode_FromEncodedObject(obj, "ascii", "strict"); - if (tmp != NULL) { - item = PyDict_GetItem(typeDict, tmp); - Py_DECREF(tmp); + if (tmp == NULL) { + goto fail; } + item = PyDict_GetItem(typeDict, tmp); + Py_DECREF(tmp); + } + else { + item = PyDict_GetItem(typeDict, obj); } +#else + item = PyDict_GetItem(typeDict, obj); #endif if (item) { /* Check for a deprecated Numeric-style typecode */ @@ -3276,7 +3282,7 @@ arraydescr_richcompare(PyArray_Descr *self, PyObject *other, int cmp_op) } static int -descr_nonzero(PyObject *self) +descr_nonzero(PyObject *NPY_UNUSED(self)) { /* `bool(np.dtype(...)) == True` for all dtypes. Needed to override default * nonzero implementation, which checks if `len(object) > 0`. */ diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index 2c4969d23..6e5d480d0 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -190,7 +190,7 @@ array_strides_set(PyArrayObject *self, PyObject *obj) static PyObject * -array_priority_get(PyArrayObject *self) +array_priority_get(PyArrayObject *NPY_UNUSED(self)) { return PyFloat_FromDouble(NPY_PRIORITY); } diff --git a/numpy/core/src/multiarray/item_selection.c b/numpy/core/src/multiarray/item_selection.c index 01d9ecfb3..a6ac902d3 100644 --- a/numpy/core/src/multiarray/item_selection.c +++ b/numpy/core/src/multiarray/item_selection.c @@ -1336,7 +1336,11 @@ PyArray_ArgPartition(PyArrayObject *op, PyArrayObject *ktharray, int axis, PyArray_ArgSortFunc *argsort; PyObject *ret; - if (which < 0 || which >= NPY_NSELECTS) { + /* + * As a C-exported function, enum NPY_SELECTKIND loses its enum property + * Check the values to make sure they are in range + */ + if ((int)which < 0 || (int)which >= NPY_NSELECTS) { PyErr_SetString(PyExc_ValueError, "not a valid partition kind"); return NULL; @@ -2214,9 +2218,13 @@ PyArray_Nonzero(PyArrayObject *self) PyArrayObject *ret = NULL; PyObject *ret_tuple; npy_intp ret_dims[2]; - PyArray_NonzeroFunc *nonzero = PyArray_DESCR(self)->f->nonzero; + + PyArray_NonzeroFunc *nonzero; + PyArray_Descr *dtype; + npy_intp nonzero_count; npy_intp added_count = 0; + int needs_api; int is_bool; NpyIter *iter; @@ -2224,6 +2232,10 @@ PyArray_Nonzero(PyArrayObject *self) NpyIter_GetMultiIndexFunc *get_multi_index; char **dataptr; + dtype = PyArray_DESCR(self); + nonzero = dtype->f->nonzero; + needs_api = PyDataType_FLAGCHK(dtype, NPY_NEEDS_PYAPI); + /* Special case - nonzero(zero_d) is nonzero(atleast_1d(zero_d)) */ if (ndim == 0) { char const* msg; @@ -2247,6 +2259,7 @@ PyArray_Nonzero(PyArrayObject *self) static npy_intp const zero_dim_shape[1] = {1}; static npy_intp const zero_dim_strides[1] = {0}; + Py_INCREF(PyArray_DESCR(self)); /* array creation steals reference */ PyArrayObject *self_1d = (PyArrayObject *)PyArray_NewFromDescrAndBase( Py_TYPE(self), PyArray_DESCR(self), 1, zero_dim_shape, zero_dim_strides, PyArray_BYTES(self), @@ -2254,7 +2267,9 @@ PyArray_Nonzero(PyArrayObject *self) if (self_1d == NULL) { return NULL; } - return PyArray_Nonzero(self_1d); + ret_tuple = PyArray_Nonzero(self_1d); + Py_DECREF(self_1d); + return ret_tuple; } /* @@ -2291,7 +2306,9 @@ PyArray_Nonzero(PyArrayObject *self) goto finish; } - NPY_BEGIN_THREADS_THRESHOLDED(count); + if (!needs_api) { + NPY_BEGIN_THREADS_THRESHOLDED(count); + } /* avoid function call for bool */ if (is_bool) { @@ -2332,6 +2349,9 @@ PyArray_Nonzero(PyArrayObject *self) } *multi_index++ = j; } + if (needs_api && PyErr_Occurred()) { + break; + } data += stride; } } @@ -2372,6 +2392,8 @@ PyArray_Nonzero(PyArrayObject *self) Py_DECREF(ret); return NULL; } + + needs_api = NpyIter_IterationNeedsAPI(iter); NPY_BEGIN_THREADS_NDITER(iter); @@ -2398,6 +2420,9 @@ PyArray_Nonzero(PyArrayObject *self) get_multi_index(iter, multi_index); multi_index += ndim; } + if (needs_api && PyErr_Occurred()) { + break; + } } while(iternext(iter)); } @@ -2407,6 +2432,11 @@ PyArray_Nonzero(PyArrayObject *self) NpyIter_Deallocate(iter); finish: + if (PyErr_Occurred()) { + Py_DECREF(ret); + return NULL; + } + /* if executed `nonzero()` check for miscount due to side-effect */ if (!is_bool && added_count != nonzero_count) { PyErr_SetString(PyExc_RuntimeError, @@ -2449,7 +2479,7 @@ finish: * array of values, which must be of length PyArray_NDIM(self). */ NPY_NO_EXPORT PyObject * -PyArray_MultiIndexGetItem(PyArrayObject *self, npy_intp *multi_index) +PyArray_MultiIndexGetItem(PyArrayObject *self, const npy_intp *multi_index) { int idim, ndim = PyArray_NDIM(self); char *data = PyArray_DATA(self); @@ -2477,7 +2507,7 @@ PyArray_MultiIndexGetItem(PyArrayObject *self, npy_intp *multi_index) * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int -PyArray_MultiIndexSetItem(PyArrayObject *self, npy_intp *multi_index, +PyArray_MultiIndexSetItem(PyArrayObject *self, const npy_intp *multi_index, PyObject *obj) { int idim, ndim = PyArray_NDIM(self); diff --git a/numpy/core/src/multiarray/item_selection.h b/numpy/core/src/multiarray/item_selection.h index 90bb5100d..2276b4db7 100644 --- a/numpy/core/src/multiarray/item_selection.h +++ b/numpy/core/src/multiarray/item_selection.h @@ -15,7 +15,7 @@ count_boolean_trues(int ndim, char *data, npy_intp *ashape, npy_intp *astrides); * array of values, which must be of length PyArray_NDIM(self). */ NPY_NO_EXPORT PyObject * -PyArray_MultiIndexGetItem(PyArrayObject *self, npy_intp *multi_index); +PyArray_MultiIndexGetItem(PyArrayObject *self, const npy_intp *multi_index); /* * Sets a single item in the array, based on a single multi-index @@ -24,7 +24,7 @@ PyArray_MultiIndexGetItem(PyArrayObject *self, npy_intp *multi_index); * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int -PyArray_MultiIndexSetItem(PyArrayObject *self, npy_intp *multi_index, +PyArray_MultiIndexSetItem(PyArrayObject *self, const npy_intp *multi_index, PyObject *obj); #endif diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index 9da811f69..e66bb36aa 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -98,7 +98,7 @@ parse_index_entry(PyObject *op, npy_intp *step_size, /* get the dataptr from its current coordinates for simple iterator */ static char* -get_ptr_simple(PyArrayIterObject* iter, npy_intp *coordinates) +get_ptr_simple(PyArrayIterObject* iter, const npy_intp *coordinates) { npy_intp i; char *ret; @@ -116,10 +116,12 @@ get_ptr_simple(PyArrayIterObject* iter, npy_intp *coordinates) * This is common initialization code between PyArrayIterObject and * PyArrayNeighborhoodIterObject * - * Increase ao refcount + * Steals a reference to the array object which gets removed at deallocation, + * if the iterator is allocated statically and its dealloc not called, it + * can be thought of as borrowing the reference. */ -static PyObject * -array_iter_base_init(PyArrayIterObject *it, PyArrayObject *ao) +NPY_NO_EXPORT void +PyArray_RawIterBaseInit(PyArrayIterObject *it, PyArrayObject *ao) { int nd, i; @@ -131,7 +133,6 @@ array_iter_base_init(PyArrayIterObject *it, PyArrayObject *ao) else { it->contiguous = 0; } - Py_INCREF(ao); it->ao = ao; it->size = PyArray_SIZE(ao); it->nd_m1 = nd - 1; @@ -155,7 +156,7 @@ array_iter_base_init(PyArrayIterObject *it, PyArrayObject *ao) it->translate = &get_ptr_simple; PyArray_ITER_RESET(it); - return (PyObject *)it; + return; } static void @@ -170,6 +171,10 @@ array_iter_base_dealloc(PyArrayIterObject *it) NPY_NO_EXPORT PyObject * PyArray_IterNew(PyObject *obj) { + /* + * Note that internall PyArray_RawIterBaseInit may be called directly on a + * statically allocated PyArrayIterObject. + */ PyArrayIterObject *it; PyArrayObject *ao; @@ -186,7 +191,8 @@ PyArray_IterNew(PyObject *obj) return NULL; } - array_iter_base_init(it, ao); + Py_INCREF(ao); /* PyArray_RawIterBaseInit steals a reference */ + PyArray_RawIterBaseInit(it, ao); return (PyObject *)it; } @@ -390,6 +396,10 @@ arrayiter_next(PyArrayIterObject *it) static void arrayiter_dealloc(PyArrayIterObject *it) { + /* + * Note that it is possible to statically allocate a PyArrayIterObject, + * which does not call this function. + */ array_iter_base_dealloc(it); PyArray_free(it); } @@ -1398,6 +1408,7 @@ arraymultiter_new(PyTypeObject *NPY_UNUSED(subtype), PyObject *args, } n = PySequence_Fast_GET_SIZE(fast_seq); if (n > NPY_MAXARGS) { + Py_DECREF(fast_seq); return multiiter_wrong_number_of_args(); } ret = multiiter_new_impl(n, PySequence_Fast_ITEMS(fast_seq)); @@ -1654,7 +1665,7 @@ static char* _set_constant(PyArrayNeighborhoodIterObject* iter, /* set the dataptr from its current coordinates */ static char* -get_ptr_constant(PyArrayIterObject* _iter, npy_intp *coordinates) +get_ptr_constant(PyArrayIterObject* _iter, const npy_intp *coordinates) { int i; npy_intp bd, _coordinates[NPY_MAXDIMS]; @@ -1709,7 +1720,7 @@ __npy_pos_remainder(npy_intp i, npy_intp n) /* set the dataptr from its current coordinates */ static char* -get_ptr_mirror(PyArrayIterObject* _iter, npy_intp *coordinates) +get_ptr_mirror(PyArrayIterObject* _iter, const npy_intp *coordinates) { int i; npy_intp bd, _coordinates[NPY_MAXDIMS], lb; @@ -1743,7 +1754,7 @@ __npy_euclidean_division(npy_intp i, npy_intp n) _coordinates[c] = lb + __npy_euclidean_division(bd, p->limits_sizes[c]); static char* -get_ptr_circular(PyArrayIterObject* _iter, npy_intp *coordinates) +get_ptr_circular(PyArrayIterObject* _iter, const npy_intp *coordinates) { int i; npy_intp bd, _coordinates[NPY_MAXDIMS], lb; @@ -1765,7 +1776,7 @@ get_ptr_circular(PyArrayIterObject* _iter, npy_intp *coordinates) * A Neighborhood Iterator object. */ NPY_NO_EXPORT PyObject* -PyArray_NeighborhoodIterNew(PyArrayIterObject *x, npy_intp *bounds, +PyArray_NeighborhoodIterNew(PyArrayIterObject *x, const npy_intp *bounds, int mode, PyArrayObject* fill) { int i; @@ -1777,7 +1788,8 @@ PyArray_NeighborhoodIterNew(PyArrayIterObject *x, npy_intp *bounds, } PyObject_Init((PyObject *)ret, &PyArrayNeighborhoodIter_Type); - array_iter_base_init((PyArrayIterObject*)ret, x->ao); + Py_INCREF(x->ao); /* PyArray_RawIterBaseInit steals a reference */ + PyArray_RawIterBaseInit((PyArrayIterObject*)ret, x->ao); Py_INCREF(x); ret->_internal_iter = x; diff --git a/numpy/core/src/multiarray/iterators.h b/numpy/core/src/multiarray/iterators.h index 376dc154a..d942f45b8 100644 --- a/numpy/core/src/multiarray/iterators.h +++ b/numpy/core/src/multiarray/iterators.h @@ -7,4 +7,7 @@ NPY_NO_EXPORT PyObject NPY_NO_EXPORT int iter_ass_subscript(PyArrayIterObject *, PyObject *, PyObject *); +NPY_NO_EXPORT void +PyArray_RawIterBaseInit(PyArrayIterObject *it, PyArrayObject *ao); + #endif diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index cc628e47e..8dcd28c84 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -176,7 +176,7 @@ unpack_tuple(PyTupleObject *index, PyObject **result, npy_intp result_n) /* Unpack a single scalar index, taking a new reference to match unpack_tuple */ static NPY_INLINE npy_intp -unpack_scalar(PyObject *index, PyObject **result, npy_intp result_n) +unpack_scalar(PyObject *index, PyObject **result, npy_intp NPY_UNUSED(result_n)) { Py_INCREF(index); result[0] = index; @@ -1198,9 +1198,9 @@ array_assign_boolean_subscript(PyArrayObject *self, if (size != PyArray_DIMS(v)[0]) { PyErr_Format(PyExc_ValueError, "NumPy boolean array indexing assignment " - "cannot assign %d input values to " - "the %d output values where the mask is true", - (int)PyArray_DIMS(v)[0], (int)size); + "cannot assign %" NPY_INTP_FMT " input values to " + "the %" NPY_INTP_FMT " output values where the mask is true", + PyArray_DIMS(v)[0], size); return -1; } v_stride = PyArray_STRIDES(v)[0]; @@ -2516,6 +2516,7 @@ PyArray_MapIterCheckIndices(PyArrayMapIterObject *mit) indval = *((npy_intp*)data); if (check_and_adjust_index(&indval, outer_dim, outer_axis, _save) < 0) { + Py_DECREF(intp_type); return -1; } data += stride; @@ -2616,7 +2617,8 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, PyArrayObject *original_extra_op = extra_op; PyArrayObject *index_arrays[NPY_MAXDIMS]; - PyArray_Descr *dtypes[NPY_MAXDIMS]; + PyArray_Descr *intp_descr; + PyArray_Descr *dtypes[NPY_MAXDIMS]; /* borrowed references */ npy_uint32 op_flags[NPY_MAXDIMS]; npy_uint32 outer_flags; @@ -2629,9 +2631,15 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, int nops; int uses_subspace; + intp_descr = PyArray_DescrFromType(NPY_INTP); + if (intp_descr == NULL) { + return NULL; + } + /* create new MapIter object */ mit = (PyArrayMapIterObject *)PyArray_malloc(sizeof(PyArrayMapIterObject)); if (mit == NULL) { + Py_DECREF(intp_descr); return NULL; } /* set all attributes of mapiter to zero */ @@ -2661,6 +2669,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, mit->nd_fancy = fancy_ndim; if (mapiter_fill_info(mit, indices, index_num, arr) < 0) { Py_DECREF(mit); + Py_DECREF(intp_descr); return NULL; } @@ -2670,7 +2679,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, for (i=0; i < index_num; i++) { if (indices[i].type & HAS_FANCY) { index_arrays[mit->numiter] = (PyArrayObject *)indices[i].object; - dtypes[mit->numiter] = PyArray_DescrFromType(NPY_INTP); + dtypes[mit->numiter] = intp_descr; op_flags[mit->numiter] = (NPY_ITER_NBO | NPY_ITER_ALIGNED | @@ -2693,9 +2702,10 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, PyArray_DescrFromType(NPY_INTP), 0); if (index_arrays[0] == NULL) { Py_DECREF(mit); + Py_DECREF(intp_descr); return NULL; } - dtypes[0] = PyArray_DescrFromType(NPY_INTP); + dtypes[0] = intp_descr; op_flags[0] = NPY_ITER_NBO | NPY_ITER_ALIGNED | NPY_ITER_READONLY; mit->fancy_dims[0] = 1; @@ -2925,7 +2935,6 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, nops += 1; index_arrays[mit->numiter] = extra_op; - Py_INCREF(extra_op_dtype); dtypes[mit->numiter] = extra_op_dtype; op_flags[mit->numiter] = (extra_op_flags | NPY_ITER_ALLOCATE | @@ -2951,9 +2960,6 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, } /* NpyIter cleanup and information: */ - for (i=0; i < nops; i++) { - Py_DECREF(dtypes[i]); - } if (dummy_array) { Py_DECREF(index_arrays[0]); } @@ -3039,6 +3045,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, /* Can now return early if no subspace is being used */ if (!uses_subspace) { Py_XDECREF(extra_op); + Py_DECREF(intp_descr); return (PyObject *)mit; } @@ -3108,6 +3115,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, } Py_XDECREF(extra_op); + Py_DECREF(intp_descr); return (PyObject *)mit; fail: @@ -3176,6 +3184,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, finish: Py_XDECREF(extra_op); + Py_DECREF(intp_descr); Py_DECREF(mit); return NULL; } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index d458638ae..e5845f2f6 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -365,6 +365,7 @@ PyArray_GetField(PyArrayObject *self, PyArray_Descr *typed, int offset) npy_cache_import("numpy.core._internal", "_getfield_is_safe", &checkfunc); if (checkfunc == NULL) { + Py_DECREF(typed); return NULL; } @@ -372,6 +373,7 @@ PyArray_GetField(PyArrayObject *self, PyArray_Descr *typed, int offset) safe = PyObject_CallFunction(checkfunc, "OOi", PyArray_DESCR(self), typed, offset); if (safe == NULL) { + Py_DECREF(typed); return NULL; } Py_DECREF(safe); @@ -382,14 +384,17 @@ PyArray_GetField(PyArrayObject *self, PyArray_Descr *typed, int offset) /* check that values are valid */ if (typed_elsize > self_elsize) { PyErr_SetString(PyExc_ValueError, "new type is larger than original type"); + Py_DECREF(typed); return NULL; } if (offset < 0) { PyErr_SetString(PyExc_ValueError, "offset is negative"); + Py_DECREF(typed); return NULL; } if (offset > self_elsize - typed_elsize) { PyErr_SetString(PyExc_ValueError, "new type plus offset is larger than original type"); + Py_DECREF(typed); return NULL; } @@ -434,6 +439,7 @@ PyArray_SetField(PyArrayObject *self, PyArray_Descr *dtype, int retval = 0; if (PyArray_FailUnlessWriteable(self, "assignment destination") < 0) { + Py_DECREF(dtype); return -1; } @@ -583,14 +589,13 @@ array_tofile(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } if (PyBytes_Check(file) || PyUnicode_Check(file)) { - file = npy_PyFile_OpenFile(file, "wb"); + Py_SETREF(file, npy_PyFile_OpenFile(file, "wb")); if (file == NULL) { return NULL; } own = 1; } else { - Py_INCREF(file); own = 0; } @@ -1046,7 +1051,7 @@ any_array_ufunc_overrides(PyObject *args, PyObject *kwds) NPY_NO_EXPORT PyObject * -array_ufunc(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_ufunc(PyArrayObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { PyObject *ufunc, *method_name, *normal_args, *ufunc_method; PyObject *result = NULL; @@ -1095,7 +1100,7 @@ cleanup: } static PyObject * -array_function(PyArrayObject *self, PyObject *c_args, PyObject *c_kwds) +array_function(PyArrayObject *NPY_UNUSED(self), PyObject *c_args, PyObject *c_kwds) { PyObject *func, *types, *args, *kwargs, *result; static char *kwlist[] = {"func", "types", "args", "kwargs", NULL}; @@ -1174,7 +1179,7 @@ array_resize(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } - ret = PyArray_Resize(self, &newshape, refcheck, NPY_CORDER); + ret = PyArray_Resize(self, &newshape, refcheck, NPY_ANYORDER); npy_free_cache_dim_obj(newshape); if (ret == NULL) { return NULL; @@ -1727,7 +1732,7 @@ array_reduce(PyArrayObject *self, PyObject *NPY_UNUSED(args)) } static PyObject * -array_reduce_ex_regular(PyArrayObject *self, int protocol) +array_reduce_ex_regular(PyArrayObject *self, int NPY_UNUSED(protocol)) { PyObject *subclass_array_reduce = NULL; PyObject *ret; @@ -1856,7 +1861,7 @@ array_reduce_ex(PyArrayObject *self, PyObject *args) PyDataType_FLAGCHK(descr, NPY_ITEM_HASOBJECT) || (PyType_IsSubtype(((PyObject*)self)->ob_type, &PyArray_Type) && ((PyObject*)self)->ob_type != &PyArray_Type) || - PyDataType_ISUNSIZED(descr)) { + descr->elsize == 0) { /* The PickleBuffer class from version 5 of the pickle protocol * can only be used for arrays backed by a contiguous data buffer. * For all other cases we fallback to the generic array_reduce @@ -2035,6 +2040,7 @@ array_setstate(PyArrayObject *self, PyObject *args) #endif npy_intp num = PyArray_NBYTES(self); if (num == 0) { + Py_DECREF(rawdata); Py_RETURN_NONE; } fa->data = PyDataMem_NEW(num); @@ -2385,7 +2391,6 @@ array_clip(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * array_conjugate(PyArrayObject *self, PyObject *args) { - PyArrayObject *out = NULL; if (!PyArg_ParseTuple(args, "|O&:conjugate", PyArray_OutputConverter, diff --git a/numpy/core/src/multiarray/methods.h b/numpy/core/src/multiarray/methods.h index b96a3c8a8..7a9a24a00 100644 --- a/numpy/core/src/multiarray/methods.h +++ b/numpy/core/src/multiarray/methods.h @@ -8,7 +8,10 @@ extern NPY_NO_EXPORT PyMethodDef array_methods[]; NPY_NO_EXPORT const char * npy_casting_to_string(NPY_CASTING casting); -/* Pathlib support */ +/* + * Pathlib support, takes a borrowed reference and returns a new one. + * The new object may be the same as the old. + */ static inline PyObject * NpyPath_PathlikeToFspath(PyObject *file) { @@ -24,6 +27,7 @@ NpyPath_PathlikeToFspath(PyObject *file) } if (!PyObject_IsInstance(file, os_PathLike)) { + Py_INCREF(file); return file; } return PyObject_CallFunctionObjArgs(os_fspath, file, NULL); diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 084a5dd46..9169814c2 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -118,6 +118,9 @@ PyArray_GetPriority(PyObject *obj, double default_) ret = PyArray_LookupSpecial_OnInstance(obj, "__array_priority__"); if (ret == NULL) { + if (PyErr_Occurred()) { + PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + } return default_; } @@ -286,45 +289,26 @@ PyArray_AsCArray(PyObject **op, void *ptr, npy_intp *dims, int nd, * Convert to a 1D C-array */ NPY_NO_EXPORT int -PyArray_As1D(PyObject **op, char **ptr, int *d1, int typecode) +PyArray_As1D(PyObject **NPY_UNUSED(op), char **NPY_UNUSED(ptr), + int *NPY_UNUSED(d1), int NPY_UNUSED(typecode)) { - npy_intp newd1; - PyArray_Descr *descr; - static const char msg[] = "PyArray_As1D: use PyArray_AsCArray."; - /* 2008-07-14, 1.5 */ - if (DEPRECATE(msg) < 0) { - return -1; - } - descr = PyArray_DescrFromType(typecode); - if (PyArray_AsCArray(op, (void *)ptr, &newd1, 1, descr) == -1) { - return -1; - } - *d1 = (int) newd1; - return 0; + PyErr_SetString(PyExc_NotImplementedError, + "PyArray_As1D: use PyArray_AsCArray."); + return -1; } /*NUMPY_API * Convert to a 2D C-array */ NPY_NO_EXPORT int -PyArray_As2D(PyObject **op, char ***ptr, int *d1, int *d2, int typecode) +PyArray_As2D(PyObject **NPY_UNUSED(op), char ***NPY_UNUSED(ptr), + int *NPY_UNUSED(d1), int *NPY_UNUSED(d2), int NPY_UNUSED(typecode)) { - npy_intp newdims[2]; - PyArray_Descr *descr; - static const char msg[] = "PyArray_As1D: use PyArray_AsCArray."; - /* 2008-07-14, 1.5 */ - if (DEPRECATE(msg) < 0) { - return -1; - } - descr = PyArray_DescrFromType(typecode); - if (PyArray_AsCArray(op, (void *)ptr, newdims, 2, descr) == -1) { - return -1; - } - *d1 = (int ) newdims[0]; - *d2 = (int ) newdims[1]; - return 0; + PyErr_SetString(PyExc_NotImplementedError, + "PyArray_As2D: use PyArray_AsCArray."); + return -1; } /* End Deprecated */ @@ -1131,6 +1115,14 @@ _pyarray_correlate(PyArrayObject *ap1, PyArrayObject *ap2, int typenum, n1 = PyArray_DIMS(ap1)[0]; n2 = PyArray_DIMS(ap2)[0]; + if (n1 == 0) { + PyErr_SetString(PyExc_ValueError, "first array argument cannot be empty"); + return NULL; + } + if (n2 == 0) { + PyErr_SetString(PyExc_ValueError, "second array argument cannot be empty"); + return NULL; + } if (n1 < n2) { ret = ap1; ap1 = ap2; @@ -2082,25 +2074,28 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) if (file == NULL) { return NULL; } - + if (offset != 0 && strcmp(sep, "") != 0) { PyErr_SetString(PyExc_TypeError, "'offset' argument only permitted for binary files"); + Py_XDECREF(type); + Py_DECREF(file); return NULL; } if (PyString_Check(file) || PyUnicode_Check(file)) { - file = npy_PyFile_OpenFile(file, "rb"); + Py_SETREF(file, npy_PyFile_OpenFile(file, "rb")); if (file == NULL) { + Py_XDECREF(type); return NULL; } own = 1; } else { - Py_INCREF(file); own = 0; } fp = npy_PyFile_Dup2(file, "rb", &orig_pos); if (fp == NULL) { Py_DECREF(file); + Py_XDECREF(type); return NULL; } if (npy_fseek(fp, offset, SEEK_CUR) != 0) { @@ -3281,7 +3276,7 @@ array_datetime_data(PyObject *NPY_UNUSED(dummy), PyObject *args) } meta = get_datetime_metadata_from_dtype(dtype); - Py_DECREF(dtype); + Py_DECREF(dtype); if (meta == NULL) { return NULL; } @@ -3798,7 +3793,7 @@ _vec_string_no_args(PyArrayObject* char_array, } static PyObject * -_vec_string(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) +_vec_string(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *NPY_UNUSED(kwds)) { PyArrayObject* char_array = NULL; PyArray_Descr *type; @@ -3824,9 +3819,11 @@ _vec_string(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) else { PyErr_SetString(PyExc_TypeError, "string operation on non-string array"); + Py_DECREF(type); goto err; } if (method == NULL) { + Py_DECREF(type); goto err; } diff --git a/numpy/core/src/multiarray/nditer_api.c b/numpy/core/src/multiarray/nditer_api.c index db0bfcece..e7fe0fa50 100644 --- a/numpy/core/src/multiarray/nditer_api.c +++ b/numpy/core/src/multiarray/nditer_api.c @@ -371,8 +371,8 @@ NpyIter_ResetToIterIndexRange(NpyIter *iter, } if (errmsg == NULL) { PyErr_Format(PyExc_ValueError, - "Out-of-bounds range [%d, %d) passed to " - "ResetToIterIndexRange", (int)istart, (int)iend); + "Out-of-bounds range [%" NPY_INTP_FMT ", %" NPY_INTP_FMT ") passed to " + "ResetToIterIndexRange", istart, iend); } else { *errmsg = "Out-of-bounds range passed to ResetToIterIndexRange"; @@ -382,8 +382,8 @@ NpyIter_ResetToIterIndexRange(NpyIter *iter, else if (iend < istart) { if (errmsg == NULL) { PyErr_Format(PyExc_ValueError, - "Invalid range [%d, %d) passed to ResetToIterIndexRange", - (int)istart, (int)iend); + "Invalid range [%" NPY_INTP_FMT ", %" NPY_INTP_FMT ") passed to ResetToIterIndexRange", + istart, iend); } else { *errmsg = "Invalid range passed to ResetToIterIndexRange"; @@ -1429,8 +1429,8 @@ NpyIter_DebugPrint(NpyIter *iter) printf("REUSE_REDUCE_LOOPS "); printf("\n"); - printf("| NDim: %d\n", (int)ndim); - printf("| NOp: %d\n", (int)nop); + printf("| NDim: %d\n", ndim); + printf("| NOp: %d\n", nop); if (NIT_MASKOP(iter) >= 0) { printf("| MaskOp: %d\n", (int)NIT_MASKOP(iter)); } diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 3b3635afe..5e770338d 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -24,7 +24,7 @@ static int npyiter_check_global_flags(npy_uint32 flags, npy_uint32* itflags); static int npyiter_check_op_axes(int nop, int oa_ndim, int **op_axes, - npy_intp *itershape); + const npy_intp *itershape); static int npyiter_calculate_ndim(int nop, PyArrayObject **op_in, int oa_ndim); @@ -55,7 +55,7 @@ npyiter_check_casting(int nop, PyArrayObject **op, static int npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itflags, char **op_dataptr, - npy_uint32 *op_flags, int **op_axes, + const npy_uint32 *op_flags, int **op_axes, npy_intp *itershape); static void npyiter_replace_axisdata(NpyIter *iter, int iop, @@ -74,23 +74,23 @@ static void npyiter_find_best_axis_ordering(NpyIter *iter); static PyArray_Descr * npyiter_get_common_dtype(int nop, PyArrayObject **op, - npyiter_opitflags *op_itflags, PyArray_Descr **op_dtype, + const npyiter_opitflags *op_itflags, PyArray_Descr **op_dtype, PyArray_Descr **op_request_dtypes, int only_inputs); static PyArrayObject * npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, npy_uint32 flags, npyiter_opitflags *op_itflags, int op_ndim, npy_intp *shape, - PyArray_Descr *op_dtype, int *op_axes); + PyArray_Descr *op_dtype, const int *op_axes); static int npyiter_allocate_arrays(NpyIter *iter, npy_uint32 flags, PyArray_Descr **op_dtype, PyTypeObject *subtype, - npy_uint32 *op_flags, npyiter_opitflags *op_itflags, + const npy_uint32 *op_flags, npyiter_opitflags *op_itflags, int **op_axes); static void npyiter_get_priority_subtype(int nop, PyArrayObject **op, - npyiter_opitflags *op_itflags, + const npyiter_opitflags *op_itflags, double *subtype_priority, PyTypeObject **subtype); static int npyiter_allocate_transfer_functions(NpyIter *iter); @@ -154,7 +154,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, if (nop > NPY_MAXARGS) { PyErr_Format(PyExc_ValueError, "Cannot construct an iterator with more than %d operands " - "(%d were requested)", (int)NPY_MAXARGS, (int)nop); + "(%d were requested)", NPY_MAXARGS, nop); return NULL; } @@ -787,7 +787,7 @@ npyiter_check_global_flags(npy_uint32 flags, npy_uint32* itflags) static int npyiter_check_op_axes(int nop, int oa_ndim, int **op_axes, - npy_intp *itershape) + const npy_intp *itershape) { char axes_dupcheck[NPY_MAXDIMS]; int iop, idim; @@ -810,7 +810,7 @@ npyiter_check_op_axes(int nop, int oa_ndim, int **op_axes, PyErr_Format(PyExc_ValueError, "Cannot construct an iterator with more than %d dimensions " "(%d were requested for op_axes)", - (int)NPY_MAXDIMS, oa_ndim); + NPY_MAXDIMS, oa_ndim); return 0; } if (op_axes == NULL) { @@ -826,14 +826,14 @@ npyiter_check_op_axes(int nop, int oa_ndim, int **op_axes, if (axes != NULL) { memset(axes_dupcheck, 0, NPY_MAXDIMS); for (idim = 0; idim < oa_ndim; ++idim) { - npy_intp i = axes[idim]; + int i = axes[idim]; if (i >= 0) { if (i >= NPY_MAXDIMS) { PyErr_Format(PyExc_ValueError, "The 'op_axes' provided to the iterator " "constructor for operand %d " "contained invalid " - "values %d", (int)iop, (int)i); + "values %d", iop, i); return 0; } else if (axes_dupcheck[i] == 1) { @@ -841,7 +841,7 @@ npyiter_check_op_axes(int nop, int oa_ndim, int **op_axes, "The 'op_axes' provided to the iterator " "constructor for operand %d " "contained duplicate " - "value %d", (int)iop, (int)i); + "value %d", iop, i); return 0; } else { @@ -1311,7 +1311,7 @@ npyiter_check_casting(int nop, PyArrayObject **op, PyObject *errmsg; errmsg = PyUString_FromFormat( "Iterator operand %d dtype could not be cast from ", - (int)iop); + iop); PyUString_ConcatAndDel(&errmsg, PyObject_Repr((PyObject *)PyArray_DESCR(op[iop]))); PyUString_ConcatAndDel(&errmsg, @@ -1342,7 +1342,7 @@ npyiter_check_casting(int nop, PyArrayObject **op, PyUString_ConcatAndDel(&errmsg, PyUString_FromFormat(", the operand %d dtype, " "according to the rule %s", - (int)iop, + iop, npyiter_casting_to_string(casting))); PyErr_SetObject(PyExc_TypeError, errmsg); Py_DECREF(errmsg); @@ -1423,7 +1423,7 @@ check_mask_for_writemasked_reduction(NpyIter *iter, int iop) static int npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itflags, char **op_dataptr, - npy_uint32 *op_flags, int **op_axes, + const npy_uint32 *op_flags, int **op_axes, npy_intp *itershape) { npy_uint32 itflags = NIT_ITFLAGS(iter); @@ -1500,8 +1500,8 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf "Iterator input op_axes[%d][%d] (==%d) " "is not a valid axis of op[%d], which " "has %d dimensions ", - (int)iop, (int)(ndim-idim-1), (int)i, - (int)iop, (int)ondim); + iop, (ndim-idim-1), i, + iop, ondim); return 0; } } @@ -2409,7 +2409,7 @@ npyiter_find_best_axis_ordering(NpyIter *iter) */ static PyArray_Descr * npyiter_get_common_dtype(int nop, PyArrayObject **op, - npyiter_opitflags *op_itflags, PyArray_Descr **op_dtype, + const npyiter_opitflags *op_itflags, PyArray_Descr **op_dtype, PyArray_Descr **op_request_dtypes, int only_inputs) { @@ -2477,7 +2477,7 @@ static PyArrayObject * npyiter_new_temp_array(NpyIter *iter, PyTypeObject *subtype, npy_uint32 flags, npyiter_opitflags *op_itflags, int op_ndim, npy_intp *shape, - PyArray_Descr *op_dtype, int *op_axes) + PyArray_Descr *op_dtype, const int *op_axes) { npy_uint32 itflags = NIT_ITFLAGS(iter); int idim, ndim = NIT_NDIM(iter); @@ -2706,7 +2706,7 @@ static int npyiter_allocate_arrays(NpyIter *iter, npy_uint32 flags, PyArray_Descr **op_dtype, PyTypeObject *subtype, - npy_uint32 *op_flags, npyiter_opitflags *op_itflags, + const npy_uint32 *op_flags, npyiter_opitflags *op_itflags, int **op_axes) { npy_uint32 itflags = NIT_ITFLAGS(iter); @@ -3109,7 +3109,7 @@ npyiter_allocate_arrays(NpyIter *iter, */ static void npyiter_get_priority_subtype(int nop, PyArrayObject **op, - npyiter_opitflags *op_itflags, + const npyiter_opitflags *op_itflags, double *subtype_priority, PyTypeObject **subtype) { diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 25d21025e..246f9d382 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -82,7 +82,8 @@ static int npyiter_cache_values(NewNpyArrayIterObject *self) } static PyObject * -npyiter_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +npyiter_new(PyTypeObject *subtype, PyObject *NPY_UNUSED(args), + PyObject *NPY_UNUSED(kwds)) { NewNpyArrayIterObject *self; @@ -572,6 +573,7 @@ npyiter_convert_op_axes(PyObject *op_axes_in, int nop, if (*oa_ndim > NPY_MAXDIMS) { PyErr_SetString(PyExc_ValueError, "Too many dimensions in op_axes"); + Py_DECREF(a); return 0; } } @@ -602,8 +604,8 @@ npyiter_convert_op_axes(PyObject *op_axes_in, int nop, } Py_DECREF(v); } - Py_DECREF(a); } + Py_DECREF(a); } if (*oa_ndim == -1) { @@ -2014,7 +2016,7 @@ npyiter_seq_item(NewNpyArrayIterObject *self, Py_ssize_t i) if (i < 0 || i >= nop) { PyErr_Format(PyExc_IndexError, - "Iterator operand index %d is out of bounds", (int)i_orig); + "Iterator operand index %zd is out of bounds", i_orig); return NULL; } @@ -2028,7 +2030,7 @@ npyiter_seq_item(NewNpyArrayIterObject *self, Py_ssize_t i) */ if (!self->readflags[i]) { PyErr_Format(PyExc_RuntimeError, - "Iterator operand %d is write-only", (int)i); + "Iterator operand %zd is write-only", i); return NULL; } #endif @@ -2145,12 +2147,12 @@ npyiter_seq_ass_item(NewNpyArrayIterObject *self, Py_ssize_t i, PyObject *v) if (i < 0 || i >= nop) { PyErr_Format(PyExc_IndexError, - "Iterator operand index %d is out of bounds", (int)i_orig); + "Iterator operand index %zd is out of bounds", i_orig); return -1; } if (!self->writeflags[i]) { PyErr_Format(PyExc_RuntimeError, - "Iterator operand %d is not writeable", (int)i_orig); + "Iterator operand %zd is not writeable", i_orig); return -1; } @@ -2364,7 +2366,7 @@ npyiter_close(NewNpyArrayIterObject *self) } static PyObject * -npyiter_exit(NewNpyArrayIterObject *self, PyObject *args) +npyiter_exit(NewNpyArrayIterObject *self, PyObject *NPY_UNUSED(args)) { /* even if called via exception handling, writeback any data */ return npyiter_close(self); diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index 0ceb994ef..dabc866ff 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -391,7 +391,8 @@ array_matrix_multiply(PyArrayObject *m1, PyObject *m2) } static PyObject * -array_inplace_matrix_multiply(PyArrayObject *m1, PyObject *m2) +array_inplace_matrix_multiply( + PyArrayObject *NPY_UNUSED(m1), PyObject *NPY_UNUSED(m2)) { PyErr_SetString(PyExc_TypeError, "In-place matrix multiplication is not (yet) supported. " diff --git a/numpy/core/src/multiarray/refcount.c b/numpy/core/src/multiarray/refcount.c index b8230c81a..6033929d9 100644 --- a/numpy/core/src/multiarray/refcount.c +++ b/numpy/core/src/multiarray/refcount.c @@ -11,6 +11,7 @@ #define _MULTIARRAYMODULE #include "numpy/arrayobject.h" #include "numpy/arrayscalars.h" +#include "iterators.h" #include "npy_config.h" @@ -210,21 +211,22 @@ PyArray_XDECREF(PyArrayObject *mp) npy_intp i, n; PyObject **data; PyObject *temp; - PyArrayIterObject *it; + /* + * statically allocating it allows this function to not modify the + * reference count of the array for use during dealloc. + * (statically is not necessary as such) + */ + PyArrayIterObject it; if (!PyDataType_REFCHK(PyArray_DESCR(mp))) { return 0; } if (PyArray_DESCR(mp)->type_num != NPY_OBJECT) { - it = (PyArrayIterObject *)PyArray_IterNew((PyObject *)mp); - if (it == NULL) { - return -1; + PyArray_RawIterBaseInit(&it, mp); + while(it.index < it.size) { + PyArray_Item_XDECREF(it.dataptr, PyArray_DESCR(mp)); + PyArray_ITER_NEXT(&it); } - while(it->index < it->size) { - PyArray_Item_XDECREF(it->dataptr, PyArray_DESCR(mp)); - PyArray_ITER_NEXT(it); - } - Py_DECREF(it); return 0; } @@ -242,16 +244,12 @@ PyArray_XDECREF(PyArrayObject *mp) } } else { /* handles misaligned data too */ - it = (PyArrayIterObject *)PyArray_IterNew((PyObject *)mp); - if (it == NULL) { - return -1; - } - while(it->index < it->size) { - NPY_COPY_PYOBJECT_PTR(&temp, it->dataptr); + PyArray_RawIterBaseInit(&it, mp); + while(it.index < it.size) { + NPY_COPY_PYOBJECT_PTR(&temp, it.dataptr); Py_XDECREF(temp); - PyArray_ITER_NEXT(it); + PyArray_ITER_NEXT(&it); } - Py_DECREF(it); } return 0; } diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index cfb21f50e..5da7f7738 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -4060,8 +4060,11 @@ initialize_casting_tables(void) _npy_can_cast_safely_table[_FROM_NUM][NPY_STRING] = 1; _npy_can_cast_safely_table[_FROM_NUM][NPY_UNICODE] = 1; - /* Allow casts from any integer to the TIMEDELTA type */ -#if @from_isint@ || @from_isuint@ +#if @from_isint@ && NPY_SIZEOF_TIMEDELTA >= _FROM_BSIZE + /* Allow casts from smaller or equal signed integers to the TIMEDELTA type */ + _npy_can_cast_safely_table[_FROM_NUM][NPY_TIMEDELTA] = 1; +#elif @from_isuint@ && NPY_SIZEOF_TIMEDELTA > _FROM_BSIZE + /* Allow casts from smaller unsigned integers to the TIMEDELTA type */ _npy_can_cast_safely_table[_FROM_NUM][NPY_TIMEDELTA] = 1; #endif @@ -4492,6 +4495,36 @@ initialize_numeric_types(void) PyArrayIter_Type.tp_iter = PyObject_SelfIter; PyArrayMapIter_Type.tp_iter = PyObject_SelfIter; + + /* + * Give types different names when they are the same size (gh-9799). + * `np.intX` always refers to the first int of that size in the sequence + * `['LONG', 'LONGLONG', 'INT', 'SHORT', 'BYTE']`. + */ +#if (NPY_SIZEOF_BYTE == NPY_SIZEOF_SHORT) + PyByteArrType_Type.tp_name = "numpy.byte"; + PyUByteArrType_Type.tp_name = "numpy.ubyte"; +#endif +#if (NPY_SIZEOF_SHORT == NPY_SIZEOF_INT) + PyShortArrType_Type.tp_name = "numpy.short"; + PyUShortArrType_Type.tp_name = "numpy.ushort"; +#endif +#if (NPY_SIZEOF_INT == NPY_SIZEOF_LONG) + PyIntArrType_Type.tp_name = "numpy.intc"; + PyUIntArrType_Type.tp_name = "numpy.uintc"; +#endif +#if (NPY_SIZEOF_LONGLONG == NPY_SIZEOF_LONG) + PyLongLongArrType_Type.tp_name = "numpy.longlong"; + PyULongLongArrType_Type.tp_name = "numpy.ulonglong"; +#endif + + /* + Do the same for longdouble + */ +#if (NPY_SIZEOF_LONGDOUBLE == NPY_SIZEOF_DOUBLE) + PyLongDoubleArrType_Type.tp_name = "numpy.longdouble"; + PyCLongDoubleArrType_Type.tp_name = "numpy.clongdouble"; +#endif } typedef struct { diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index 1dffdeaed..4e31f003b 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -26,7 +26,7 @@ static int _fix_unknown_dimension(PyArray_Dims *newshape, PyArrayObject *arr); static int -_attempt_nocopy_reshape(PyArrayObject *self, int newnd, npy_intp* newdims, +_attempt_nocopy_reshape(PyArrayObject *self, int newnd, const npy_intp *newdims, npy_intp *newstrides, int is_f_order); static void @@ -40,7 +40,7 @@ _putzero(char *optr, PyObject *zero, PyArray_Descr *dtype); */ NPY_NO_EXPORT PyObject * PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck, - NPY_ORDER order) + NPY_ORDER NPY_UNUSED(order)) { npy_intp oldnbytes, newnbytes; npy_intp oldsize, newsize; @@ -361,7 +361,7 @@ _putzero(char *optr, PyObject *zero, PyArray_Descr *dtype) * stride of the next-fastest index. */ static int -_attempt_nocopy_reshape(PyArrayObject *self, int newnd, npy_intp* newdims, +_attempt_nocopy_reshape(PyArrayObject *self, int newnd, const npy_intp *newdims, npy_intp *newstrides, int is_f_order) { int oldnd; @@ -766,7 +766,7 @@ static int _npy_stride_sort_item_comparator(const void *a, const void *b) * [(2, 12), (0, 4), (1, -2)]. */ NPY_NO_EXPORT void -PyArray_CreateSortedStridePerm(int ndim, npy_intp *strides, +PyArray_CreateSortedStridePerm(int ndim, npy_intp const *strides, npy_stride_sort_item *out_strideperm) { int i; @@ -1048,7 +1048,7 @@ build_shape_string(npy_intp n, npy_intp *vals) * from a reduction result once its computation is complete. */ NPY_NO_EXPORT void -PyArray_RemoveAxesInPlace(PyArrayObject *arr, npy_bool *flags) +PyArray_RemoveAxesInPlace(PyArrayObject *arr, const npy_bool *flags) { PyArrayObject_fields *fa = (PyArrayObject_fields *)arr; npy_intp *shape = fa->dimensions, *strides = fa->strides; diff --git a/numpy/core/src/multiarray/vdot.c b/numpy/core/src/multiarray/vdot.c index 424a21710..9b5d19522 100644 --- a/numpy/core/src/multiarray/vdot.c +++ b/numpy/core/src/multiarray/vdot.c @@ -15,17 +15,17 @@ CFLOAT_vdot(char *ip1, npy_intp is1, char *ip2, npy_intp is2, char *op, npy_intp n, void *NPY_UNUSED(ignore)) { #if defined(HAVE_CBLAS) - int is1b = blas_stride(is1, sizeof(npy_cfloat)); - int is2b = blas_stride(is2, sizeof(npy_cfloat)); + CBLAS_INT is1b = blas_stride(is1, sizeof(npy_cfloat)); + CBLAS_INT is2b = blas_stride(is2, sizeof(npy_cfloat)); if (is1b && is2b) { double sum[2] = {0., 0.}; /* double for stability */ while (n > 0) { - int chunk = n < NPY_CBLAS_CHUNK ? n : NPY_CBLAS_CHUNK; + CBLAS_INT chunk = n < NPY_CBLAS_CHUNK ? n : NPY_CBLAS_CHUNK; float tmp[2]; - cblas_cdotc_sub((int)n, ip1, is1b, ip2, is2b, tmp); + CBLAS_FUNC(cblas_cdotc_sub)((CBLAS_INT)n, ip1, is1b, ip2, is2b, tmp); sum[0] += (double)tmp[0]; sum[1] += (double)tmp[1]; /* use char strides here */ @@ -66,17 +66,17 @@ CDOUBLE_vdot(char *ip1, npy_intp is1, char *ip2, npy_intp is2, char *op, npy_intp n, void *NPY_UNUSED(ignore)) { #if defined(HAVE_CBLAS) - int is1b = blas_stride(is1, sizeof(npy_cdouble)); - int is2b = blas_stride(is2, sizeof(npy_cdouble)); + CBLAS_INT is1b = blas_stride(is1, sizeof(npy_cdouble)); + CBLAS_INT is2b = blas_stride(is2, sizeof(npy_cdouble)); if (is1b && is2b) { double sum[2] = {0., 0.}; /* double for stability */ while (n > 0) { - int chunk = n < NPY_CBLAS_CHUNK ? n : NPY_CBLAS_CHUNK; + CBLAS_INT chunk = n < NPY_CBLAS_CHUNK ? n : NPY_CBLAS_CHUNK; double tmp[2]; - cblas_zdotc_sub((int)n, ip1, is1b, ip2, is2b, tmp); + CBLAS_FUNC(cblas_zdotc_sub)((CBLAS_INT)n, ip1, is1b, ip2, is2b, tmp); sum[0] += (double)tmp[0]; sum[1] += (double)tmp[1]; /* use char strides here */ diff --git a/numpy/core/src/npymath/npy_math_complex.c.src b/numpy/core/src/npymath/npy_math_complex.c.src index dad381232..8c432e483 100644 --- a/numpy/core/src/npymath/npy_math_complex.c.src +++ b/numpy/core/src/npymath/npy_math_complex.c.src @@ -40,13 +40,14 @@ * flag in an efficient way. The flag is IEEE specific. See * https://github.com/freebsd/freebsd/blob/4c6378299/lib/msun/src/catrig.c#L42 */ +#if !defined(HAVE_CACOSF) || !defined(HAVE_CACOSL) || !defined(HAVE_CASINHF) || !defined(HAVE_CASINHL) #define raise_inexact() do { \ volatile npy_float NPY_UNUSED(junk) = 1 + tiny; \ } while (0) static const volatile npy_float tiny = 3.9443045e-31f; - +#endif /**begin repeat * #type = npy_float, npy_double, npy_longdouble# @@ -64,9 +65,6 @@ static const volatile npy_float tiny = 3.9443045e-31f; * Constants *=========================================================*/ static const @ctype@ c_1@c@ = {1.0@C@, 0.0}; -static const @ctype@ c_half@c@ = {0.5@C@, 0.0}; -static const @ctype@ c_i@c@ = {0.0, 1.0@C@}; -static const @ctype@ c_ihalf@c@ = {0.0, 0.5@C@}; /*========================================================== * Helper functions @@ -76,22 +74,6 @@ static const @ctype@ c_ihalf@c@ = {0.0, 0.5@C@}; *=========================================================*/ static NPY_INLINE @ctype@ -cadd@c@(@ctype@ a, @ctype@ b) -{ - return npy_cpack@c@(npy_creal@c@(a) + npy_creal@c@(b), - npy_cimag@c@(a) + npy_cimag@c@(b)); -} - -static NPY_INLINE -@ctype@ -csub@c@(@ctype@ a, @ctype@ b) -{ - return npy_cpack@c@(npy_creal@c@(a) - npy_creal@c@(b), - npy_cimag@c@(a) - npy_cimag@c@(b)); -} - -static NPY_INLINE -@ctype@ cmul@c@(@ctype@ a, @ctype@ b) { @type@ ar, ai, br, bi; @@ -132,20 +114,6 @@ cdiv@c@(@ctype@ a, @ctype@ b) } } -static NPY_INLINE -@ctype@ -cneg@c@(@ctype@ a) -{ - return npy_cpack@c@(-npy_creal@c@(a), -npy_cimag@c@(a)); -} - -static NPY_INLINE -@ctype@ -cmuli@c@(@ctype@ a) -{ - return npy_cpack@c@(-npy_cimag@c@(a), npy_creal@c@(a)); -} - /*========================================================== * Custom implementation of missing complex C99 functions *=========================================================*/ diff --git a/numpy/core/src/npymath/npy_math_internal.h.src b/numpy/core/src/npymath/npy_math_internal.h.src index fa820baac..18b6d1434 100644 --- a/numpy/core/src/npymath/npy_math_internal.h.src +++ b/numpy/core/src/npymath/npy_math_internal.h.src @@ -716,3 +716,44 @@ npy_@func@@c@(@type@ a, @type@ b) return npy_@func@u@c@(a < 0 ? -a : a, b < 0 ? -b : b); } /**end repeat**/ + +/* Unlike LCM and GCD, we need byte and short variants for the shift operators, + * since the result is dependent on the width of the type + */ +/**begin repeat + * + * #type = byte, short, int, long, longlong# + * #c = hh,h,,l,ll# + */ +/**begin repeat1 + * + * #u = u,# + * #is_signed = 0,1# + */ +NPY_INPLACE npy_@u@@type@ +npy_lshift@u@@c@(npy_@u@@type@ a, npy_@u@@type@ b) +{ + if (NPY_LIKELY((size_t)b < sizeof(a) * CHAR_BIT)) { + return a << b; + } + else { + return 0; + } +} +NPY_INPLACE npy_@u@@type@ +npy_rshift@u@@c@(npy_@u@@type@ a, npy_@u@@type@ b) +{ + if (NPY_LIKELY((size_t)b < sizeof(a) * CHAR_BIT)) { + return a >> b; + } +#if @is_signed@ + else if (a < 0) { + return (npy_@u@@type@)-1; /* preserve the sign bit */ + } +#endif + else { + return 0; + } +} +/**end repeat1**/ +/**end repeat**/ diff --git a/numpy/core/src/npysort/npysort_common.h b/numpy/core/src/npysort/npysort_common.h index 5fd03b96f..2a6e4d421 100644 --- a/numpy/core/src/npysort/npysort_common.h +++ b/numpy/core/src/npysort/npysort_common.h @@ -329,6 +329,14 @@ UNICODE_LT(const npy_ucs4 *s1, const npy_ucs4 *s2, size_t len) NPY_INLINE static int DATETIME_LT(npy_datetime a, npy_datetime b) { + if (a == NPY_DATETIME_NAT) { + return 0; + } + + if (b == NPY_DATETIME_NAT) { + return 1; + } + return a < b; } @@ -336,6 +344,14 @@ DATETIME_LT(npy_datetime a, npy_datetime b) NPY_INLINE static int TIMEDELTA_LT(npy_timedelta a, npy_timedelta b) { + if (a == NPY_DATETIME_NAT) { + return 0; + } + + if (b == NPY_DATETIME_NAT) { + return 1; + } + return a < b; } diff --git a/numpy/core/src/umath/_rational_tests.c.src b/numpy/core/src/umath/_rational_tests.c.src index 9e74845df..615e395c7 100644 --- a/numpy/core/src/umath/_rational_tests.c.src +++ b/numpy/core/src/umath/_rational_tests.c.src @@ -539,11 +539,11 @@ static PyObject* pyrational_str(PyObject* self) { rational x = ((PyRational*)self)->r; if (d(x)!=1) { - return PyString_FromFormat( + return PyUString_FromFormat( "%ld/%ld",(long)x.n,(long)d(x)); } else { - return PyString_FromFormat( + return PyUString_FromFormat( "%ld",(long)x.n); } } diff --git a/numpy/core/src/umath/cpuid.c b/numpy/core/src/umath/cpuid.c index 8673f1736..72c6493e8 100644 --- a/numpy/core/src/umath/cpuid.c +++ b/numpy/core/src/umath/cpuid.c @@ -48,6 +48,25 @@ int os_avx512_support(void) #endif } +static NPY_INLINE +int cpu_supports_fma(void) +{ +#ifdef __x86_64__ + unsigned int feature = 0x01; + unsigned int a, b, c, d; + __asm__ volatile ( + "cpuid" "\n\t" + : "=a" (a), "=b" (b), "=c" (c), "=d" (d) + : "a" (feature)); + /* + * FMA is the 12th bit of ECX + */ + return (c >> 12) & 1; +#else + return 0; +#endif +} + /* * Primitive cpu feature detect function * Currently only supports checking for avx on gcc compatible compilers. @@ -63,6 +82,9 @@ npy_cpu_supports(const char * feature) return 0; #endif } + else if (strcmp(feature, "fma") == 0) { + return cpu_supports_fma() && __builtin_cpu_supports("avx2") && os_avx_support(); + } else if (strcmp(feature, "avx2") == 0) { return __builtin_cpu_supports("avx2") && os_avx_support(); } diff --git a/numpy/core/src/umath/funcs.inc.src b/numpy/core/src/umath/funcs.inc.src index c2732f925..10ed66e50 100644 --- a/numpy/core/src/umath/funcs.inc.src +++ b/numpy/core/src/umath/funcs.inc.src @@ -161,7 +161,7 @@ npy_ObjectLogicalNot(PyObject *i1) static PyObject * npy_ObjectFloor(PyObject *obj) { - PyObject *math_floor_func = NULL; + static PyObject *math_floor_func = NULL; npy_cache_import("math", "floor", &math_floor_func); if (math_floor_func == NULL) { @@ -172,7 +172,7 @@ npy_ObjectFloor(PyObject *obj) { static PyObject * npy_ObjectCeil(PyObject *obj) { - PyObject *math_ceil_func = NULL; + static PyObject *math_ceil_func = NULL; npy_cache_import("math", "ceil", &math_ceil_func); if (math_ceil_func == NULL) { @@ -183,7 +183,7 @@ npy_ObjectCeil(PyObject *obj) { static PyObject * npy_ObjectTrunc(PyObject *obj) { - PyObject *math_trunc_func = NULL; + static PyObject *math_trunc_func = NULL; npy_cache_import("math", "trunc", &math_trunc_func); if (math_trunc_func == NULL) { @@ -228,7 +228,8 @@ npy_ObjectGCD(PyObject *i1, PyObject *i2) return NULL; } /* _gcd has some unusual behaviour regarding sign */ - return PyNumber_Absolute(gcd); + Py_SETREF(gcd, PyNumber_Absolute(gcd)); + return gcd; } } @@ -246,17 +247,19 @@ npy_ObjectLCM(PyObject *i1, PyObject *i2) * no remainder */ tmp = PyNumber_FloorDivide(i1, gcd); + Py_DECREF(gcd); if(tmp == NULL) { return NULL; } - tmp = PyNumber_Multiply(tmp, i2); + Py_SETREF(tmp, PyNumber_Multiply(tmp, i2)); if(tmp == NULL) { return NULL; } /* even though we fix gcd to be positive, we need to do it again here */ - return PyNumber_Absolute(tmp); + Py_SETREF(tmp, PyNumber_Absolute(tmp)); + return tmp; } diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index 1931cd100..8a2e5bc40 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -54,210 +54,123 @@ ** GENERIC FLOAT LOOPS ** *****************************************************************************/ +/* direct loops using a suitable callback */ -typedef float halfUnaryFunc(npy_half x); -typedef float floatUnaryFunc(float x); -typedef double doubleUnaryFunc(double x); -typedef npy_longdouble longdoubleUnaryFunc(npy_longdouble x); -typedef npy_half halfBinaryFunc(npy_half x, npy_half y); -typedef float floatBinaryFunc(float x, float y); -typedef double doubleBinaryFunc(double x, double y); -typedef npy_longdouble longdoubleBinaryFunc(npy_longdouble x, npy_longdouble y); - +/**begin repeat + * #c = e, f, d, g# + * #type = npy_half, npy_float, npy_double, npy_longdouble# + **/ /*UFUNC_API*/ NPY_NO_EXPORT void -PyUFunc_e_e(char **args, npy_intp *dimensions, npy_intp *steps, void *func) +PyUFunc_@c@_@c@(char **args, npy_intp *dimensions, npy_intp *steps, void *func) { - halfUnaryFunc *f = (halfUnaryFunc *)func; + typedef @type@ func_type(@type@); + func_type *f = (func_type *)func; UNARY_LOOP { - const npy_half in1 = *(npy_half *)ip1; - *(npy_half *)op1 = f(in1); + const @type@ in1 = *(@type@ *)ip1; + *(@type@ *)op1 = f(in1); } } /*UFUNC_API*/ NPY_NO_EXPORT void -PyUFunc_e_e_As_f_f(char **args, npy_intp *dimensions, npy_intp *steps, void *func) +PyUFunc_@c@@c@_@c@(char **args, npy_intp *dimensions, npy_intp *steps, void *func) { - floatUnaryFunc *f = (floatUnaryFunc *)func; - UNARY_LOOP { - const float in1 = npy_half_to_float(*(npy_half *)ip1); - *(npy_half *)op1 = npy_float_to_half(f(in1)); + typedef @type@ func_type(@type@, @type@); + func_type *f = (func_type *)func; + BINARY_LOOP { + @type@ in1 = *(@type@ *)ip1; + @type@ in2 = *(@type@ *)ip2; + *(@type@ *)op1 = f(in1, in2); } } -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_e_e_As_d_d(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - doubleUnaryFunc *f = (doubleUnaryFunc *)func; - UNARY_LOOP { - const double in1 = npy_half_to_double(*(npy_half *)ip1); - *(npy_half *)op1 = npy_double_to_half(f(in1)); - } -} +/**end repeat**/ -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_f_f(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - floatUnaryFunc *f = (floatUnaryFunc *)func; - UNARY_LOOP { - const float in1 = *(float *)ip1; - *(float *)op1 = f(in1); - } -} +/* indirect loops with casting */ +/**begin repeat + * #c1 = e, e, f# + * #type1 = npy_half, npy_half, npy_float# + * #c2 = f, d, d# + * #type2 = npy_float, npy_double, npy_double# + * + * #conv12 = npy_half_to_float, npy_half_to_double, (double)# + * #conv21 = npy_float_to_half, npy_double_to_half, (float)# + **/ /*UFUNC_API*/ NPY_NO_EXPORT void -PyUFunc_f_f_As_d_d(char **args, npy_intp *dimensions, npy_intp *steps, void *func) +PyUFunc_@c1@_@c1@_As_@c2@_@c2@(char **args, npy_intp *dimensions, npy_intp *steps, void *func) { - doubleUnaryFunc *f = (doubleUnaryFunc *)func; + typedef @type2@ func_type(@type2@); + func_type *f = (func_type *)func; UNARY_LOOP { - const float in1 = *(float *)ip1; - *(float *)op1 = (float)f((double)in1); - } -} - -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_ee_e(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - halfBinaryFunc *f = (halfBinaryFunc *)func; - BINARY_LOOP { - npy_half in1 = *(npy_half *)ip1; - npy_half in2 = *(npy_half *)ip2; - *(npy_half *)op1 = f(in1, in2); - } -} - -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_ee_e_As_ff_f(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - floatBinaryFunc *f = (floatBinaryFunc *)func; - BINARY_LOOP { - float in1 = npy_half_to_float(*(npy_half *)ip1); - float in2 = npy_half_to_float(*(npy_half *)ip2); - *(npy_half *)op1 = npy_float_to_half(f(in1, in2)); + const @type2@ in1 = @conv12@(*(@type1@ *)ip1); + *(@type1@ *)op1 = @conv21@(f(in1)); } } - /*UFUNC_API*/ NPY_NO_EXPORT void -PyUFunc_ee_e_As_dd_d(char **args, npy_intp *dimensions, npy_intp *steps, void *func) +PyUFunc_@c1@@c1@_@c1@_As_@c2@@c2@_@c2@(char **args, npy_intp *dimensions, npy_intp *steps, void *func) { - doubleBinaryFunc *f = (doubleBinaryFunc *)func; + typedef @type2@ func_type(@type2@, @type2@); + func_type *f = (func_type *)func; BINARY_LOOP { - double in1 = npy_half_to_double(*(npy_half *)ip1); - double in2 = npy_half_to_double(*(npy_half *)ip2); - *(npy_half *)op1 = npy_double_to_half(f(in1, in2)); + const @type2@ in1 = @conv12@(*(@type1@ *)ip1); + const @type2@ in2 = @conv12@(*(@type1@ *)ip2); + *(@type1@ *)op1 = @conv21@(f(in1, in2)); } } -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_ff_f(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - floatBinaryFunc *f = (floatBinaryFunc *)func; - BINARY_LOOP { - float in1 = *(float *)ip1; - float in2 = *(float *)ip2; - *(float *)op1 = f(in1, in2); - } -} - -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_ff_f_As_dd_d(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - doubleBinaryFunc *f = (doubleBinaryFunc *)func; - BINARY_LOOP { - float in1 = *(float *)ip1; - float in2 = *(float *)ip2; - *(float *)op1 = (double)f((double)in1, (double)in2); - } -} +/**end repeat**/ -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_d_d(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - doubleUnaryFunc *f = (doubleUnaryFunc *)func; - UNARY_LOOP { - double in1 = *(double *)ip1; - *(double *)op1 = f(in1); - } -} +/****************************************************************************** + ** GENERIC COMPLEX LOOPS ** + *****************************************************************************/ -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_dd_d(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - doubleBinaryFunc *f = (doubleBinaryFunc *)func; - BINARY_LOOP { - double in1 = *(double *)ip1; - double in2 = *(double *)ip2; - *(double *)op1 = f(in1, in2); - } -} +/* direct loops using a suitable callback */ +/**begin repeat + * #c = F, D, G# + * #type = npy_cfloat, npy_cdouble, npy_clongdouble# + **/ /*UFUNC_API*/ NPY_NO_EXPORT void -PyUFunc_g_g(char **args, npy_intp *dimensions, npy_intp *steps, void *func) +PyUFunc_@c@_@c@(char **args, npy_intp *dimensions, npy_intp *steps, void *func) { - longdoubleUnaryFunc *f = (longdoubleUnaryFunc *)func; + typedef void func_type(@type@ *, @type@ *); + func_type *f = (func_type *)func; UNARY_LOOP { - npy_longdouble in1 = *(npy_longdouble *)ip1; - *(npy_longdouble *)op1 = f(in1); + @type@ in1 = *(@type@ *)ip1; + @type@ *out = (@type@ *)op1; + f(&in1, out); } } /*UFUNC_API*/ NPY_NO_EXPORT void -PyUFunc_gg_g(char **args, npy_intp *dimensions, npy_intp *steps, void *func) +PyUFunc_@c@@c@_@c@(char **args, npy_intp *dimensions, npy_intp *steps, void *func) { - longdoubleBinaryFunc *f = (longdoubleBinaryFunc *)func; + typedef void func_type(@type@ *, @type@ *, @type@ *); + func_type *f = (func_type *)func; BINARY_LOOP { - npy_longdouble in1 = *(npy_longdouble *)ip1; - npy_longdouble in2 = *(npy_longdouble *)ip2; - *(npy_longdouble *)op1 = f(in1, in2); + @type@ in1 = *(@type@ *)ip1; + @type@ in2 = *(@type@ *)ip2; + @type@ *out = (@type@ *)op1; + f(&in1, &in2, out); } } +/**end repeat**/ - -/****************************************************************************** - ** GENERIC COMPLEX LOOPS ** - *****************************************************************************/ - - -typedef void cdoubleUnaryFunc(npy_cdouble *x, npy_cdouble *r); -typedef void cfloatUnaryFunc(npy_cfloat *x, npy_cfloat *r); -typedef void clongdoubleUnaryFunc(npy_clongdouble *x, npy_clongdouble *r); -typedef void cdoubleBinaryFunc(npy_cdouble *x, npy_cdouble *y, npy_cdouble *r); -typedef void cfloatBinaryFunc(npy_cfloat *x, npy_cfloat *y, npy_cfloat *r); -typedef void clongdoubleBinaryFunc(npy_clongdouble *x, npy_clongdouble *y, - npy_clongdouble *r); - -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_F_F(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - cfloatUnaryFunc *f = (cfloatUnaryFunc *)func; - UNARY_LOOP { - npy_cfloat in1 = *(npy_cfloat *)ip1; - npy_cfloat *out = (npy_cfloat *)op1; - f(&in1, out); - } -} - +/* indirect loops with casting */ /*UFUNC_API*/ NPY_NO_EXPORT void PyUFunc_F_F_As_D_D(char **args, npy_intp *dimensions, npy_intp *steps, void *func) { - cdoubleUnaryFunc *f = (cdoubleUnaryFunc *)func; + typedef void func_type(npy_cdouble *, npy_cdouble *); + func_type *f = (func_type *)func; UNARY_LOOP { npy_cdouble tmp, out; tmp.real = (double)((float *)ip1)[0]; @@ -270,22 +183,10 @@ PyUFunc_F_F_As_D_D(char **args, npy_intp *dimensions, npy_intp *steps, void *fun /*UFUNC_API*/ NPY_NO_EXPORT void -PyUFunc_FF_F(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - cfloatBinaryFunc *f = (cfloatBinaryFunc *)func; - BINARY_LOOP { - npy_cfloat in1 = *(npy_cfloat *)ip1; - npy_cfloat in2 = *(npy_cfloat *)ip2; - npy_cfloat *out = (npy_cfloat *)op1; - f(&in1, &in2, out); - } -} - -/*UFUNC_API*/ -NPY_NO_EXPORT void PyUFunc_FF_F_As_DD_D(char **args, npy_intp *dimensions, npy_intp *steps, void *func) { - cdoubleBinaryFunc *f = (cdoubleBinaryFunc *)func; + typedef void func_type(npy_cdouble *, npy_cdouble *, npy_cdouble *); + func_type *f = (func_type *)func; BINARY_LOOP { npy_cdouble tmp1, tmp2, out; tmp1.real = (double)((float *)ip1)[0]; @@ -298,56 +199,6 @@ PyUFunc_FF_F_As_DD_D(char **args, npy_intp *dimensions, npy_intp *steps, void *f } } -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_D_D(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - cdoubleUnaryFunc *f = (cdoubleUnaryFunc *)func; - UNARY_LOOP { - npy_cdouble in1 = *(npy_cdouble *)ip1; - npy_cdouble *out = (npy_cdouble *)op1; - f(&in1, out); - } -} - -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_DD_D(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - cdoubleBinaryFunc *f = (cdoubleBinaryFunc *)func; - BINARY_LOOP { - npy_cdouble in1 = *(npy_cdouble *)ip1; - npy_cdouble in2 = *(npy_cdouble *)ip2; - npy_cdouble *out = (npy_cdouble *)op1; - f(&in1, &in2, out); - } -} - -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_G_G(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - clongdoubleUnaryFunc *f = (clongdoubleUnaryFunc *)func; - UNARY_LOOP { - npy_clongdouble in1 = *(npy_clongdouble *)ip1; - npy_clongdouble *out = (npy_clongdouble *)op1; - f(&in1, out); - } -} - -/*UFUNC_API*/ -NPY_NO_EXPORT void -PyUFunc_GG_G(char **args, npy_intp *dimensions, npy_intp *steps, void *func) -{ - clongdoubleBinaryFunc *f = (clongdoubleBinaryFunc *)func; - BINARY_LOOP { - npy_clongdouble in1 = *(npy_clongdouble *)ip1; - npy_clongdouble in2 = *(npy_clongdouble *)ip2; - npy_clongdouble *out = (npy_clongdouble *)op1; - f(&in1, &in2, out); - } -} - /****************************************************************************** ** GENERIC OBJECT lOOPS ** @@ -398,6 +249,7 @@ PyUFunc_O_O_method(char **args, npy_intp *dimensions, npy_intp *steps, void *fun i, type->tp_name, meth); npy_PyErr_ChainExceptionsCause(exc, val, tb); Py_DECREF(tup); + Py_XDECREF(func); return; } ret = PyObject_Call(func, tup, NULL); @@ -702,6 +554,7 @@ BOOL_@kind@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED * #ftype = npy_float, npy_float, npy_float, npy_float, npy_double, npy_double, * npy_double, npy_double, npy_double, npy_double# * #SIGNED = 1, 0, 1, 0, 1, 0, 1, 0, 1, 0# + * #c = hh,uhh,h,uh,,u,l,ul,ll,ull# */ #define @TYPE@_floor_divide @TYPE@_divide @@ -779,16 +632,15 @@ NPY_NO_EXPORT NPY_GCC_OPT_3 @ATTR@ void /**begin repeat2 * Arithmetic - * #kind = add, subtract, multiply, bitwise_and, bitwise_or, bitwise_xor, - * left_shift, right_shift# - * #OP = +, -,*, &, |, ^, <<, >># + * #kind = add, subtract, multiply, bitwise_and, bitwise_or, bitwise_xor# + * #OP = +, -, *, &, |, ^# */ #if @CHK@ NPY_NO_EXPORT NPY_GCC_OPT_3 @ATTR@ void @TYPE@_@kind@@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)) { - if(IS_BINARY_REDUCE) { + if (IS_BINARY_REDUCE) { BINARY_REDUCE_LOOP(@type@) { io1 @OP@= *(@type@ *)ip2; } @@ -802,6 +654,47 @@ NPY_NO_EXPORT NPY_GCC_OPT_3 @ATTR@ void /**end repeat2**/ +/* + * Arithmetic bit shift operations. + * + * Intel hardware masks bit shift values, so large shifts wrap around + * and can produce surprising results. The special handling ensures that + * behavior is independent of compiler or hardware. + * TODO: We could implement consistent behavior for negative shifts, + * which is undefined in C. + */ + +#define INT_left_shift_needs_clear_floatstatus +#define UINT_left_shift_needs_clear_floatstatus + +NPY_NO_EXPORT NPY_GCC_OPT_3 void +@TYPE@_left_shift@isa@(char **args, npy_intp *dimensions, npy_intp *steps, + void *NPY_UNUSED(func)) +{ + BINARY_LOOP_FAST(@type@, @type@, *out = npy_lshift@c@(in1, in2)); + +#ifdef @TYPE@_left_shift_needs_clear_floatstatus + // For some reason, our macOS CI sets an "invalid" flag here, but only + // for some types. + npy_clear_floatstatus_barrier((char*)dimensions); +#endif +} + +#undef INT_left_shift_needs_clear_floatstatus +#undef UINT_left_shift_needs_clear_floatstatus + +NPY_NO_EXPORT +#ifndef NPY_DO_NOT_OPTIMIZE_@TYPE@_right_shift +NPY_GCC_OPT_3 +#endif +void +@TYPE@_right_shift@isa@(char **args, npy_intp *dimensions, npy_intp *steps, + void *NPY_UNUSED(func)) +{ + BINARY_LOOP_FAST(@type@, @type@, *out = npy_rshift@c@(in1, in2)); +} + + /**begin repeat2 * #kind = equal, not_equal, greater, greater_equal, less, less_equal, * logical_and, logical_or# @@ -1208,6 +1101,12 @@ NPY_NO_EXPORT void } NPY_NO_EXPORT void +@TYPE@_isinf(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)) +{ + UNARY_LOOP_FAST(npy_bool, npy_bool, (void)in; *out = NPY_FALSE); +} + +NPY_NO_EXPORT void @TYPE@__ones_like(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)) { OUTPUT_LOOP { @@ -1256,13 +1155,36 @@ NPY_NO_EXPORT void const @type@ in1 = *(@type@ *)ip1; const @type@ in2 = *(@type@ *)ip2; if (in1 == NPY_DATETIME_NAT) { + *((@type@ *)op1) = in1; + } + else if (in2 == NPY_DATETIME_NAT) { + *((@type@ *)op1) = in2; + } + else { + *((@type@ *)op1) = (in1 @OP@ in2) ? in1 : in2; + } + } +} +/**end repeat1**/ + +/**begin repeat1 + * #kind = fmax, fmin# + * #OP = >=, <=# + **/ +NPY_NO_EXPORT void +@TYPE@_@kind@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)) +{ + BINARY_LOOP { + const @type@ in1 = *(@type@ *)ip1; + const @type@ in2 = *(@type@ *)ip2; + if (in1 == NPY_DATETIME_NAT) { *((@type@ *)op1) = in2; } else if (in2 == NPY_DATETIME_NAT) { *((@type@ *)op1) = in1; } else { - *((@type@ *)op1) = (in1 @OP@ in2) ? in1 : in2; + *((@type@ *)op1) = in1 @OP@ in2 ? in1 : in2; } } } @@ -1597,8 +1519,32 @@ NPY_NO_EXPORT void /**end repeat**/ /**begin repeat - * #func = exp, log# - * #scalarf = npy_expf, npy_logf# + * #func = rint, ceil, floor, trunc# + * #scalarf = npy_rint, npy_ceil, npy_floor, npy_trunc# + */ + +/**begin repeat1 +* #TYPE = FLOAT, DOUBLE# +* #type = npy_float, npy_double# +* #typesub = f, # +*/ + +NPY_NO_EXPORT NPY_GCC_OPT_3 void +@TYPE@_@func@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)) +{ + UNARY_LOOP { + const @type@ in1 = *(@type@ *)ip1; + *(@type@ *)op1 = @scalarf@@typesub@(in1); + } +} + + +/**end repeat1**/ +/**end repeat**/ + +/**begin repeat + * #func = sin, cos, exp, log# + * #scalarf = npy_sinf, npy_cosf, npy_expf, npy_logf# */ NPY_NO_EXPORT NPY_GCC_OPT_3 void @@ -1613,12 +1559,84 @@ FLOAT_@func@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSE /**end repeat**/ /**begin repeat - * #isa = avx512f, avx2# - * #ISA = AVX512F, AVX2# + * #isa = avx512f, fma# + * #ISA = AVX512F, FMA# * #CHK = HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS, HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS# */ /**begin repeat1 + * #TYPE = FLOAT, DOUBLE# + * #type = npy_float, npy_double# + * #typesub = f, # + */ + +NPY_NO_EXPORT NPY_GCC_OPT_3 void +@TYPE@_sqrt_@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)) +{ + if (!run_unary_@isa@_sqrt_@TYPE@(args, dimensions, steps)) { + UNARY_LOOP { + const @type@ in1 = *(@type@ *)ip1; + *(@type@ *)op1 = npy_sqrt@typesub@(in1); + } + } +} + +NPY_NO_EXPORT NPY_GCC_OPT_3 void +@TYPE@_absolute_@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)) +{ + if (!run_unary_@isa@_absolute_@TYPE@(args, dimensions, steps)) { + UNARY_LOOP { + const @type@ in1 = *(@type@ *)ip1; + const @type@ tmp = in1 > 0 ? in1 : -in1; + /* add 0 to clear -0.0 */ + *((@type@ *)op1) = tmp + 0; + } + } + npy_clear_floatstatus_barrier((char*)dimensions); +} + +NPY_NO_EXPORT NPY_GCC_OPT_3 void +@TYPE@_square_@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)) +{ + if (!run_unary_@isa@_square_@TYPE@(args, dimensions, steps)) { + UNARY_LOOP { + const @type@ in1 = *(@type@ *)ip1; + *(@type@ *)op1 = in1*in1; + } + } +} + +NPY_NO_EXPORT NPY_GCC_OPT_3 void +@TYPE@_reciprocal_@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)) +{ + if (!run_unary_@isa@_reciprocal_@TYPE@(args, dimensions, steps)) { + UNARY_LOOP { + const @type@ in1 = *(@type@ *)ip1; + *(@type@ *)op1 = 1.0f/in1; + } + } +} + +/**begin repeat2 + * #func = rint, ceil, floor, trunc# + * #scalarf = npy_rint, npy_ceil, npy_floor, npy_trunc# + */ + +NPY_NO_EXPORT NPY_GCC_OPT_3 void +@TYPE@_@func@_@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)) +{ + if (!run_unary_@isa@_@func@_@TYPE@(args, dimensions, steps)) { + UNARY_LOOP { + const @type@ in1 = *(@type@ *)ip1; + *(@type@ *)op1 = @scalarf@@typesub@(in1); + } + } +} + +/**end repeat2**/ +/**end repeat1**/ + +/**begin repeat1 * #func = exp, log# * #scalarf = npy_expf, npy_logf# */ @@ -1645,8 +1663,32 @@ FLOAT_@func@_@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY } /**end repeat1**/ + +/**begin repeat1 + * #func = cos, sin# + * #enum = npy_compute_cos, npy_compute_sin# + * #scalarf = npy_cosf, npy_sinf# + */ + +NPY_NO_EXPORT NPY_GCC_OPT_3 void +FLOAT_@func@_@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)) +{ + if (!run_unary_@isa@_sincos_FLOAT(args, dimensions, steps, @enum@)) { + UNARY_LOOP { +#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS + @ISA@_sincos_FLOAT((npy_float *)op1, (npy_float *)ip1, 1, steps[0], @enum@); +#else + const npy_float in1 = *(npy_float *)ip1; + *(npy_float *)op1 = @scalarf@(in1); +#endif + } + } +} + +/**end repeat1**/ /**end repeat**/ + /**begin repeat * Float types * #type = npy_float, npy_double, npy_longdouble, npy_float# diff --git a/numpy/core/src/umath/loops.h.src b/numpy/core/src/umath/loops.h.src index 7f05a693a..7558de0bb 100644 --- a/numpy/core/src/umath/loops.h.src +++ b/numpy/core/src/umath/loops.h.src @@ -7,14 +7,12 @@ #define _NPY_UMATH_LOOPS_H_ #define BOOL_invert BOOL_logical_not -#define BOOL_negative BOOL_logical_not #define BOOL_add BOOL_logical_or #define BOOL_bitwise_and BOOL_logical_and #define BOOL_bitwise_or BOOL_logical_or #define BOOL_logical_xor BOOL_not_equal #define BOOL_bitwise_xor BOOL_logical_xor #define BOOL_multiply BOOL_logical_and -#define BOOL_subtract BOOL_logical_xor #define BOOL_maximum BOOL_logical_or #define BOOL_minimum BOOL_logical_and #define BOOL_fmax BOOL_maximum @@ -175,16 +173,29 @@ NPY_NO_EXPORT void */ NPY_NO_EXPORT void @TYPE@_sqrt(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); + +/**begin repeat1 + * #isa = avx512f, fma# + */ + +/**begin repeat2 + * #func = sqrt, absolute, square, reciprocal# + */ +NPY_NO_EXPORT void +@TYPE@_@func@_@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); + +/**end repeat2**/ +/**end repeat1**/ /**end repeat**/ /**begin repeat - * #func = exp, log# + * #func = sin, cos, exp, log# */ NPY_NO_EXPORT void FLOAT_@func@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); /**begin repeat1 - * #isa = avx512f, avx2# + * #isa = avx512f, fma# */ NPY_NO_EXPORT void @@ -194,6 +205,26 @@ FLOAT_@func@_@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY /**end repeat**/ /**begin repeat + * #func = rint, ceil, floor, trunc# + */ + +/**begin repeat1 +* #TYPE = FLOAT, DOUBLE# +*/ + +NPY_NO_EXPORT NPY_GCC_OPT_3 void +@TYPE@_@func@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)); + +/**begin repeat2 + * #isa = avx512f, fma# + */ +NPY_NO_EXPORT NPY_GCC_OPT_3 void +@TYPE@_@func@_@isa@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)); +/**end repeat2**/ +/**end repeat1**/ +/**end repeat**/ + +/**begin repeat * Float types * #TYPE = HALF, FLOAT, DOUBLE, LONGDOUBLE# * #c = f, f, , l# @@ -447,6 +478,11 @@ NPY_NO_EXPORT void @TYPE@_isfinite(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); NPY_NO_EXPORT void +@TYPE@_isinf(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); + +#define @TYPE@_isnan @TYPE@_isnat + +NPY_NO_EXPORT void @TYPE@__ones_like(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(data)); /**begin repeat1 @@ -458,8 +494,7 @@ NPY_NO_EXPORT void /**end repeat1**/ /**begin repeat1 - * #kind = maximum, minimum# - * #OP = >, <# + * #kind = maximum, minimum, fmin, fmax# **/ NPY_NO_EXPORT void @TYPE@_@kind@(char **args, npy_intp *dimensions, npy_intp *steps, void *NPY_UNUSED(func)); @@ -523,10 +558,6 @@ TIMEDELTA_mm_qm_divmod(char **args, npy_intp *dimensions, npy_intp *steps, void #define TIMEDELTA_mq_m_floor_divide TIMEDELTA_mq_m_divide #define TIMEDELTA_md_m_floor_divide TIMEDELTA_md_m_divide /* #define TIMEDELTA_mm_d_floor_divide TIMEDELTA_mm_d_divide */ -#define TIMEDELTA_fmin TIMEDELTA_minimum -#define TIMEDELTA_fmax TIMEDELTA_maximum -#define DATETIME_fmin DATETIME_minimum -#define DATETIME_fmax DATETIME_maximum /* ***************************************************************************** diff --git a/numpy/core/src/umath/matmul.c.src b/numpy/core/src/umath/matmul.c.src index b5204eca5..c8f7c654d 100644 --- a/numpy/core/src/umath/matmul.c.src +++ b/numpy/core/src/umath/matmul.c.src @@ -31,7 +31,11 @@ * -1 to be conservative, in case blas internally uses a for loop with an * inclusive upper bound */ +#ifndef HAVE_BLAS_ILP64 #define BLAS_MAXSIZE (NPY_MAX_INT - 1) +#else +#define BLAS_MAXSIZE (NPY_MAX_INT64 - 1) +#endif /* * Determine if a 2d matrix can be used by BLAS @@ -84,25 +88,25 @@ NPY_NO_EXPORT void * op: data in c order, m shape */ enum CBLAS_ORDER order; - int M, N, lda; + CBLAS_INT M, N, lda; assert(m <= BLAS_MAXSIZE && n <= BLAS_MAXSIZE); assert (is_blasable2d(is2_n, sizeof(@typ@), n, 1, sizeof(@typ@))); - M = (int)m; - N = (int)n; + M = (CBLAS_INT)m; + N = (CBLAS_INT)n; if (is_blasable2d(is1_m, is1_n, m, n, sizeof(@typ@))) { order = CblasColMajor; - lda = (int)(is1_m / sizeof(@typ@)); + lda = (CBLAS_INT)(is1_m / sizeof(@typ@)); } else { /* If not ColMajor, caller should have ensured we are RowMajor */ /* will not assert in release mode */ order = CblasRowMajor; assert(is_blasable2d(is1_n, is1_m, n, m, sizeof(@typ@))); - lda = (int)(is1_n / sizeof(@typ@)); + lda = (CBLAS_INT)(is1_n / sizeof(@typ@)); } - cblas_@prefix@gemv(order, CblasTrans, N, M, @step1@, ip1, lda, ip2, + CBLAS_FUNC(cblas_@prefix@gemv)(order, CblasTrans, N, M, @step1@, ip1, lda, ip2, is2_n / sizeof(@typ@), @step0@, op, op_m / sizeof(@typ@)); } @@ -117,37 +121,37 @@ NPY_NO_EXPORT void */ enum CBLAS_ORDER order = CblasRowMajor; enum CBLAS_TRANSPOSE trans1, trans2; - int M, N, P, lda, ldb, ldc; + CBLAS_INT M, N, P, lda, ldb, ldc; assert(m <= BLAS_MAXSIZE && n <= BLAS_MAXSIZE && p <= BLAS_MAXSIZE); - M = (int)m; - N = (int)n; - P = (int)p; + M = (CBLAS_INT)m; + N = (CBLAS_INT)n; + P = (CBLAS_INT)p; assert(is_blasable2d(os_m, os_p, m, p, sizeof(@typ@))); - ldc = (int)(os_m / sizeof(@typ@)); + ldc = (CBLAS_INT)(os_m / sizeof(@typ@)); if (is_blasable2d(is1_m, is1_n, m, n, sizeof(@typ@))) { trans1 = CblasNoTrans; - lda = (int)(is1_m / sizeof(@typ@)); + lda = (CBLAS_INT)(is1_m / sizeof(@typ@)); } else { /* If not ColMajor, caller should have ensured we are RowMajor */ /* will not assert in release mode */ assert(is_blasable2d(is1_n, is1_m, n, m, sizeof(@typ@))); trans1 = CblasTrans; - lda = (int)(is1_n / sizeof(@typ@)); + lda = (CBLAS_INT)(is1_n / sizeof(@typ@)); } if (is_blasable2d(is2_n, is2_p, n, p, sizeof(@typ@))) { trans2 = CblasNoTrans; - ldb = (int)(is2_n / sizeof(@typ@)); + ldb = (CBLAS_INT)(is2_n / sizeof(@typ@)); } else { /* If not ColMajor, caller should have ensured we are RowMajor */ /* will not assert in release mode */ assert(is_blasable2d(is2_p, is2_n, p, n, sizeof(@typ@))); trans2 = CblasTrans; - ldb = (int)(is2_p / sizeof(@typ@)); + ldb = (CBLAS_INT)(is2_p / sizeof(@typ@)); } /* * Use syrk if we have a case of a matrix times its transpose. @@ -162,12 +166,14 @@ NPY_NO_EXPORT void ) { npy_intp i,j; if (trans1 == CblasNoTrans) { - cblas_@prefix@syrk(order, CblasUpper, trans1, P, N, @step1@, - ip1, lda, @step0@, op, ldc); + CBLAS_FUNC(cblas_@prefix@syrk)( + order, CblasUpper, trans1, P, N, @step1@, + ip1, lda, @step0@, op, ldc); } else { - cblas_@prefix@syrk(order, CblasUpper, trans1, P, N, @step1@, - ip1, ldb, @step0@, op, ldc); + CBLAS_FUNC(cblas_@prefix@syrk)( + order, CblasUpper, trans1, P, N, @step1@, + ip1, ldb, @step0@, op, ldc); } /* Copy the triangle */ for (i = 0; i < P; i++) { @@ -178,8 +184,9 @@ NPY_NO_EXPORT void } else { - cblas_@prefix@gemm(order, trans1, trans2, M, P, N, @step1@, ip1, lda, - ip2, ldb, @step0@, op, ldc); + CBLAS_FUNC(cblas_@prefix@gemm)( + order, trans1, trans2, M, P, N, @step1@, ip1, lda, + ip2, ldb, @step0@, op, ldc); } } diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c index 8d67f96ac..43bed425c 100644 --- a/numpy/core/src/umath/override.c +++ b/numpy/core/src/umath/override.c @@ -494,32 +494,18 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, } else { /* not a tuple */ - if (nout > 1 && DEPRECATE("passing a single argument to the " - "'out' keyword argument of a " - "ufunc with\n" - "more than one output will " - "result in an error in the " - "future") < 0) { - /* - * If the deprecation is removed, also remove the loop - * below setting tuple items to None (but keep this future - * error message.) - */ + if (nout > 1) { PyErr_SetString(PyExc_TypeError, "'out' must be a tuple of arguments"); goto fail; } if (out != Py_None) { /* not already a tuple and not None */ - PyObject *out_tuple = PyTuple_New(nout); + PyObject *out_tuple = PyTuple_New(1); if (out_tuple == NULL) { goto fail; } - for (i = 1; i < nout; i++) { - Py_INCREF(Py_None); - PyTuple_SET_ITEM(out_tuple, i, Py_None); - } /* out was borrowed ref; make it permanent */ Py_INCREF(out); /* steals reference */ diff --git a/numpy/core/src/umath/reduction.c b/numpy/core/src/umath/reduction.c index fda2a12f6..4ce8d8ab7 100644 --- a/numpy/core/src/umath/reduction.c +++ b/numpy/core/src/umath/reduction.c @@ -36,7 +36,7 @@ * If 'dtype' isn't NULL, this function steals its reference. */ static PyArrayObject * -allocate_reduce_result(PyArrayObject *arr, npy_bool *axis_flags, +allocate_reduce_result(PyArrayObject *arr, const npy_bool *axis_flags, PyArray_Descr *dtype, int subok) { npy_intp strides[NPY_MAXDIMS], stride; @@ -84,7 +84,7 @@ allocate_reduce_result(PyArrayObject *arr, npy_bool *axis_flags, * The return value is a view into 'out'. */ static PyArrayObject * -conform_reduce_result(int ndim, npy_bool *axis_flags, +conform_reduce_result(int ndim, const npy_bool *axis_flags, PyArrayObject *out, int keepdims, const char *funcname, int need_copy) { @@ -251,7 +251,7 @@ PyArray_CreateReduceResult(PyArrayObject *operand, PyArrayObject *out, * Count the number of dimensions selected in 'axis_flags' */ static int -count_axes(int ndim, npy_bool *axis_flags) +count_axes(int ndim, const npy_bool *axis_flags) { int idim; int naxes = 0; @@ -299,7 +299,7 @@ count_axes(int ndim, npy_bool *axis_flags) NPY_NO_EXPORT PyArrayObject * PyArray_InitializeReduceResult( PyArrayObject *result, PyArrayObject *operand, - npy_bool *axis_flags, + const npy_bool *axis_flags, npy_intp *out_skip_first_count, const char *funcname) { npy_intp *strides, *shape, shape_orig[NPY_MAXDIMS]; @@ -528,7 +528,9 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, NPY_ITER_ALIGNED; if (wheremask != NULL) { op[2] = wheremask; - op_dtypes[2] = PyArray_DescrFromType(NPY_BOOL); + /* wheremask is guaranteed to be NPY_BOOL, so borrow its reference */ + op_dtypes[2] = PyArray_DESCR(wheremask); + assert(op_dtypes[2]->type_num == NPY_BOOL); if (op_dtypes[2] == NULL) { goto fail; } diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index a7987acda..d5d8d659b 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -246,25 +246,26 @@ static void /**end repeat**/ - -/* QUESTION: Should we check for overflow / underflow in (l,r)shift? */ - /**begin repeat * #name = byte, ubyte, short, ushort, int, uint, * long, ulong, longlong, ulonglong# * #type = npy_byte, npy_ubyte, npy_short, npy_ushort, npy_int, npy_uint, * npy_long, npy_ulong, npy_longlong, npy_ulonglong# + * #suffix = hh,uhh,h,uh,,u,l,ul,ll,ull# */ /**begin repeat1 - * #oper = and, xor, or, lshift, rshift# - * #op = &, ^, |, <<, >># + * #oper = and, xor, or# + * #op = &, ^, |# */ #define @name@_ctype_@oper@(arg1, arg2, out) *(out) = (arg1) @op@ (arg2) /**end repeat1**/ +#define @name@_ctype_lshift(arg1, arg2, out) *(out) = npy_lshift@suffix@(arg1, arg2) +#define @name@_ctype_rshift(arg1, arg2, out) *(out) = npy_rshift@suffix@(arg1, arg2) + /**end repeat**/ /**begin repeat @@ -405,21 +406,22 @@ half_ctype_divmod(npy_half a, npy_half b, npy_half *out1, npy_half *out2) { /**begin repeat * #name = float, double, longdouble# * #type = npy_float, npy_double, npy_longdouble# + * #c = f,,l# */ -static npy_@name@ (*_basic_@name@_pow)(@type@ a, @type@ b); static void @name@_ctype_power(@type@ a, @type@ b, @type@ *out) { - *out = _basic_@name@_pow(a, b); + *out = npy_pow@c@(a, b); } + /**end repeat**/ static void half_ctype_power(npy_half a, npy_half b, npy_half *out) { const npy_float af = npy_half_to_float(a); const npy_float bf = npy_half_to_float(b); - const npy_float outf = _basic_float_pow(af,bf); + const npy_float outf = npy_powf(af,bf); *out = npy_float_to_half(outf); } @@ -476,14 +478,10 @@ static void } /**end repeat**/ -/* - * Get the nc_powf, nc_pow, and nc_powl functions from - * the data area of the power ufunc in umathmodule. - */ - /**begin repeat * #name = cfloat, cdouble, clongdouble# * #type = npy_cfloat, npy_cdouble, npy_clongdouble# + * #c = f,,l# */ static void @name@_ctype_positive(@type@ a, @type@ *out) @@ -492,12 +490,10 @@ static void out->imag = a.imag; } -static void (*_basic_@name@_pow)(@type@ *, @type@ *, @type@ *); - static void @name@_ctype_power(@type@ a, @type@ b, @type@ *out) { - _basic_@name@_pow(&a, &b, out); + *out = npy_cpow@c@(a, b); } /**end repeat**/ @@ -570,7 +566,7 @@ static void * 1) Convert the types to the common type if both are scalars (0 return) * 2) If both are not scalars use ufunc machinery (-2 return) * 3) If both are scalars but cannot be cast to the right type - * return NotImplmented (-1 return) + * return NotImplemented (-1 return) * * 4) Perform the function on the C-type. * 5) If an error condition occurred, check to see @@ -1429,24 +1425,53 @@ static PyObject * /**begin repeat * + * #name = byte, ubyte, short, ushort, int, uint, + * long, ulong, longlong, ulonglong, + * half, float, double, longdouble, + * cfloat, cdouble, clongdouble# + * #Name = Byte, UByte, Short, UShort, Int, UInt, + * Long, ULong, LongLong, ULongLong, + * Half, Float, Double, LongDouble, + * CFloat, CDouble, CLongDouble# + * #cmplx = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1# + * #to_ctype = , , , , , , , , , , npy_half_to_double, , , , , , # + * #func = PyFloat_FromDouble*17# + */ +static NPY_INLINE PyObject * +@name@_float(PyObject *obj) +{ +#if @cmplx@ + if (emit_complexwarning() < 0) { + return NULL; + } + return @func@(@to_ctype@(PyArrayScalar_VAL(obj, @Name@).real)); +#else + return @func@(@to_ctype@(PyArrayScalar_VAL(obj, @Name@))); +#endif +} +/**end repeat**/ + + +#if !defined(NPY_PY3K) + +/**begin repeat + * * #name = (byte, ubyte, short, ushort, int, uint, * long, ulong, longlong, ulonglong, * half, float, double, longdouble, - * cfloat, cdouble, clongdouble)*2# + * cfloat, cdouble, clongdouble)# * #Name = (Byte, UByte, Short, UShort, Int, UInt, * Long, ULong, LongLong, ULongLong, * Half, Float, Double, LongDouble, - * CFloat, CDouble, CLongDouble)*2# - * #cmplx = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1)*2# - * #to_ctype = (, , , , , , , , , , npy_half_to_double, , , , , , )*2# - * #which = long*17, float*17# + * CFloat, CDouble, CLongDouble)# + * #cmplx = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1)# + * #to_ctype = (, , , , , , , , , , npy_half_to_double, , , , , , )# * #func = (PyLong_FromLongLong, PyLong_FromUnsignedLongLong)*5, * PyLong_FromDouble*3, npy_longdouble_to_PyLong, - * PyLong_FromDouble*2, npy_longdouble_to_PyLong, - * PyFloat_FromDouble*17# + * PyLong_FromDouble*2, npy_longdouble_to_PyLong# */ static NPY_INLINE PyObject * -@name@_@which@(PyObject *obj) +@name@_long(PyObject *obj) { #if @cmplx@ if (emit_complexwarning() < 0) { @@ -1459,8 +1484,6 @@ static NPY_INLINE PyObject * } /**end repeat**/ -#if !defined(NPY_PY3K) - /**begin repeat * * #name = (byte, ubyte, short, ushort, int, uint, @@ -1652,52 +1675,9 @@ add_scalarmath(void) /**end repeat**/ } -static int -get_functions(PyObject * mm) -{ - PyObject *obj; - void **funcdata; - char *signatures; - int i, j; - int ret = -1; - - /* Get the nc_pow functions */ - /* Get the pow functions */ - obj = PyObject_GetAttrString(mm, "power"); - if (obj == NULL) { - goto fail; - } - funcdata = ((PyUFuncObject *)obj)->data; - signatures = ((PyUFuncObject *)obj)->types; - - i = 0; - j = 0; - while (signatures[i] != NPY_FLOAT) { - i += 3; - j++; - } - _basic_float_pow = funcdata[j]; - _basic_double_pow = funcdata[j + 1]; - _basic_longdouble_pow = funcdata[j + 2]; - _basic_cfloat_pow = funcdata[j + 3]; - _basic_cdouble_pow = funcdata[j + 4]; - _basic_clongdouble_pow = funcdata[j + 5]; - Py_DECREF(obj); - - return ret = 0; - - fail: - Py_DECREF(mm); - return ret; -} - NPY_NO_EXPORT int initscalarmath(PyObject * m) { - if (get_functions(m) < 0) { - return -1; - } - add_scalarmath(); return 0; diff --git a/numpy/core/src/umath/simd.inc.src b/numpy/core/src/umath/simd.inc.src index 9816a1da4..74f52cc9d 100644 --- a/numpy/core/src/umath/simd.inc.src +++ b/numpy/core/src/umath/simd.inc.src @@ -130,18 +130,50 @@ abs_ptrdiff(char *a, char *b) */ /**begin repeat - * #ISA = AVX2, AVX512F# - * #isa = avx2, avx512f# + * #ISA = FMA, AVX512F# + * #isa = fma, avx512f# + * #CHK = HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS, HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS# * #REGISTER_SIZE = 32, 64# */ /* prototypes */ /**begin repeat1 + * #type = npy_float, npy_double# + * #TYPE = FLOAT, DOUBLE# + */ + +/**begin repeat2 + * #func = sqrt, absolute, square, reciprocal, rint, floor, ceil, trunc# + */ + +#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS +static NPY_INLINE NPY_GCC_TARGET_@ISA@ void +@ISA@_@func@_@TYPE@(@type@ *, @type@ *, const npy_intp n, const npy_intp stride); +#endif + +static NPY_INLINE int +run_unary_@isa@_@func@_@TYPE@(char **args, npy_intp *dimensions, npy_intp *steps) +{ +#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS + if (IS_OUTPUT_BLOCKABLE_UNARY(sizeof(@type@), @REGISTER_SIZE@)) { + @ISA@_@func@_@TYPE@((@type@*)args[1], (@type@*)args[0], dimensions[0], steps[0]); + return 1; + } + else + return 0; +#endif + return 0; +} + +/**end repeat2**/ +/**end repeat1**/ + +/**begin repeat1 * #func = exp, log# */ -#if defined HAVE_ATTRIBUTE_TARGET_@ISA@_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS static NPY_INLINE void @ISA@_@func@_FLOAT(npy_float *, npy_float *, const npy_intp n, const npy_intp stride); #endif @@ -149,7 +181,7 @@ static NPY_INLINE void static NPY_INLINE int run_unary_@isa@_@func@_FLOAT(char **args, npy_intp *dimensions, npy_intp *steps) { -#if defined HAVE_ATTRIBUTE_TARGET_@ISA@_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS if (IS_OUTPUT_BLOCKABLE_UNARY(sizeof(npy_float), @REGISTER_SIZE@)) { @ISA@_@func@_FLOAT((npy_float*)args[1], (npy_float*)args[0], dimensions[0], steps[0]); return 1; @@ -162,8 +194,26 @@ run_unary_@isa@_@func@_FLOAT(char **args, npy_intp *dimensions, npy_intp *steps) /**end repeat1**/ -/**end repeat**/ +#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS +static NPY_INLINE void +@ISA@_sincos_FLOAT(npy_float *, npy_float *, const npy_intp n, const npy_intp steps, NPY_TRIG_OP); +#endif +static NPY_INLINE int +run_unary_@isa@_sincos_FLOAT(char **args, npy_intp *dimensions, npy_intp *steps, NPY_TRIG_OP my_trig_op) +{ +#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS + if (IS_OUTPUT_BLOCKABLE_UNARY(sizeof(npy_float), @REGISTER_SIZE@)) { + @ISA@_sincos_FLOAT((npy_float*)args[1], (npy_float*)args[0], dimensions[0], steps[0], my_trig_op); + return 1; + } + else + return 0; +#endif + return 0; +} + +/**end repeat**/ /**begin repeat @@ -997,7 +1047,7 @@ sse2_sqrt_@TYPE@(@type@ * op, @type@ * ip, const npy_intp n) LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) { op[i] = @scalarf@(ip[i]); } - assert(n < (VECTOR_SIZE_BYTES / sizeof(@type@)) || + assert((npy_uintp)n < (VECTOR_SIZE_BYTES / sizeof(@type@)) || npy_is_aligned(&op[i], VECTOR_SIZE_BYTES)); if (npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)) { LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @@ -1049,7 +1099,7 @@ sse2_@kind@_@TYPE@(@type@ * op, @type@ * ip, const npy_intp n) LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) { op[i] = @scalar@_@type@(ip[i]); } - assert(n < (VECTOR_SIZE_BYTES / sizeof(@type@)) || + assert((npy_uintp)n < (VECTOR_SIZE_BYTES / sizeof(@type@)) || npy_is_aligned(&op[i], VECTOR_SIZE_BYTES)); if (npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)) { LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { @@ -1084,7 +1134,7 @@ sse2_@kind@_@TYPE@(@type@ * ip, @type@ * op, const npy_intp n) /* Order of operations important for MSVC 2015 */ *op = (*op @OP@ ip[i] || npy_isnan(*op)) ? *op : ip[i]; } - assert(n < (stride) || npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)); + assert((npy_uintp)n < (stride) || npy_is_aligned(&ip[i], VECTOR_SIZE_BYTES)); if (i + 3 * stride <= n) { /* load the first elements */ @vtype@ c1 = @vpre@_load_@vsuf@((@type@*)&ip[i]); @@ -1123,56 +1173,110 @@ sse2_@kind@_@TYPE@(@type@ * ip, @type@ * op, const npy_intp n) /* bunch of helper functions used in ISA_exp/log_FLOAT*/ #if defined HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -avx2_fmadd(__m256 a, __m256 b, __m256 c) +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_get_full_load_mask_ps(void) { - return _mm256_add_ps(_mm256_mul_ps(a, b), c); + return _mm256_set1_ps(-1.0); } -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -avx2_get_full_load_mask(void) +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256i +fma_get_full_load_mask_pd(void) { - return _mm256_set1_ps(-1.0); + return _mm256_castpd_si256(_mm256_set1_pd(-1.0)); } -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -avx2_get_partial_load_mask(const npy_int num_lanes, const npy_int total_elem) +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_get_partial_load_mask_ps(const npy_int num_elem, const npy_int num_lanes) { float maskint[16] = {-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0, 1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0}; - float* addr = maskint + total_elem - num_lanes; + float* addr = maskint + num_lanes - num_elem; return _mm256_loadu_ps(addr); } -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -avx2_masked_gather(__m256 src, - npy_float* addr, - __m256i vindex, - __m256 mask) +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256i +fma_get_partial_load_mask_pd(const npy_int num_elem, const npy_int num_lanes) +{ + npy_int maskint[16] = {-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1}; + npy_int* addr = maskint + 2*num_lanes - 2*num_elem; + return _mm256_loadu_si256((__m256i*) addr); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_masked_gather_ps(__m256 src, + npy_float* addr, + __m256i vindex, + __m256 mask) { return _mm256_mask_i32gather_ps(src, addr, vindex, mask, 4); } -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -avx2_masked_load(__m256 mask, npy_float* addr) +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256d +fma_masked_gather_pd(__m256d src, + npy_double* addr, + __m128i vindex, + __m256d mask) +{ + return _mm256_mask_i32gather_pd(src, addr, vindex, mask, 8); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_masked_load_ps(__m256 mask, npy_float* addr) { return _mm256_maskload_ps(addr, _mm256_cvtps_epi32(mask)); } -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -avx2_set_masked_lanes(__m256 x, __m256 val, __m256 mask) +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256d +fma_masked_load_pd(__m256i mask, npy_double* addr) +{ + return _mm256_maskload_pd(addr, mask); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_set_masked_lanes_ps(__m256 x, __m256 val, __m256 mask) { return _mm256_blendv_ps(x, val, mask); } -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -avx2_blend(__m256 x, __m256 y, __m256 ymask) +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256d +fma_set_masked_lanes_pd(__m256d x, __m256d val, __m256d mask) +{ + return _mm256_blendv_pd(x, val, mask); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_blend(__m256 x, __m256 y, __m256 ymask) { return _mm256_blendv_ps(x, y, ymask); } -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -avx2_get_exponent(__m256 x) +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_invert_mask_ps(__m256 ymask) +{ + return _mm256_andnot_ps(ymask, _mm256_set1_ps(-1.0)); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256i +fma_invert_mask_pd(__m256i ymask) +{ + return _mm256_andnot_si256(ymask, _mm256_set1_epi32(0xFFFFFFFF)); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_should_calculate_sine(__m256i k, __m256i andop, __m256i cmp) +{ + return _mm256_cvtepi32_ps( + _mm256_cmpeq_epi32(_mm256_and_si256(k, andop), cmp)); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_should_negate(__m256i k, __m256i andop, __m256i cmp) +{ + return fma_should_calculate_sine(k, andop, cmp); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_get_exponent(__m256 x) { /* * Special handling of denormals: @@ -1198,8 +1302,8 @@ avx2_get_exponent(__m256 x) return _mm256_blendv_ps(exp, denorm_exp, denormal_mask); } -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -avx2_get_mantissa(__m256 x) +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 +fma_get_mantissa(__m256 x) { /* * Special handling of denormals: @@ -1225,7 +1329,7 @@ avx2_get_mantissa(__m256 x) } static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -avx2_scalef_ps(__m256 poly, __m256 quadrant) +fma_scalef_ps(__m256 poly, __m256 quadrant) { /* * Handle denormals (which occur when quadrant <= -125): @@ -1263,48 +1367,145 @@ avx2_scalef_ps(__m256 poly, __m256 quadrant) } } +/**begin repeat + * #vsub = ps, pd# + * #vtype = __m256, __m256d# + */ +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA @vtype@ +fma_abs_@vsub@(@vtype@ x) +{ + return _mm256_andnot_@vsub@(_mm256_set1_@vsub@(-0.0), x); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA @vtype@ +fma_reciprocal_@vsub@(@vtype@ x) +{ + return _mm256_div_@vsub@(_mm256_set1_@vsub@(1.0f), x); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA @vtype@ +fma_rint_@vsub@(@vtype@ x) +{ + return _mm256_round_@vsub@(x, _MM_FROUND_TO_NEAREST_INT); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA @vtype@ +fma_floor_@vsub@(@vtype@ x) +{ + return _mm256_round_@vsub@(x, _MM_FROUND_TO_NEG_INF); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA @vtype@ +fma_ceil_@vsub@(@vtype@ x) +{ + return _mm256_round_@vsub@(x, _MM_FROUND_TO_POS_INF); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA @vtype@ +fma_trunc_@vsub@(@vtype@ x) +{ + return _mm256_round_@vsub@(x, _MM_FROUND_TO_ZERO); +} +/**end repeat**/ #endif #if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask16 -avx512_get_full_load_mask(void) +avx512_get_full_load_mask_ps(void) { return 0xFFFF; } +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask8 +avx512_get_full_load_mask_pd(void) +{ + return 0xFF; +} + static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask16 -avx512_get_partial_load_mask(const npy_int num_elem, const npy_int total_elem) +avx512_get_partial_load_mask_ps(const npy_int num_elem, const npy_int total_elem) { return (0x0001 << num_elem) - 0x0001; } +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask8 +avx512_get_partial_load_mask_pd(const npy_int num_elem, const npy_int total_elem) +{ + return (0x01 << num_elem) - 0x01; +} + static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512 -avx512_masked_gather(__m512 src, - npy_float* addr, - __m512i vindex, - __mmask16 kmask) +avx512_masked_gather_ps(__m512 src, + npy_float* addr, + __m512i vindex, + __mmask16 kmask) { return _mm512_mask_i32gather_ps(src, kmask, vindex, addr, 4); } +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512d +avx512_masked_gather_pd(__m512d src, + npy_double* addr, + __m256i vindex, + __mmask8 kmask) +{ + return _mm512_mask_i32gather_pd(src, kmask, vindex, addr, 8); +} + static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512 -avx512_masked_load(__mmask16 mask, npy_float* addr) +avx512_masked_load_ps(__mmask16 mask, npy_float* addr) { return _mm512_maskz_loadu_ps(mask, (__m512 *)addr); } +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512d +avx512_masked_load_pd(__mmask8 mask, npy_double* addr) +{ + return _mm512_maskz_loadu_pd(mask, (__m512d *)addr); +} + static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512 -avx512_set_masked_lanes(__m512 x, __m512 val, __mmask16 mask) +avx512_set_masked_lanes_ps(__m512 x, __m512 val, __mmask16 mask) { return _mm512_mask_blend_ps(mask, x, val); } +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512d +avx512_set_masked_lanes_pd(__m512d x, __m512d val, __mmask8 mask) +{ + return _mm512_mask_blend_pd(mask, x, val); +} + static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512 avx512_blend(__m512 x, __m512 y, __mmask16 ymask) { return _mm512_mask_mov_ps(x, ymask, y); } +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask16 +avx512_invert_mask_ps(__mmask16 ymask) +{ + return _mm512_knot(ymask); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask8 +avx512_invert_mask_pd(__mmask8 ymask) +{ + return _mm512_knot(ymask); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask16 +avx512_should_calculate_sine(__m512i k, __m512i andop, __m512i cmp) +{ + return _mm512_cmpeq_epi32_mask(_mm512_and_epi32(k, andop), cmp); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask16 +avx512_should_negate(__m512i k, __m512i andop, __m512i cmp) +{ + return avx512_should_calculate_sine(k, andop, cmp); +} + static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512 avx512_get_exponent(__m512 x) { @@ -1322,20 +1523,74 @@ avx512_scalef_ps(__m512 poly, __m512 quadrant) { return _mm512_scalef_ps(poly, quadrant); } +/**begin repeat + * #vsub = ps, pd# + * #epi_vsub = epi32, epi64# + * #vtype = __m512, __m512d# + * #and_const = 0x7fffffff, 0x7fffffffffffffffLL# + */ +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_abs_@vsub@(@vtype@ x) +{ + return (@vtype@) _mm512_and_@epi_vsub@((__m512i) x, + _mm512_set1_@epi_vsub@ (@and_const@)); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_reciprocal_@vsub@(@vtype@ x) +{ + return _mm512_div_@vsub@(_mm512_set1_@vsub@(1.0f), x); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_rint_@vsub@(@vtype@ x) +{ + return _mm512_roundscale_@vsub@(x, 0x08); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_floor_@vsub@(@vtype@ x) +{ + return _mm512_roundscale_@vsub@(x, 0x09); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_ceil_@vsub@(@vtype@ x) +{ + return _mm512_roundscale_@vsub@(x, 0x0A); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F @vtype@ +avx512_trunc_@vsub@(@vtype@ x) +{ + return _mm512_roundscale_@vsub@(x, 0x0B); +} +/**end repeat**/ #endif /**begin repeat - * #ISA = AVX2, AVX512F# - * #isa = avx2, avx512# + * #ISA = FMA, AVX512F# + * #isa = fma, avx512# * #vtype = __m256, __m512# * #vsize = 256, 512# * #or = or_ps, kor# * #vsub = , _mask# * #mask = __m256, __mmask16# - * #fmadd = avx2_fmadd,_mm512_fmadd_ps# + * #fmadd = _mm256_fmadd_ps, _mm512_fmadd_ps# + * #CHK = HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS, HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS# **/ -#if defined HAVE_ATTRIBUTE_TARGET_@ISA@_WITH_INTRINSICS +#if defined @CHK@ + +/* + * Vectorized Cody-Waite range reduction technique + * Performs the reduction step x* = x - y*C in three steps: + * 1) x* = x - y*c1 + * 2) x* = x - y*c2 + * 3) x* = x - y*c3 + * c1, c2 are exact floating points, c3 = C - c1 - c2 simulates higher precision + */ + static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ @isa@_range_reduction(@vtype@ x, @vtype@ y, @vtype@ c1, @vtype@ c2, @vtype@ c3) { @@ -1344,12 +1599,236 @@ static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ reduced_x = @fmadd@(y, c3, reduced_x); return reduced_x; } + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @mask@ +@isa@_in_range_mask(@vtype@ x, npy_float fmax, npy_float fmin) +{ + @mask@ m1 = _mm@vsize@_cmp_ps@vsub@( + x, _mm@vsize@_set1_ps(fmax), _CMP_GT_OQ); + @mask@ m2 = _mm@vsize@_cmp_ps@vsub@( + x, _mm@vsize@_set1_ps(fmin), _CMP_LT_OQ); + return _mm@vsize@_@or@(m1,m2); +} + +/* + * Approximate cosine algorithm for x \in [-PI/4, PI/4] + * Maximum ULP across all 32-bit floats = 0.875 + */ + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ +@isa@_cosine(@vtype@ x2, @vtype@ invf8, @vtype@ invf6, @vtype@ invf4, + @vtype@ invf2, @vtype@ invf0) +{ + @vtype@ cos = @fmadd@(invf8, x2, invf6); + cos = @fmadd@(cos, x2, invf4); + cos = @fmadd@(cos, x2, invf2); + cos = @fmadd@(cos, x2, invf0); + return cos; +} + +/* + * Approximate sine algorithm for x \in [-PI/4, PI/4] + * Maximum ULP across all 32-bit floats = 0.647 + */ + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ +@isa@_sine(@vtype@ x, @vtype@ x2, @vtype@ invf9, @vtype@ invf7, + @vtype@ invf5, @vtype@ invf3, + @vtype@ zero) +{ + @vtype@ sin = @fmadd@(invf9, x2, invf7); + sin = @fmadd@(sin, x2, invf5); + sin = @fmadd@(sin, x2, invf3); + sin = @fmadd@(sin, x2, zero); + sin = @fmadd@(sin, x, x); + return sin; +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ +@isa@_sqrt_ps(@vtype@ x) +{ + return _mm@vsize@_sqrt_ps(x); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@d +@isa@_sqrt_pd(@vtype@d x) +{ + return _mm@vsize@_sqrt_pd(x); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ +@isa@_square_ps(@vtype@ x) +{ + return _mm@vsize@_mul_ps(x,x); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@d +@isa@_square_pd(@vtype@d x) +{ + return _mm@vsize@_mul_pd(x,x); +} + +#endif +/**end repeat**/ + + +/**begin repeat + * #ISA = FMA, AVX512F# + * #isa = fma, avx512# + * #vsize = 256, 512# + * #BYTES = 32, 64# + * #cvtps_epi32 = _mm256_cvtps_epi32, # + * #mask = __m256, __mmask16# + * #vsub = , _mask# + * #vtype = __m256, __m512# + * #cvtps_epi32 = _mm256_cvtps_epi32, # + * #masked_store = _mm256_maskstore_ps, _mm512_mask_storeu_ps# + * #CHK = HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS, HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS# + */ + +/**begin repeat1 + * #func = sqrt, absolute, square, reciprocal, rint, ceil, floor, trunc# + * #vectorf = sqrt, abs, square, reciprocal, rint, ceil, floor, trunc# + * #replace_0_with_1 = 0, 0, 0, 1, 0, 0, 0, 0# + */ + +#if defined @CHK@ +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void +@ISA@_@func@_FLOAT(npy_float* op, + npy_float* ip, + const npy_intp array_size, + const npy_intp steps) +{ + const npy_intp stride = steps/sizeof(npy_float); + const npy_int num_lanes = @BYTES@/sizeof(npy_float); + npy_intp num_remaining_elements = array_size; + @vtype@ ones_f = _mm@vsize@_set1_ps(1.0f); + @mask@ load_mask = @isa@_get_full_load_mask_ps(); +#if @replace_0_with_1@ + @mask@ inv_load_mask = @isa@_invert_mask_ps(load_mask); +#endif + npy_int indexarr[16]; + for (npy_int ii = 0; ii < 16; ii++) { + indexarr[ii] = ii*stride; + } + @vtype@i vindex = _mm@vsize@_loadu_si@vsize@((@vtype@i*)&indexarr[0]); + + while (num_remaining_elements > 0) { + if (num_remaining_elements < num_lanes) { + load_mask = @isa@_get_partial_load_mask_ps(num_remaining_elements, + num_lanes); +#if @replace_0_with_1@ + inv_load_mask = @isa@_invert_mask_ps(load_mask); +#endif + } + @vtype@ x; + if (stride == 1) { + x = @isa@_masked_load_ps(load_mask, ip); +#if @replace_0_with_1@ + /* + * Replace masked elements with 1.0f to avoid divide by zero fp + * exception in reciprocal + */ + x = @isa@_set_masked_lanes_ps(x, ones_f, inv_load_mask); +#endif + } + else { + x = @isa@_masked_gather_ps(ones_f, ip, vindex, load_mask); + } + @vtype@ out = @isa@_@vectorf@_ps(x); + @masked_store@(op, @cvtps_epi32@(load_mask), out); + + ip += num_lanes*stride; + op += num_lanes; + num_remaining_elements -= num_lanes; + } +} +#endif +/**end repeat1**/ +/**end repeat**/ + +/**begin repeat + * #ISA = FMA, AVX512F# + * #isa = fma, avx512# + * #vsize = 256, 512# + * #BYTES = 32, 64# + * #cvtps_epi32 = _mm256_cvtps_epi32, # + * #mask = __m256i, __mmask8# + * #vsub = , _mask# + * #vtype = __m256d, __m512d# + * #vindextype = __m128i, __m256i# + * #vindexsize = 128, 256# + * #vindexload = _mm_loadu_si128, _mm256_loadu_si256# + * #cvtps_epi32 = _mm256_cvtpd_epi32, # + * #castmask = _mm256_castsi256_pd, # + * #masked_store = _mm256_maskstore_pd, _mm512_mask_storeu_pd# + * #CHK = HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS, HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS# + */ + +/**begin repeat1 + * #func = sqrt, absolute, square, reciprocal, rint, ceil, floor, trunc# + * #vectorf = sqrt, abs, square, reciprocal, rint, ceil, floor, trunc# + * #replace_0_with_1 = 0, 0, 0, 1, 0, 0, 0, 0# + */ + +#if defined @CHK@ +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void +@ISA@_@func@_DOUBLE(npy_double* op, + npy_double* ip, + const npy_intp array_size, + const npy_intp steps) +{ + const npy_intp stride = steps/sizeof(npy_double); + const npy_int num_lanes = @BYTES@/sizeof(npy_double); + npy_intp num_remaining_elements = array_size; + @mask@ load_mask = @isa@_get_full_load_mask_pd(); +#if @replace_0_with_1@ + @mask@ inv_load_mask = @isa@_invert_mask_pd(load_mask); +#endif + @vtype@ ones_d = _mm@vsize@_set1_pd(1.0f); + npy_int indexarr[8]; + for (npy_int ii = 0; ii < 8; ii++) { + indexarr[ii] = ii*stride; + } + @vindextype@ vindex = @vindexload@((@vindextype@*)&indexarr[0]); + + while (num_remaining_elements > 0) { + if (num_remaining_elements < num_lanes) { + load_mask = @isa@_get_partial_load_mask_pd(num_remaining_elements, + num_lanes); +#if @replace_0_with_1@ + inv_load_mask = @isa@_invert_mask_pd(load_mask); +#endif + } + @vtype@ x; + if (stride == 1) { + x = @isa@_masked_load_pd(load_mask, ip); +#if @replace_0_with_1@ + /* + * Replace masked elements with 1.0f to avoid divide by zero fp + * exception in reciprocal + */ + x = @isa@_set_masked_lanes_pd(x, ones_d, @castmask@(inv_load_mask)); #endif + } + else { + x = @isa@_masked_gather_pd(ones_d, ip, vindex, @castmask@(load_mask)); + } + @vtype@ out = @isa@_@vectorf@_pd(x); + @masked_store@(op, load_mask, out); + + ip += num_lanes*stride; + op += num_lanes; + num_remaining_elements -= num_lanes; + } +} +#endif +/**end repeat1**/ /**end repeat**/ /**begin repeat - * #ISA = AVX2, AVX512F# - * #isa = avx2, avx512# + * #ISA = FMA, AVX512F# + * #isa = fma, avx512# * #vtype = __m256, __m512# * #vsize = 256, 512# * #BYTES = 32, 64# @@ -1358,13 +1837,164 @@ static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ * #or_masks =_mm256_or_ps, _mm512_kor# * #and_masks =_mm256_and_ps, _mm512_kand# * #xor_masks =_mm256_xor_ps, _mm512_kxor# - * #fmadd = avx2_fmadd,_mm512_fmadd_ps# + * #fmadd = _mm256_fmadd_ps, _mm512_fmadd_ps# * #mask_to_int = _mm256_movemask_ps, # * #full_mask= 0xFF, 0xFFFF# * #masked_store = _mm256_maskstore_ps, _mm512_mask_storeu_ps# * #cvtps_epi32 = _mm256_cvtps_epi32, # + * #CHK = HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS, HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS# */ +/* + * Vectorized approximate sine/cosine algorithms: The following code is a + * vectorized version of the algorithm presented here: + * https://stackoverflow.com/questions/30463616/payne-hanek-algorithm-implementation-in-c/30465751#30465751 + * (1) Load data in ZMM/YMM registers and generate mask for elements that are + * within range [-71476.0625f, 71476.0625f] for cosine and [-117435.992f, + * 117435.992f] for sine. + * (2) For elements within range, perform range reduction using Cody-Waite's + * method: x* = x - y*PI/2, where y = rint(x*2/PI). x* \in [-PI/4, PI/4]. + * (3) Map cos(x) to (+/-)sine or (+/-)cosine of x* based on the quadrant k = + * int(y). + * (4) For elements outside that range, Cody-Waite reduction peforms poorly + * leading to catastrophic cancellation. We compute cosine by calling glibc in + * a scalar fashion. + * (5) Vectorized implementation has a max ULP of 1.49 and performs at least + * 5-7x faster than scalar implementations when magnitude of all elements in + * the array < 71476.0625f (117435.992f for sine). Worst case performance is + * when all the elements are large leading to about 1-2% reduction in + * performance. + */ + +#if defined @CHK@ +static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void +@ISA@_sincos_FLOAT(npy_float * op, + npy_float * ip, + const npy_intp array_size, + const npy_intp steps, + NPY_TRIG_OP my_trig_op) +{ + const npy_intp stride = steps/sizeof(npy_float); + const npy_int num_lanes = @BYTES@/sizeof(npy_float); + npy_float large_number = 71476.0625f; + if (my_trig_op == npy_compute_sin) { + large_number = 117435.992f; + } + + /* Load up frequently used constants */ + @vtype@i zeros = _mm@vsize@_set1_epi32(0); + @vtype@i ones = _mm@vsize@_set1_epi32(1); + @vtype@i twos = _mm@vsize@_set1_epi32(2); + @vtype@ two_over_pi = _mm@vsize@_set1_ps(NPY_TWO_O_PIf); + @vtype@ codyw_c1 = _mm@vsize@_set1_ps(NPY_CODY_WAITE_PI_O_2_HIGHf); + @vtype@ codyw_c2 = _mm@vsize@_set1_ps(NPY_CODY_WAITE_PI_O_2_MEDf); + @vtype@ codyw_c3 = _mm@vsize@_set1_ps(NPY_CODY_WAITE_PI_O_2_LOWf); + @vtype@ cos_invf0 = _mm@vsize@_set1_ps(NPY_COEFF_INVF0_COSINEf); + @vtype@ cos_invf2 = _mm@vsize@_set1_ps(NPY_COEFF_INVF2_COSINEf); + @vtype@ cos_invf4 = _mm@vsize@_set1_ps(NPY_COEFF_INVF4_COSINEf); + @vtype@ cos_invf6 = _mm@vsize@_set1_ps(NPY_COEFF_INVF6_COSINEf); + @vtype@ cos_invf8 = _mm@vsize@_set1_ps(NPY_COEFF_INVF8_COSINEf); + @vtype@ sin_invf3 = _mm@vsize@_set1_ps(NPY_COEFF_INVF3_SINEf); + @vtype@ sin_invf5 = _mm@vsize@_set1_ps(NPY_COEFF_INVF5_SINEf); + @vtype@ sin_invf7 = _mm@vsize@_set1_ps(NPY_COEFF_INVF7_SINEf); + @vtype@ sin_invf9 = _mm@vsize@_set1_ps(NPY_COEFF_INVF9_SINEf); + @vtype@ cvt_magic = _mm@vsize@_set1_ps(NPY_RINT_CVT_MAGICf); + @vtype@ zero_f = _mm@vsize@_set1_ps(0.0f); + @vtype@ quadrant, reduced_x, reduced_x2, cos, sin; + @vtype@i iquadrant; + @mask@ nan_mask, glibc_mask, sine_mask, negate_mask; + @mask@ load_mask = @isa@_get_full_load_mask_ps(); + npy_intp num_remaining_elements = array_size; + npy_int indexarr[16]; + for (npy_int ii = 0; ii < 16; ii++) { + indexarr[ii] = ii*stride; + } + @vtype@i vindex = _mm@vsize@_loadu_si@vsize@((@vtype@i*)&indexarr[0]); + + while (num_remaining_elements > 0) { + + if (num_remaining_elements < num_lanes) { + load_mask = @isa@_get_partial_load_mask_ps(num_remaining_elements, + num_lanes); + } + + @vtype@ x; + if (stride == 1) { + x = @isa@_masked_load_ps(load_mask, ip); + } + else { + x = @isa@_masked_gather_ps(zero_f, ip, vindex, load_mask); + } + + /* + * For elements outside of this range, Cody-Waite's range reduction + * becomes inaccurate and we will call glibc to compute cosine for + * these numbers + */ + + glibc_mask = @isa@_in_range_mask(x, large_number,-large_number); + glibc_mask = @and_masks@(load_mask, glibc_mask); + nan_mask = _mm@vsize@_cmp_ps@vsub@(x, x, _CMP_NEQ_UQ); + x = @isa@_set_masked_lanes_ps(x, zero_f, @or_masks@(nan_mask, glibc_mask)); + npy_int iglibc_mask = @mask_to_int@(glibc_mask); + + if (iglibc_mask != @full_mask@) { + quadrant = _mm@vsize@_mul_ps(x, two_over_pi); + + /* round to nearest */ + quadrant = _mm@vsize@_add_ps(quadrant, cvt_magic); + quadrant = _mm@vsize@_sub_ps(quadrant, cvt_magic); + + /* Cody-Waite's range reduction algorithm */ + reduced_x = @isa@_range_reduction(x, quadrant, + codyw_c1, codyw_c2, codyw_c3); + reduced_x2 = _mm@vsize@_mul_ps(reduced_x, reduced_x); + + /* compute cosine and sine */ + cos = @isa@_cosine(reduced_x2, cos_invf8, cos_invf6, cos_invf4, + cos_invf2, cos_invf0); + sin = @isa@_sine(reduced_x, reduced_x2, sin_invf9, sin_invf7, + sin_invf5, sin_invf3, zero_f); + + iquadrant = _mm@vsize@_cvtps_epi32(quadrant); + if (my_trig_op == npy_compute_cos) { + iquadrant = _mm@vsize@_add_epi32(iquadrant, ones); + } + + /* blend sin and cos based on the quadrant */ + sine_mask = @isa@_should_calculate_sine(iquadrant, ones, zeros); + cos = @isa@_blend(cos, sin, sine_mask); + + /* multiply by -1 for appropriate elements */ + negate_mask = @isa@_should_negate(iquadrant, twos, twos); + cos = @isa@_blend(cos, _mm@vsize@_sub_ps(zero_f, cos), negate_mask); + cos = @isa@_set_masked_lanes_ps(cos, _mm@vsize@_set1_ps(NPY_NANF), nan_mask); + + @masked_store@(op, @cvtps_epi32@(load_mask), cos); + } + + /* process elements using glibc for large elements */ + if (my_trig_op == npy_compute_cos) { + for (int ii = 0; iglibc_mask != 0; ii++) { + if (iglibc_mask & 0x01) { + op[ii] = npy_cosf(ip[ii]); + } + iglibc_mask = iglibc_mask >> 1; + } + } + else { + for (int ii = 0; iglibc_mask != 0; ii++) { + if (iglibc_mask & 0x01) { + op[ii] = npy_sinf(ip[ii]); + } + iglibc_mask = iglibc_mask >> 1; + } + } + ip += num_lanes*stride; + op += num_lanes; + num_remaining_elements -= num_lanes; + } +} /* * Vectorized implementation of exp using AVX2 and AVX512: @@ -1381,7 +2011,6 @@ static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ * same x = 0xc2781e37) */ -#if defined HAVE_ATTRIBUTE_TARGET_@ISA@_WITH_INTRINSICS static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void @ISA@_exp_FLOAT(npy_float * op, npy_float * ip, @@ -1417,27 +2046,27 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void @vtype@i vindex = _mm@vsize@_loadu_si@vsize@((@vtype@i*)&indexarr[0]); @mask@ xmax_mask, xmin_mask, nan_mask, inf_mask; - @mask@ overflow_mask = @isa@_get_partial_load_mask(0, num_lanes); - @mask@ load_mask = @isa@_get_full_load_mask(); + @mask@ overflow_mask = @isa@_get_partial_load_mask_ps(0, num_lanes); + @mask@ load_mask = @isa@_get_full_load_mask_ps(); npy_intp num_remaining_elements = array_size; while (num_remaining_elements > 0) { if (num_remaining_elements < num_lanes) { - load_mask = @isa@_get_partial_load_mask(num_remaining_elements, - num_lanes); + load_mask = @isa@_get_partial_load_mask_ps(num_remaining_elements, + num_lanes); } @vtype@ x; if (stride == 1) { - x = @isa@_masked_load(load_mask, ip); + x = @isa@_masked_load_ps(load_mask, ip); } else { - x = @isa@_masked_gather(zeros_f, ip, vindex, load_mask); + x = @isa@_masked_gather_ps(zeros_f, ip, vindex, load_mask); } nan_mask = _mm@vsize@_cmp_ps@vsub@(x, x, _CMP_NEQ_UQ); - x = @isa@_set_masked_lanes(x, zeros_f, nan_mask); + x = @isa@_set_masked_lanes_ps(x, zeros_f, nan_mask); xmax_mask = _mm@vsize@_cmp_ps@vsub@(x, _mm@vsize@_set1_ps(xmax), _CMP_GE_OQ); xmin_mask = _mm@vsize@_cmp_ps@vsub@(x, _mm@vsize@_set1_ps(xmin), _CMP_LE_OQ); @@ -1445,7 +2074,7 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void overflow_mask = @or_masks@(overflow_mask, @xor_masks@(xmax_mask, inf_mask)); - x = @isa@_set_masked_lanes(x, zeros_f, @or_masks@( + x = @isa@_set_masked_lanes_ps(x, zeros_f, @or_masks@( @or_masks@(nan_mask, xmin_mask), xmax_mask)); quadrant = _mm@vsize@_mul_ps(x, log2e); @@ -1478,9 +2107,9 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void * elem < xmin; return 0.0f * elem = +/- nan, return nan */ - poly = @isa@_set_masked_lanes(poly, _mm@vsize@_set1_ps(NPY_NANF), nan_mask); - poly = @isa@_set_masked_lanes(poly, inf, xmax_mask); - poly = @isa@_set_masked_lanes(poly, zeros_f, xmin_mask); + poly = @isa@_set_masked_lanes_ps(poly, _mm@vsize@_set1_ps(NPY_NANF), nan_mask); + poly = @isa@_set_masked_lanes_ps(poly, inf, xmax_mask); + poly = @isa@_set_masked_lanes_ps(poly, zeros_f, xmin_mask); @masked_store@(op, @cvtps_epi32@(load_mask), poly); @@ -1545,24 +2174,24 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void @vtype@ poly, num_poly, denom_poly, exponent; @mask@ inf_mask, nan_mask, sqrt2_mask, zero_mask, negx_mask; - @mask@ invalid_mask = @isa@_get_partial_load_mask(0, num_lanes); + @mask@ invalid_mask = @isa@_get_partial_load_mask_ps(0, num_lanes); @mask@ divide_by_zero_mask = invalid_mask; - @mask@ load_mask = @isa@_get_full_load_mask(); + @mask@ load_mask = @isa@_get_full_load_mask_ps(); npy_intp num_remaining_elements = array_size; while (num_remaining_elements > 0) { if (num_remaining_elements < num_lanes) { - load_mask = @isa@_get_partial_load_mask(num_remaining_elements, - num_lanes); + load_mask = @isa@_get_partial_load_mask_ps(num_remaining_elements, + num_lanes); } @vtype@ x_in; if (stride == 1) { - x_in = @isa@_masked_load(load_mask, ip); + x_in = @isa@_masked_load_ps(load_mask, ip); } else { - x_in = @isa@_masked_gather(zeros_f, ip, vindex, load_mask); + x_in = @isa@_masked_gather_ps(zeros_f, ip, vindex, load_mask); } negx_mask = _mm@vsize@_cmp_ps@vsub@(x_in, zeros_f, _CMP_LT_OQ); @@ -1573,7 +2202,7 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void @and_masks@(zero_mask, load_mask)); invalid_mask = @or_masks@(invalid_mask, negx_mask); - @vtype@ x = @isa@_set_masked_lanes(x_in, zeros_f, negx_mask); + @vtype@ x = @isa@_set_masked_lanes_ps(x_in, zeros_f, negx_mask); /* set x = normalized mantissa */ exponent = @isa@_get_exponent(x); @@ -1607,10 +2236,10 @@ static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void * x = +/- NAN; return NAN * x = 0.0f; return -INF */ - poly = @isa@_set_masked_lanes(poly, nan, nan_mask); - poly = @isa@_set_masked_lanes(poly, neg_nan, negx_mask); - poly = @isa@_set_masked_lanes(poly, neg_inf, zero_mask); - poly = @isa@_set_masked_lanes(poly, inf, inf_mask); + poly = @isa@_set_masked_lanes_ps(poly, nan, nan_mask); + poly = @isa@_set_masked_lanes_ps(poly, neg_nan, negx_mask); + poly = @isa@_set_masked_lanes_ps(poly, neg_inf, zero_mask); + poly = @isa@_set_masked_lanes_ps(poly, inf, inf_mask); @masked_store@(op, @cvtps_epi32@(load_mask), poly); diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 174703fb1..1dc581977 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -1193,34 +1193,11 @@ get_ufunc_arguments(PyUFuncObject *ufunc, } } else { - /* - * If the deprecated behavior is ever removed, - * keep only the else branch of this if-else - */ - if (PyArray_Check(out_kwd) || out_kwd == Py_None) { - if (DEPRECATE("passing a single array to the " - "'out' keyword argument of a " - "ufunc with\n" - "more than one output will " - "result in an error in the " - "future") < 0) { - /* The future error message */ - PyErr_SetString(PyExc_TypeError, - "'out' must be a tuple of arrays"); - goto fail; - } - if (_set_out_array(out_kwd, out_op+nin) < 0) { - goto fail; - } - } - else { - PyErr_SetString(PyExc_TypeError, - nout > 1 ? "'out' must be a tuple " - "of arrays" : - "'out' must be an array or a " - "tuple of a single array"); - goto fail; - } + PyErr_SetString(PyExc_TypeError, + nout > 1 ? "'out' must be a tuple of arrays" : + "'out' must be an array or a tuple with " + "a single array"); + goto fail; } } /* @@ -2297,7 +2274,7 @@ _parse_axes_arg(PyUFuncObject *ufunc, int op_core_num_dims[], PyObject *axes, * Returns 0 on success, and -1 on failure */ static int -_parse_axis_arg(PyUFuncObject *ufunc, int core_num_dims[], PyObject *axis, +_parse_axis_arg(PyUFuncObject *ufunc, const int core_num_dims[], PyObject *axis, PyArrayObject **op, int broadcast_ndim, int **remap_axis) { int nop = ufunc->nargs; int iop, axis_int; @@ -2368,7 +2345,7 @@ _parse_axis_arg(PyUFuncObject *ufunc, int core_num_dims[], PyObject *axis, */ static int _get_coredim_sizes(PyUFuncObject *ufunc, PyArrayObject **op, - int *op_core_num_dims, npy_uint32 *core_dim_flags, + const int *op_core_num_dims, npy_uint32 *core_dim_flags, npy_intp *core_dim_sizes, int **remap_axis) { int i; int nin = ufunc->nin; @@ -4081,8 +4058,8 @@ PyUFunc_Reduceat(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *ind, for (i = 0; i < ind_size; ++i) { if (reduceat_ind[i] < 0 || reduceat_ind[i] >= red_axis_size) { PyErr_Format(PyExc_IndexError, - "index %d out-of-bounds in %s.%s [0, %d)", - (int)reduceat_ind[i], ufunc_name, opname, (int)red_axis_size); + "index %" NPY_INTP_FMT " out-of-bounds in %s.%s [0, %" NPY_INTP_FMT ")", + reduceat_ind[i], ufunc_name, opname, red_axis_size); return NULL; } } @@ -4766,6 +4743,9 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) wrapped = _apply_array_wrap(wraparr[i], mps[j], &context); mps[j] = NULL; /* Prevent fail double-freeing this */ if (wrapped == NULL) { + for (j = 0; j < i; j++) { + Py_DECREF(retobj[j]); + } goto fail; } @@ -4863,7 +4843,7 @@ ufunc_seterr(PyObject *NPY_UNUSED(dummy), PyObject *args) NPY_NO_EXPORT int PyUFunc_ReplaceLoopBySignature(PyUFuncObject *func, PyUFuncGenericFunction newfunc, - int *signature, + const int *signature, PyUFuncGenericFunction *oldfunc) { int i, j; @@ -4918,7 +4898,7 @@ PyUFunc_FromFuncAndDataAndSignatureAndIdentity(PyUFuncGenericFunction *func, voi char *types, int ntypes, int nin, int nout, int identity, const char *name, const char *doc, - int unused, const char *signature, + const int unused, const char *signature, PyObject *identity_value) { PyUFuncObject *ufunc; @@ -5220,7 +5200,7 @@ NPY_NO_EXPORT int PyUFunc_RegisterLoopForType(PyUFuncObject *ufunc, int usertype, PyUFuncGenericFunction function, - int *arg_types, + const int *arg_types, void *data) { PyArray_Descr *descr; @@ -5693,18 +5673,13 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) * Create dtypes array for either one or two input operands. * The output operand is set to the first input operand */ - dtypes[0] = PyArray_DESCR(op1_array); operands[0] = op1_array; if (op2_array != NULL) { - dtypes[1] = PyArray_DESCR(op2_array); - dtypes[2] = dtypes[0]; operands[1] = op2_array; operands[2] = op1_array; nop = 3; } else { - dtypes[1] = dtypes[0]; - dtypes[2] = NULL; operands[1] = op1_array; operands[2] = NULL; nop = 2; @@ -5861,9 +5836,10 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) Py_XDECREF(op2_array); Py_XDECREF(iter); Py_XDECREF(iter2); - Py_XDECREF(array_operands[0]); - Py_XDECREF(array_operands[1]); - Py_XDECREF(array_operands[2]); + for (i = 0; i < 3; i++) { + Py_XDECREF(dtypes[i]); + Py_XDECREF(array_operands[i]); + } if (needs_api && PyErr_Occurred()) { return NULL; @@ -5880,9 +5856,10 @@ fail: Py_XDECREF(op2_array); Py_XDECREF(iter); Py_XDECREF(iter2); - Py_XDECREF(array_operands[0]); - Py_XDECREF(array_operands[1]); - Py_XDECREF(array_operands[2]); + for (i = 0; i < 3; i++) { + Py_XDECREF(dtypes[i]); + Py_XDECREF(array_operands[i]); + } return NULL; } diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index 25dd002ac..f93d8229e 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -548,6 +548,7 @@ PyUFunc_SimpleUniformOperationTypeResolver( } out_dtypes[0] = ensure_dtype_nbo(dtype); + Py_DECREF(dtype); if (out_dtypes[0] == NULL) { return -1; } @@ -882,7 +883,7 @@ PyUFunc_SubtractionTypeResolver(PyUFuncObject *ufunc, /* The type resolver would have upcast already */ if (out_dtypes[0]->type_num == NPY_BOOL) { PyErr_Format(PyExc_TypeError, - "numpy boolean subtract, the `-` operator, is deprecated, " + "numpy boolean subtract, the `-` operator, is not supported, " "use the bitwise_xor, the `^` operator, or the logical_xor " "function instead."); return -1; @@ -1957,7 +1958,8 @@ linear_search_type_resolver(PyUFuncObject *self, npy_intp i, j, nin = self->nin, nop = nin + self->nout; int types[NPY_MAXARGS]; const char *ufunc_name; - int no_castable_output, use_min_scalar; + int no_castable_output = 0; + int use_min_scalar; /* For making a better error message on coercion error */ char err_dst_typecode = '-', err_src_typecode = '-'; @@ -2264,7 +2266,6 @@ PyUFunc_DivmodTypeResolver(PyUFuncObject *ufunc, out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); out_dtypes[2] = PyArray_DescrFromType(NPY_LONGLONG); - Py_INCREF(out_dtypes[2]); out_dtypes[3] = out_dtypes[0]; Py_INCREF(out_dtypes[3]); } diff --git a/numpy/core/tests/test__exceptions.py b/numpy/core/tests/test__exceptions.py new file mode 100644 index 000000000..494b51f34 --- /dev/null +++ b/numpy/core/tests/test__exceptions.py @@ -0,0 +1,42 @@ +""" +Tests of the ._exceptions module. Primarily for exercising the __str__ methods. +""" +import numpy as np + +_ArrayMemoryError = np.core._exceptions._ArrayMemoryError + +class TestArrayMemoryError: + def test_str(self): + e = _ArrayMemoryError((1023,), np.dtype(np.uint8)) + str(e) # not crashing is enough + + # testing these properties is easier than testing the full string repr + def test__size_to_string(self): + """ Test e._size_to_string """ + f = _ArrayMemoryError._size_to_string + Ki = 1024 + assert f(0) == '0 bytes' + assert f(1) == '1 bytes' + assert f(1023) == '1023 bytes' + assert f(Ki) == '1.00 KiB' + assert f(Ki+1) == '1.00 KiB' + assert f(10*Ki) == '10.0 KiB' + assert f(int(999.4*Ki)) == '999. KiB' + assert f(int(1023.4*Ki)) == '1023. KiB' + assert f(int(1023.5*Ki)) == '1.00 MiB' + assert f(Ki*Ki) == '1.00 MiB' + + # 1023.9999 Mib should round to 1 GiB + assert f(int(Ki*Ki*Ki*0.9999)) == '1.00 GiB' + assert f(Ki*Ki*Ki*Ki*Ki*Ki) == '1.00 EiB' + # larger than sys.maxsize, adding larger prefices isn't going to help + # anyway. + assert f(Ki*Ki*Ki*Ki*Ki*Ki*123456) == '123456. EiB' + + def test__total_size(self): + """ Test e._total_size """ + e = _ArrayMemoryError((1,), np.dtype(np.uint8)) + assert e._total_size == 1 + + e = _ArrayMemoryError((2, 4), np.dtype((np.uint64, 16))) + assert e._total_size == 1024 diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py index 32e2ea537..89fc2b0b9 100644 --- a/numpy/core/tests/test_api.py +++ b/numpy/core/tests/test_api.py @@ -296,7 +296,7 @@ def test_array_astype(): ) def test_array_astype_warning(t): # test ComplexWarning when casting from complex to float or int - a = np.array(10, dtype=np.complex) + a = np.array(10, dtype=np.complex_) assert_warns(np.ComplexWarning, a.astype, t) def test_copyto_fromscalar(): diff --git a/numpy/core/tests/test_arrayprint.py b/numpy/core/tests/test_arrayprint.py index f2b8fdca7..702e68e76 100644 --- a/numpy/core/tests/test_arrayprint.py +++ b/numpy/core/tests/test_arrayprint.py @@ -262,11 +262,6 @@ class TestArray2String(object): assert_(np.array2string(s, formatter={'numpystr':lambda s: s*2}) == '[abcabc defdef]') - # check for backcompat that using FloatFormat works and emits warning - with assert_warns(DeprecationWarning): - fmt = np.core.arrayprint.FloatFormat(x, 9, 'maxprec', False) - assert_equal(np.array2string(x, formatter={'float_kind': fmt}), - '[0. 1. 2.]') def test_structure_format(self): dt = np.dtype([('name', np.str_, 16), ('grades', np.float64, (2,))]) @@ -847,9 +842,9 @@ class TestPrintOptions(object): ) def test_bad_args(self): - assert_raises(ValueError, np.set_printoptions, threshold='nan') - assert_raises(ValueError, np.set_printoptions, threshold=u'1') - assert_raises(ValueError, np.set_printoptions, threshold=b'1') + assert_raises(ValueError, np.set_printoptions, threshold=float('nan')) + assert_raises(TypeError, np.set_printoptions, threshold='1') + assert_raises(TypeError, np.set_printoptions, threshold=b'1') def test_unicode_object_array(): import sys diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index f99c0f72b..d38444ef7 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -75,6 +75,15 @@ class TestDateTime(object): # Can cast safely/same_kind from integer to timedelta assert_(np.can_cast('i8', 'm8', casting='same_kind')) assert_(np.can_cast('i8', 'm8', casting='safe')) + assert_(np.can_cast('i4', 'm8', casting='same_kind')) + assert_(np.can_cast('i4', 'm8', casting='safe')) + assert_(np.can_cast('u4', 'm8', casting='same_kind')) + assert_(np.can_cast('u4', 'm8', casting='safe')) + + # Cannot cast safely from unsigned integer of the same size, which + # could overflow + assert_(np.can_cast('u8', 'm8', casting='same_kind')) + assert_(not np.can_cast('u8', 'm8', casting='safe')) # Cannot cast safely/same_kind from float to timedelta assert_(not np.can_cast('f4', 'm8', casting='same_kind')) @@ -136,6 +145,50 @@ class TestDateTime(object): assert_(np.datetime64('NaT') != np.datetime64('NaT', 'us')) assert_(np.datetime64('NaT', 'us') != np.datetime64('NaT')) + @pytest.mark.parametrize("size", [ + 3, 21, 217, 1000]) + def test_datetime_nat_argsort_stability(self, size): + # NaT < NaT should be False internally for + # sort stability + expected = np.arange(size) + arr = np.tile(np.datetime64('NaT'), size) + assert_equal(np.argsort(arr, kind='mergesort'), expected) + + @pytest.mark.parametrize("size", [ + 3, 21, 217, 1000]) + def test_timedelta_nat_argsort_stability(self, size): + # NaT < NaT should be False internally for + # sort stability + expected = np.arange(size) + arr = np.tile(np.timedelta64('NaT'), size) + assert_equal(np.argsort(arr, kind='mergesort'), expected) + + @pytest.mark.parametrize("arr, expected", [ + # the example provided in gh-12629 + (['NaT', 1, 2, 3], + [1, 2, 3, 'NaT']), + # multiple NaTs + (['NaT', 9, 'NaT', -707], + [-707, 9, 'NaT', 'NaT']), + # this sort explores another code path for NaT + ([1, -2, 3, 'NaT'], + [-2, 1, 3, 'NaT']), + # 2-D array + ([[51, -220, 'NaT'], + [-17, 'NaT', -90]], + [[-220, 51, 'NaT'], + [-90, -17, 'NaT']]), + ]) + @pytest.mark.parametrize("dtype", [ + 'M8[ns]', 'M8[us]', + 'm8[ns]', 'm8[us]']) + def test_datetime_timedelta_sort_nat(self, arr, expected, dtype): + # fix for gh-12629 and gh-15063; NaT sorting to end of array + arr = np.array(arr, dtype=dtype) + expected = np.array(expected, dtype=dtype) + arr.sort() + assert_equal(arr, expected) + def test_datetime_scalar_construction(self): # Construct with different units assert_equal(np.datetime64('1950-03-12', 'D'), @@ -483,6 +536,30 @@ class TestDateTime(object): assert_equal(np.datetime64(a, '[Y]'), np.datetime64('NaT', '[Y]')) assert_equal(np.datetime64(a, '[W]'), np.datetime64('NaT', '[W]')) + # NaN -> NaT + nan = np.array([np.nan] * 8) + fnan = nan.astype('f') + lnan = nan.astype('g') + cnan = nan.astype('D') + cfnan = nan.astype('F') + clnan = nan.astype('G') + + nat = np.array([np.datetime64('NaT')] * 8) + assert_equal(nan.astype('M8[ns]'), nat) + assert_equal(fnan.astype('M8[ns]'), nat) + assert_equal(lnan.astype('M8[ns]'), nat) + assert_equal(cnan.astype('M8[ns]'), nat) + assert_equal(cfnan.astype('M8[ns]'), nat) + assert_equal(clnan.astype('M8[ns]'), nat) + + nat = np.array([np.timedelta64('NaT')] * 8) + assert_equal(nan.astype('timedelta64[ns]'), nat) + assert_equal(fnan.astype('timedelta64[ns]'), nat) + assert_equal(lnan.astype('timedelta64[ns]'), nat) + assert_equal(cnan.astype('timedelta64[ns]'), nat) + assert_equal(cfnan.astype('timedelta64[ns]'), nat) + assert_equal(clnan.astype('timedelta64[ns]'), nat) + def test_days_creation(self): assert_equal(np.array('1599', dtype='M8[D]').astype('i8'), (1600-1970)*365 - (1972-1600)/4 + 3 - 365) @@ -1333,10 +1410,14 @@ class TestDateTime(object): # Interaction with NaT a = np.array('1999-03-12T13', dtype='M8[2m]') dtnat = np.array('NaT', dtype='M8[h]') - assert_equal(np.minimum(a, dtnat), a) - assert_equal(np.minimum(dtnat, a), a) - assert_equal(np.maximum(a, dtnat), a) - assert_equal(np.maximum(dtnat, a), a) + assert_equal(np.minimum(a, dtnat), dtnat) + assert_equal(np.minimum(dtnat, a), dtnat) + assert_equal(np.maximum(a, dtnat), dtnat) + assert_equal(np.maximum(dtnat, a), dtnat) + assert_equal(np.fmin(dtnat, a), a) + assert_equal(np.fmin(a, dtnat), a) + assert_equal(np.fmax(dtnat, a), a) + assert_equal(np.fmax(a, dtnat), a) # Also do timedelta a = np.array(3, dtype='m8[h]') @@ -1831,7 +1912,7 @@ class TestDateTime(object): def test_timedelta_arange_no_dtype(self): d = np.array(5, dtype="m8[D]") assert_equal(np.arange(d, d + 1), d) - assert_raises(ValueError, np.arange, d) + assert_equal(np.arange(d), np.arange(0, d)) def test_datetime_maximum_reduce(self): a = np.array(['2010-01-02', '1999-03-14', '1833-03'], dtype='M8[D]') @@ -2208,7 +2289,7 @@ class TestDateTime(object): continue assert_raises(TypeError, np.isnat, np.zeros(10, t)) - def test_isfinite(self): + def test_isfinite_scalar(self): assert_(not np.isfinite(np.datetime64('NaT', 'ms'))) assert_(not np.isfinite(np.datetime64('NaT', 'ns'))) assert_(np.isfinite(np.datetime64('2038-01-19T03:14:07'))) @@ -2216,18 +2297,25 @@ class TestDateTime(object): assert_(not np.isfinite(np.timedelta64('NaT', "ms"))) assert_(np.isfinite(np.timedelta64(34, "ms"))) - res = np.array([True, True, False]) - for unit in ['Y', 'M', 'W', 'D', - 'h', 'm', 's', 'ms', 'us', - 'ns', 'ps', 'fs', 'as']: - arr = np.array([123, -321, "NaT"], dtype='<datetime64[%s]' % unit) - assert_equal(np.isfinite(arr), res) - arr = np.array([123, -321, "NaT"], dtype='>datetime64[%s]' % unit) - assert_equal(np.isfinite(arr), res) - arr = np.array([123, -321, "NaT"], dtype='<timedelta64[%s]' % unit) - assert_equal(np.isfinite(arr), res) - arr = np.array([123, -321, "NaT"], dtype='>timedelta64[%s]' % unit) - assert_equal(np.isfinite(arr), res) + @pytest.mark.parametrize('unit', ['Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', + 'us', 'ns', 'ps', 'fs', 'as']) + @pytest.mark.parametrize('dstr', ['<datetime64[%s]', '>datetime64[%s]', + '<timedelta64[%s]', '>timedelta64[%s]']) + def test_isfinite_isinf_isnan_units(self, unit, dstr): + '''check isfinite, isinf, isnan for all units of <M, >M, <m, >m dtypes + ''' + arr_val = [123, -321, "NaT"] + arr = np.array(arr_val, dtype= dstr % unit) + pos = np.array([True, True, False]) + neg = np.array([False, False, True]) + false = np.array([False, False, False]) + assert_equal(np.isfinite(arr), pos) + assert_equal(np.isinf(arr), false) + assert_equal(np.isnan(arr), neg) + + def test_assert_equal(self): + assert_raises(AssertionError, assert_equal, + np.datetime64('nat'), np.timedelta64('nat')) def test_corecursive_input(self): # construct a co-recursive list diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 58ebea024..363ff26db 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -10,12 +10,16 @@ import sys import operator import warnings import pytest +import shutil +import tempfile import numpy as np from numpy.testing import ( - assert_raises, assert_warns, assert_ + assert_raises, assert_warns, assert_, assert_array_equal ) +from numpy.core._multiarray_tests import fromstring_null_term_c_api + try: import pytz _has_pytz = True @@ -149,16 +153,6 @@ class TestNonTupleNDIndexDeprecation(object): a[[0, 1]] -class TestRankDeprecation(_DeprecationTestCase): - """Test that np.rank is deprecated. The function should simply be - removed. The VisibleDeprecationWarning may become unnecessary. - """ - - def test(self): - a = np.arange(10) - assert_warns(np.VisibleDeprecationWarning, np.rank, a) - - class TestComparisonDeprecations(_DeprecationTestCase): """This tests the deprecation, for non-element-wise comparison logic. This used to mean that when an error occurred during element-wise comparison @@ -178,7 +172,7 @@ class TestComparisonDeprecations(_DeprecationTestCase): # (warning is issued a couple of times here) self.assert_deprecated(op, args=(a, a[:-1]), num=None) - # Element comparison error (numpy array can't be compared). + # ragged array comparison returns True/False a = np.array([1, np.array([1,2,3])], dtype=object) b = np.array([1, np.array([1,2,3])], dtype=object) self.assert_deprecated(op, args=(a, b), num=None) @@ -281,36 +275,6 @@ class TestNonCContiguousViewDeprecation(_DeprecationTestCase): self.assert_deprecated(np.ones((2,2)).T.view, args=(np.int8,)) -class TestInvalidOrderParameterInputForFlattenArrayDeprecation(_DeprecationTestCase): - """Invalid arguments to the ORDER parameter in array.flatten() should not be - allowed and should raise an error. However, in the interests of not breaking - code that may inadvertently pass invalid arguments to this parameter, a - DeprecationWarning will be issued instead for the time being to give developers - time to refactor relevant code. - """ - - def test_flatten_array_non_string_arg(self): - x = np.zeros((3, 5)) - self.message = ("Non-string object detected for " - "the array ordering. Please pass " - "in 'C', 'F', 'A', or 'K' instead") - self.assert_deprecated(x.flatten, args=(np.pi,)) - - def test_flatten_array_invalid_string_arg(self): - # Tests that a DeprecationWarning is raised - # when a string of length greater than one - # starting with "C", "F", "A", or "K" (case- - # and unicode-insensitive) is passed in for - # the ORDER parameter. Otherwise, a TypeError - # will be raised! - - x = np.zeros((3, 5)) - self.message = ("Non length-one string passed " - "in for the array ordering. Please " - "pass in 'C', 'F', 'A', or 'K' instead") - self.assert_deprecated(x.flatten, args=("FACK",)) - - class TestArrayDataAttributeAssignmentDeprecation(_DeprecationTestCase): """Assigning the 'data' attribute of an ndarray is unsafe as pointed out in gh-7093. Eventually, such assignment should NOT be allowed, but @@ -329,22 +293,6 @@ class TestArrayDataAttributeAssignmentDeprecation(_DeprecationTestCase): self.assert_deprecated(a.__setattr__, args=('data', b.data)) -class TestLinspaceInvalidNumParameter(_DeprecationTestCase): - """Argument to the num parameter in linspace that cannot be - safely interpreted as an integer is deprecated in 1.12.0. - - Argument to the num parameter in linspace that cannot be - safely interpreted as an integer should not be allowed. - In the interest of not breaking code that passes - an argument that could still be interpreted as an integer, a - DeprecationWarning will be issued for the time being to give - developers time to refactor relevant code. - """ - def test_float_arg(self): - # 2016-02-25, PR#7328 - self.assert_deprecated(np.linspace, args=(0, 10, 2.5)) - - class TestBinaryReprInsufficientWidthParameterForRepresentation(_DeprecationTestCase): """ If a 'width' parameter is passed into ``binary_repr`` that is insufficient to @@ -452,6 +400,18 @@ class TestNPY_CHAR(_DeprecationTestCase): assert_(npy_char_deprecation() == 'S1') +class TestPyArray_AS1D(_DeprecationTestCase): + def test_npy_pyarrayas1d_deprecation(self): + from numpy.core._multiarray_tests import npy_pyarrayas1d_deprecation + assert_raises(NotImplementedError, npy_pyarrayas1d_deprecation) + + +class TestPyArray_AS2D(_DeprecationTestCase): + def test_npy_pyarrayas2d_deprecation(self): + from numpy.core._multiarray_tests import npy_pyarrayas2d_deprecation + assert_raises(NotImplementedError, npy_pyarrayas2d_deprecation) + + class Test_UPDATEIFCOPY(_DeprecationTestCase): """ v1.14 deprecates creating an array with the UPDATEIFCOPY flag, use @@ -499,6 +459,12 @@ class TestBincount(_DeprecationTestCase): self.assert_deprecated(lambda: np.bincount([1, 2, 3], minlength=None)) +class TestAlen(_DeprecationTestCase): + # 2019-08-02, 1.18.0 + def test_alen(self): + self.assert_deprecated(lambda: np.alen(np.array([1, 2, 3]))) + + class TestGeneratorSum(_DeprecationTestCase): # 2018-02-25, 1.15.0 def test_generator_sum(self): @@ -518,17 +484,71 @@ class TestPositiveOnNonNumerical(_DeprecationTestCase): def test_positive_on_non_number(self): self.assert_deprecated(operator.pos, args=(np.array('foo'),)) + class TestFromstring(_DeprecationTestCase): # 2017-10-19, 1.14 def test_fromstring(self): self.assert_deprecated(np.fromstring, args=('\x00'*80,)) + +class TestFromStringAndFileInvalidData(_DeprecationTestCase): + # 2019-06-08, 1.17.0 + # Tests should be moved to real tests when deprecation is done. + message = "string or file could not be read to its end" + + @pytest.mark.parametrize("invalid_str", [",invalid_data", "invalid_sep"]) + def test_deprecate_unparsable_data_file(self, invalid_str): + x = np.array([1.51, 2, 3.51, 4], dtype=float) + + with tempfile.TemporaryFile(mode="w") as f: + x.tofile(f, sep=',', format='%.2f') + f.write(invalid_str) + + f.seek(0) + self.assert_deprecated(lambda: np.fromfile(f, sep=",")) + f.seek(0) + self.assert_deprecated(lambda: np.fromfile(f, sep=",", count=5)) + # Should not raise: + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + f.seek(0) + res = np.fromfile(f, sep=",", count=4) + assert_array_equal(res, x) + + @pytest.mark.parametrize("invalid_str", [",invalid_data", "invalid_sep"]) + def test_deprecate_unparsable_string(self, invalid_str): + x = np.array([1.51, 2, 3.51, 4], dtype=float) + x_str = "1.51,2,3.51,4{}".format(invalid_str) + + self.assert_deprecated(lambda: np.fromstring(x_str, sep=",")) + self.assert_deprecated(lambda: np.fromstring(x_str, sep=",", count=5)) + + # The C-level API can use not fixed size, but 0 terminated strings, + # so test that as well: + bytestr = x_str.encode("ascii") + self.assert_deprecated(lambda: fromstring_null_term_c_api(bytestr)) + + with assert_warns(DeprecationWarning): + # this is slightly strange, in that fromstring leaves data + # potentially uninitialized (would be good to error when all is + # read, but count is larger then actual data maybe). + res = np.fromstring(x_str, sep=",", count=5) + assert_array_equal(res[:-1], x) + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + + # Should not raise: + res = np.fromstring(x_str, sep=",", count=4) + assert_array_equal(res, x) + + class Test_GetSet_NumericOps(_DeprecationTestCase): # 2018-09-20, 1.16.0 def test_get_numeric_ops(self): from numpy.core._multiarray_tests import getset_numericops self.assert_deprecated(getset_numericops, num=2) - + # empty kwargs prevents any state actually changing which would break # other tests. self.assert_deprecated(np.set_numeric_ops, kwargs={}) diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index f60eab696..e18e66c64 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -25,7 +25,7 @@ def assert_dtype_not_equal(a, b): class TestBuiltin(object): @pytest.mark.parametrize('t', [int, float, complex, np.int32, str, object, - np.unicode]) + np.compat.unicode]) def test_run(self, t): """Only test hash runs at all.""" dt = np.dtype(t) @@ -419,6 +419,31 @@ class TestRecord(object): assert_raises(ValueError, np.dtype, {'formats': ['i4', 'i4'], 'f0': ('i4', 0), 'f1':('i4', 4)}) + def test_fieldless_views(self): + a = np.zeros(2, dtype={'names':[], 'formats':[], 'offsets':[], + 'itemsize':8}) + assert_raises(ValueError, a.view, np.dtype([])) + + d = np.dtype((np.dtype([]), 10)) + assert_equal(d.shape, (10,)) + assert_equal(d.itemsize, 0) + assert_equal(d.base, np.dtype([])) + + arr = np.fromiter((() for i in range(10)), []) + assert_equal(arr.dtype, np.dtype([])) + assert_raises(ValueError, np.frombuffer, b'', dtype=[]) + assert_equal(np.frombuffer(b'', dtype=[], count=2), + np.empty(2, dtype=[])) + + assert_raises(ValueError, np.dtype, ([], 'f8')) + assert_raises(ValueError, np.zeros(1, dtype='i4').view, []) + + assert_equal(np.zeros(2, dtype=[]) == np.zeros(2, dtype=[]), + np.ones(2, dtype=bool)) + + assert_equal(np.zeros((1, 2), dtype=[]) == a, + np.ones((1, 2), dtype=bool)) + class TestSubarray(object): def test_single_subarray(self): @@ -938,13 +963,6 @@ class TestDtypeAttributes(object): new_dtype = np.dtype(dtype.descr) assert_equal(new_dtype.itemsize, 16) - @pytest.mark.parametrize('t', np.typeDict.values()) - def test_name_builtin(self, t): - name = t.__name__ - if name.endswith('_'): - name = name[:-1] - assert_equal(np.dtype(t).name, name) - def test_name_dtype_subclass(self): # Ticket #4357 class user_def_subcls(np.void): @@ -968,7 +986,7 @@ class TestPickling(object): assert_equal(x[0], y[0]) @pytest.mark.parametrize('t', [int, float, complex, np.int32, str, object, - np.unicode, bool]) + np.compat.unicode, bool]) def test_builtin(self, t): self.check_pickling(np.dtype(t)) diff --git a/numpy/core/tests/test_function_base.py b/numpy/core/tests/test_function_base.py index 8b820bd75..c8a7cb6ce 100644 --- a/numpy/core/tests/test_function_base.py +++ b/numpy/core/tests/test_function_base.py @@ -49,7 +49,7 @@ class TestLogspace(object): assert_(len(y) == 50) y = logspace(0, 6, num=100) assert_(y[-1] == 10 ** 6) - y = logspace(0, 6, endpoint=0) + y = logspace(0, 6, endpoint=False) assert_(y[-1] < 10 ** 6) y = logspace(0, 6, num=7) assert_array_equal(y, [1, 10, 100, 1e3, 1e4, 1e5, 1e6]) @@ -229,17 +229,14 @@ class TestLinspace(object): assert_(len(y) == 50) y = linspace(2, 10, num=100) assert_(y[-1] == 10) - y = linspace(2, 10, endpoint=0) + y = linspace(2, 10, endpoint=False) assert_(y[-1] < 10) assert_raises(ValueError, linspace, 0, 10, num=-1) def test_corner(self): y = list(linspace(0, 1, 1)) assert_(y == [0.0], y) - with suppress_warnings() as sup: - sup.filter(DeprecationWarning, ".*safely interpreted as an integer") - y = list(linspace(0, 1, 2.5)) - assert_(y == [0.0, 1.0]) + assert_raises(TypeError, linspace, 0, 1, num=2.5) def test_type(self): t1 = linspace(0, 1, 0).dtype @@ -354,14 +351,20 @@ class TestLinspace(object): arange(j+1, dtype=int)) def test_retstep(self): - y = linspace(0, 1, 2, retstep=True) - assert_(isinstance(y, tuple) and len(y) == 2) - for num in (0, 1): - for ept in (False, True): + for num in [0, 1, 2]: + for ept in [False, True]: y = linspace(0, 1, num, endpoint=ept, retstep=True) - assert_(isinstance(y, tuple) and len(y) == 2 and - len(y[0]) == num and isnan(y[1]), - 'num={0}, endpoint={1}'.format(num, ept)) + assert isinstance(y, tuple) and len(y) == 2 + if num == 2: + y0_expect = [0.0, 1.0] if ept else [0.0, 0.5] + assert_array_equal(y[0], y0_expect) + assert_equal(y[1], y0_expect[1]) + elif num == 1 and not ept: + assert_array_equal(y[0], [0.0]) + assert_equal(y[1], 1.0) + else: + assert_array_equal(y[0], [0.0][:num]) + assert isnan(y[1]) def test_object(self): start = array(1, dtype='O') diff --git a/numpy/core/tests/test_issue14735.py b/numpy/core/tests/test_issue14735.py new file mode 100644 index 000000000..6105c8e6a --- /dev/null +++ b/numpy/core/tests/test_issue14735.py @@ -0,0 +1,29 @@ +import pytest +import warnings +import numpy as np + + +class Wrapper: + def __init__(self, array): + self.array = array + + def __len__(self): + return len(self.array) + + def __getitem__(self, item): + return type(self)(self.array[item]) + + def __getattr__(self, name): + if name.startswith("__array_"): + warnings.warn("object got converted", UserWarning, stacklevel=1) + + return getattr(self.array, name) + + def __repr__(self): + return "<Wrapper({self.array})>".format(self=self) + +@pytest.mark.filterwarnings("error") +def test_getattr_warning(): + array = Wrapper(np.arange(10)) + with pytest.raises(UserWarning, match="object got converted"): + np.asarray(array) diff --git a/numpy/core/tests/test_longdouble.py b/numpy/core/tests/test_longdouble.py index ee4197f8f..2b6e1c5a2 100644 --- a/numpy/core/tests/test_longdouble.py +++ b/numpy/core/tests/test_longdouble.py @@ -5,7 +5,8 @@ import pytest import numpy as np from numpy.testing import ( - assert_, assert_equal, assert_raises, assert_array_equal, temppath, + assert_, assert_equal, assert_raises, assert_warns, assert_array_equal, + temppath, ) from numpy.core.tests._locales import CommaDecimalPointLocale @@ -70,19 +71,54 @@ def test_fromstring(): err_msg="reading '%s'" % s) +def test_fromstring_complex(): + for ctype in ["complex", "cdouble", "cfloat"]: + # Check spacing between separator + assert_equal(np.fromstring("1, 2 , 3 ,4", sep=",", dtype=ctype), + np.array([1., 2., 3., 4.])) + # Real component not specified + assert_equal(np.fromstring("1j, -2j, 3j, 4e1j", sep=",", dtype=ctype), + np.array([1.j, -2.j, 3.j, 40.j])) + # Both components specified + assert_equal(np.fromstring("1+1j,2-2j, -3+3j, -4e1+4j", sep=",", dtype=ctype), + np.array([1. + 1.j, 2. - 2.j, - 3. + 3.j, - 40. + 4j])) + # Spaces at wrong places + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("1+2 j,3", dtype=ctype, sep=","), + np.array([1.])) + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("1+ 2j,3", dtype=ctype, sep=","), + np.array([1.])) + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("1 +2j,3", dtype=ctype, sep=","), + np.array([1.])) + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("1+j", dtype=ctype, sep=","), + np.array([1.])) + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("1+", dtype=ctype, sep=","), + np.array([1.])) + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("1j+1", dtype=ctype, sep=","), + np.array([1j])) + + def test_fromstring_bogus(): - assert_equal(np.fromstring("1. 2. 3. flop 4.", dtype=float, sep=" "), - np.array([1., 2., 3.])) + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("1. 2. 3. flop 4.", dtype=float, sep=" "), + np.array([1., 2., 3.])) def test_fromstring_empty(): - assert_equal(np.fromstring("xxxxx", sep="x"), - np.array([])) + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("xxxxx", sep="x"), + np.array([])) def test_fromstring_missing(): - assert_equal(np.fromstring("1xx3x4x5x6", sep="x"), - np.array([1])) + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("1xx3x4x5x6", sep="x"), + np.array([1])) class TestFileBased(object): @@ -95,9 +131,93 @@ class TestFileBased(object): with temppath() as path: with open(path, 'wt') as f: f.write("1. 2. 3. flop 4.\n") - res = np.fromfile(path, dtype=float, sep=" ") + + with assert_warns(DeprecationWarning): + res = np.fromfile(path, dtype=float, sep=" ") assert_equal(res, np.array([1., 2., 3.])) + def test_fromfile_complex(self): + for ctype in ["complex", "cdouble", "cfloat"]: + # Check spacing between separator and only real component specified + with temppath() as path: + with open(path, 'wt') as f: + f.write("1, 2 , 3 ,4\n") + + res = np.fromfile(path, dtype=ctype, sep=",") + assert_equal(res, np.array([1., 2., 3., 4.])) + + # Real component not specified + with temppath() as path: + with open(path, 'wt') as f: + f.write("1j, -2j, 3j, 4e1j\n") + + res = np.fromfile(path, dtype=ctype, sep=",") + assert_equal(res, np.array([1.j, -2.j, 3.j, 40.j])) + + # Both components specified + with temppath() as path: + with open(path, 'wt') as f: + f.write("1+1j,2-2j, -3+3j, -4e1+4j\n") + + res = np.fromfile(path, dtype=ctype, sep=",") + assert_equal(res, np.array([1. + 1.j, 2. - 2.j, - 3. + 3.j, - 40. + 4j])) + + # Spaces at wrong places + with temppath() as path: + with open(path, 'wt') as f: + f.write("1+2 j,3\n") + + with assert_warns(DeprecationWarning): + res = np.fromfile(path, dtype=ctype, sep=",") + assert_equal(res, np.array([1.])) + + # Spaces at wrong places + with temppath() as path: + with open(path, 'wt') as f: + f.write("1+ 2j,3\n") + + with assert_warns(DeprecationWarning): + res = np.fromfile(path, dtype=ctype, sep=",") + assert_equal(res, np.array([1.])) + + # Spaces at wrong places + with temppath() as path: + with open(path, 'wt') as f: + f.write("1 +2j,3\n") + + with assert_warns(DeprecationWarning): + res = np.fromfile(path, dtype=ctype, sep=",") + assert_equal(res, np.array([1.])) + + # Spaces at wrong places + with temppath() as path: + with open(path, 'wt') as f: + f.write("1+j\n") + + with assert_warns(DeprecationWarning): + res = np.fromfile(path, dtype=ctype, sep=",") + assert_equal(res, np.array([1.])) + + # Spaces at wrong places + with temppath() as path: + with open(path, 'wt') as f: + f.write("1+\n") + + with assert_warns(DeprecationWarning): + res = np.fromfile(path, dtype=ctype, sep=",") + assert_equal(res, np.array([1.])) + + # Spaces at wrong places + with temppath() as path: + with open(path, 'wt') as f: + f.write("1j+1\n") + + with assert_warns(DeprecationWarning): + res = np.fromfile(path, dtype=ctype, sep=",") + assert_equal(res, np.array([1.j])) + + + @pytest.mark.skipif(string_to_longdouble_inaccurate, reason="Need strtold_l") def test_fromfile(self): @@ -186,12 +306,14 @@ class TestCommaDecimalPointLocale(CommaDecimalPointLocale): assert_equal(a[0], f) def test_fromstring_best_effort_float(self): - assert_equal(np.fromstring("1,234", dtype=float, sep=" "), - np.array([1.])) + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("1,234", dtype=float, sep=" "), + np.array([1.])) def test_fromstring_best_effort(self): - assert_equal(np.fromstring("1,234", dtype=np.longdouble, sep=" "), - np.array([1.])) + with assert_warns(DeprecationWarning): + assert_equal(np.fromstring("1,234", dtype=np.longdouble, sep=" "), + np.array([1.])) def test_fromstring_foreign(self): s = "1.234" @@ -204,8 +326,10 @@ class TestCommaDecimalPointLocale(CommaDecimalPointLocale): assert_array_equal(a, b) def test_fromstring_foreign_value(self): - b = np.fromstring("1,234", dtype=np.longdouble, sep=" ") - assert_array_equal(b[0], 1) + with assert_warns(DeprecationWarning): + b = np.fromstring("1,234", dtype=np.longdouble, sep=" ") + assert_array_equal(b[0], 1) + @pytest.mark.parametrize("int_val", [ # cases discussed in gh-10723 diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 6fa8548a0..22d550ecc 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -20,6 +20,7 @@ import gc import weakref import pytest from contextlib import contextmanager +from test.support import no_tracing from numpy.compat import pickle @@ -114,7 +115,7 @@ class TestFlags(object): # Ensure that any base being writeable is sufficient to change flag; # this is especially interesting for arrays from an array interface. arr = np.arange(10) - + class subclass(np.ndarray): pass @@ -964,7 +965,7 @@ class TestCreation(object): @pytest.mark.skipif(sys.version_info[0] >= 3, reason="Not Python 2") def test_sequence_long(self): - assert_equal(np.array([long(4), long(4)]).dtype, np.long) + assert_equal(np.array([long(4), long(4)]).dtype, long) assert_equal(np.array([long(4), 2**80]).dtype, object) assert_equal(np.array([long(4), 2**80, long(4)]).dtype, object) assert_equal(np.array([2**80, long(4)]).dtype, object) @@ -1784,7 +1785,7 @@ class TestMethods(object): # test unicode sorts. s = 'aaaaaaaa' - a = np.array([s + chr(i) for i in range(101)], dtype=np.unicode) + a = np.array([s + chr(i) for i in range(101)], dtype=np.unicode_) b = a[::-1].copy() for kind in self.sort_kinds: msg = "unicode sort, kind=%s" % kind @@ -2036,7 +2037,7 @@ class TestMethods(object): # test unicode argsorts. s = 'aaaaaaaa' - a = np.array([s + chr(i) for i in range(101)], dtype=np.unicode) + a = np.array([s + chr(i) for i in range(101)], dtype=np.unicode_) b = a[::-1] r = np.arange(101) rr = r[::-1] @@ -2119,7 +2120,7 @@ class TestMethods(object): a = np.array(['aaaaaaaaa' for i in range(100)]) assert_equal(a.argsort(kind='m'), r) # unicode - a = np.array(['aaaaaaaaa' for i in range(100)], dtype=np.unicode) + a = np.array(['aaaaaaaaa' for i in range(100)], dtype=np.unicode_) assert_equal(a.argsort(kind='m'), r) def test_sort_unicode_kind(self): @@ -2248,7 +2249,7 @@ class TestMethods(object): 'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100197_1', 'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100198_1', 'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100199_1'], - dtype=np.unicode) + dtype=np.unicode_) ind = np.arange(len(a)) assert_equal([a.searchsorted(v, 'left') for v in a], ind) assert_equal([a.searchsorted(v, 'right') for v in a], ind + 1) @@ -2766,6 +2767,12 @@ class TestMethods(object): assert_equal(x1.flatten('F'), y1f) assert_equal(x1.flatten('F'), x1.T.flatten()) + def test_flatten_invalid_order(self): + # invalid after gh-14596 + for order in ['Z', 'c', False, True, 0, 8]: + x = np.array([[1, 2, 3], [4, 5, 6]], np.int32) + assert_raises(ValueError, x.flatten, {"order": order}) + @pytest.mark.parametrize('func', (np.dot, np.matmul)) def test_arr_mult(self, func): a = np.array([[1, 0], [0, 1]]) @@ -3573,10 +3580,10 @@ class TestBinop(object): assert_equal(np.modf(dummy, out=(None, a)), (1,)) assert_equal(np.modf(dummy, out=(dummy, a)), (1,)) assert_equal(np.modf(a, out=(dummy, a)), 0) - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', DeprecationWarning) - assert_equal(np.modf(dummy, out=a), (0,)) - assert_(w[0].category is DeprecationWarning) + with assert_raises(TypeError): + # Out argument must be tuple, since there are multiple outputs + np.modf(dummy, out=a) + assert_raises(ValueError, np.modf, dummy, out=(a,)) # 2 inputs, 1 output @@ -3941,13 +3948,13 @@ class TestPickling(object): def test_datetime64_byteorder(self): original = np.array([['2015-02-24T00:00:00.000000000']], dtype='datetime64[ns]') - + original_byte_reversed = original.copy(order='K') original_byte_reversed.dtype = original_byte_reversed.dtype.newbyteorder('S') original_byte_reversed.byteswap(inplace=True) new = pickle.loads(pickle.dumps(original_byte_reversed)) - + assert_equal(original.dtype, new.dtype) @@ -4076,17 +4083,17 @@ class TestArgmax(object): np.datetime64('2010-01-03T05:14:12'), np.datetime64('NaT'), np.datetime64('2015-09-23T10:10:13'), - np.datetime64('1932-10-10T03:50:30')], 4), + np.datetime64('1932-10-10T03:50:30')], 0), ([np.datetime64('2059-03-14T12:43:12'), np.datetime64('1996-09-21T14:43:15'), np.datetime64('NaT'), np.datetime64('2022-12-25T16:02:16'), np.datetime64('1963-10-04T03:14:12'), - np.datetime64('2013-05-08T18:15:23')], 0), + np.datetime64('2013-05-08T18:15:23')], 2), ([np.timedelta64(2, 's'), np.timedelta64(1, 's'), np.timedelta64('NaT', 's'), - np.timedelta64(3, 's')], 3), + np.timedelta64(3, 's')], 2), ([np.timedelta64('NaT', 's')] * 3, 0), ([timedelta(days=5, seconds=14), timedelta(days=2, seconds=35), @@ -4160,6 +4167,7 @@ class TestArgmax(object): assert_equal(a.argmax(out=out1, axis=0), np.argmax(a, out=out2, axis=0)) assert_equal(out1, out2) + @pytest.mark.leaks_references(reason="replaces None with NULL.") def test_object_argmax_with_NULLs(self): # See gh-6032 a = np.empty(4, dtype='O') @@ -4210,17 +4218,17 @@ class TestArgmin(object): np.datetime64('2010-01-03T05:14:12'), np.datetime64('NaT'), np.datetime64('2015-09-23T10:10:13'), - np.datetime64('1932-10-10T03:50:30')], 5), + np.datetime64('1932-10-10T03:50:30')], 0), ([np.datetime64('2059-03-14T12:43:12'), np.datetime64('1996-09-21T14:43:15'), np.datetime64('NaT'), np.datetime64('2022-12-25T16:02:16'), np.datetime64('1963-10-04T03:14:12'), - np.datetime64('2013-05-08T18:15:23')], 4), + np.datetime64('2013-05-08T18:15:23')], 2), ([np.timedelta64(2, 's'), np.timedelta64(1, 's'), np.timedelta64('NaT', 's'), - np.timedelta64(3, 's')], 1), + np.timedelta64(3, 's')], 2), ([np.timedelta64('NaT', 's')] * 3, 0), ([timedelta(days=5, seconds=14), timedelta(days=2, seconds=35), @@ -4308,6 +4316,7 @@ class TestArgmin(object): assert_equal(a.argmin(out=out1, axis=0), np.argmin(a, out=out2, axis=0)) assert_equal(out1, out2) + @pytest.mark.leaks_references(reason="replaces None with NULL.") def test_object_argmin_with_NULLs(self): # See gh-6032 a = np.empty(4, dtype='O') @@ -4335,18 +4344,14 @@ class TestMinMax(object): assert_equal(np.amax([[1, 2, 3]], axis=1), 3) def test_datetime(self): - # NaTs are ignored + # Do not ignore NaT for dtype in ('m8[s]', 'm8[Y]'): a = np.arange(10).astype(dtype) - a[3] = 'NaT' assert_equal(np.amin(a), a[0]) assert_equal(np.amax(a), a[9]) - a[0] = 'NaT' - assert_equal(np.amin(a), a[1]) - assert_equal(np.amax(a), a[9]) - a.fill('NaT') - assert_equal(np.amin(a), a[0]) - assert_equal(np.amax(a), a[0]) + a[3] = 'NaT' + assert_equal(np.amin(a), a[3]) + assert_equal(np.amax(a), a[3]) class TestNewaxis(object): @@ -4848,7 +4853,7 @@ class TestIO(object): offset_bytes = self.dtype.itemsize z = np.fromfile(f, dtype=self.dtype, offset=offset_bytes) assert_array_equal(z, self.x.flat[offset_items+count_items+1:]) - + with open(self.filename, 'wb') as f: self.x.tofile(f, sep=",") @@ -4938,7 +4943,8 @@ class TestIO(object): self._check_from(b'1,2,3,4', [1., 2., 3., 4.], dtype=float, sep=',') def test_malformed(self): - self._check_from(b'1.234 1,234', [1.234, 1.], sep=' ') + with assert_warns(DeprecationWarning): + self._check_from(b'1.234 1,234', [1.234, 1.], sep=' ') def test_long_sep(self): self._check_from(b'1_x_3_x_4_x_5', [1, 3, 4, 5], sep='_x_') @@ -4991,6 +4997,19 @@ class TestIO(object): self.test_tofile_sep() self.test_tofile_format() + def test_fromfile_subarray_binary(self): + # Test subarray dtypes which are absorbed into the shape + x = np.arange(24, dtype="i4").reshape(2, 3, 4) + x.tofile(self.filename) + res = np.fromfile(self.filename, dtype="(3,4)i4") + assert_array_equal(x, res) + + x_str = x.tobytes() + with assert_warns(DeprecationWarning): + # binary fromstring is deprecated + res = np.fromstring(x_str, dtype="(3,4)i4") + assert_array_equal(x, res) + class TestFromBuffer(object): @pytest.mark.parametrize('byteorder', ['<', '>']) @@ -5078,6 +5097,8 @@ class TestFlat(object): class TestResize(object): + + @no_tracing def test_basic(self): x = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) if IS_PYPY: @@ -5094,6 +5115,7 @@ class TestResize(object): assert_raises(ValueError, x.resize, (5, 1)) del y # avoid pyflakes unused variable warning. + @no_tracing def test_int_shape(self): x = np.eye(3) if IS_PYPY: @@ -5127,6 +5149,7 @@ class TestResize(object): assert_raises(TypeError, np.eye(3).resize, order=1) assert_raises(TypeError, np.eye(3).resize, refcheck='hi') + @no_tracing def test_freeform_shape(self): x = np.eye(3) if IS_PYPY: @@ -5135,6 +5158,7 @@ class TestResize(object): x.resize(3, 2, 1) assert_(x.shape == (3, 2, 1)) + @no_tracing def test_zeros_appended(self): x = np.eye(3) if IS_PYPY: @@ -5144,6 +5168,7 @@ class TestResize(object): assert_array_equal(x[0], np.eye(3)) assert_array_equal(x[1], np.zeros((3, 3))) + @no_tracing def test_obj_obj(self): # check memory is initialized on resize, gh-4857 a = np.ones(10, dtype=[('k', object, 2)]) @@ -6205,14 +6230,14 @@ class TestMatmul(MatmulCommon): r3 = np.matmul(args[0].copy(), args[1].copy()) assert_equal(r1, r3) - + def test_matmul_object(self): import fractions f = np.vectorize(fractions.Fraction) def random_ints(): return np.random.randint(1, 1000, size=(10, 3, 3)) - M1 = f(random_ints(), random_ints()) + M1 = f(random_ints(), random_ints()) M2 = f(random_ints(), random_ints()) M3 = self.matmul(M1, M2) @@ -6402,20 +6427,22 @@ class TestInner(object): class TestAlen(object): def test_basic(self): - m = np.array([1, 2, 3]) - assert_equal(np.alen(m), 3) + with pytest.warns(DeprecationWarning): + m = np.array([1, 2, 3]) + assert_equal(np.alen(m), 3) - m = np.array([[1, 2, 3], [4, 5, 7]]) - assert_equal(np.alen(m), 2) + m = np.array([[1, 2, 3], [4, 5, 7]]) + assert_equal(np.alen(m), 2) - m = [1, 2, 3] - assert_equal(np.alen(m), 3) + m = [1, 2, 3] + assert_equal(np.alen(m), 3) - m = [[1, 2, 3], [4, 5, 7]] - assert_equal(np.alen(m), 2) + m = [[1, 2, 3], [4, 5, 7]] + assert_equal(np.alen(m), 2) def test_singleton(self): - assert_equal(np.alen(5), 1) + with pytest.warns(DeprecationWarning): + assert_equal(np.alen(5), 1) class TestChoose(object): @@ -7202,6 +7229,13 @@ class TestNewBufferProtocol(object): RuntimeError, "ndim", np.array, m) + # The above seems to create some deep cycles, clean them up for + # easier reference count debugging: + del c_u8_33d, m + for i in range(33): + if gc.collect() == 0: + break + def test_error_pointer_type(self): # gh-6741 m = memoryview(ctypes.pointer(ctypes.c_uint8())) @@ -7746,6 +7780,7 @@ if not IS_PYPY: d = np.ones(100) assert_(sys.getsizeof(d) < sys.getsizeof(d.reshape(100, 1, 1).copy())) + @no_tracing def test_resize(self): d = np.ones(100) old = sys.getsizeof(d) @@ -7880,20 +7915,20 @@ class TestBytestringArrayNonzero(object): class TestUnicodeArrayNonzero(object): def test_empty_ustring_array_is_falsey(self): - assert_(not np.array([''], dtype=np.unicode)) + assert_(not np.array([''], dtype=np.unicode_)) def test_whitespace_ustring_array_is_falsey(self): - a = np.array(['eggs'], dtype=np.unicode) + a = np.array(['eggs'], dtype=np.unicode_) a[0] = ' \0\0' assert_(not a) def test_all_null_ustring_array_is_falsey(self): - a = np.array(['eggs'], dtype=np.unicode) + a = np.array(['eggs'], dtype=np.unicode_) a[0] = '\0\0\0\0' assert_(not a) def test_null_inside_ustring_array_is_truthy(self): - a = np.array(['eggs'], dtype=np.unicode) + a = np.array(['eggs'], dtype=np.unicode_) a[0] = ' \0 \0' assert_(a) @@ -8094,6 +8129,8 @@ class TestWritebackIfCopy(object): arr_wb[...] = 100 assert_equal(arr, -100) + @pytest.mark.leaks_references( + reason="increments self in dealloc; ignore since deprecated path.") def test_dealloc_warning(self): with suppress_warnings() as sup: sup.record(RuntimeWarning) diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index cf66751f8..daec9ce6d 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -2104,7 +2104,7 @@ def test_iter_buffering_string(): assert_equal(i[0], b'abc') assert_equal(i[0].dtype, np.dtype('S6')) - a = np.array(['abc', 'a', 'abcd'], dtype=np.unicode) + a = np.array(['abc', 'a', 'abcd'], dtype=np.unicode_) assert_equal(a.dtype, np.dtype('U4')) assert_raises(TypeError, nditer, a, ['buffered'], ['readonly'], op_dtypes='U2') diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index 935b84234..ffebdf648 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -1254,6 +1254,39 @@ class TestNonzero(object): a = np.array([[False], [TrueThenFalse()]]) assert_raises(RuntimeError, np.nonzero, a) + def test_nonzero_exception_safe(self): + # gh-13930 + + class ThrowsAfter: + def __init__(self, iters): + self.iters_left = iters + + def __bool__(self): + if self.iters_left == 0: + raise ValueError("called `iters` times") + + self.iters_left -= 1 + return True + + """ + Test that a ValueError is raised instead of a SystemError + + If the __bool__ function is called after the error state is set, + Python (cpython) will raise a SystemError. + """ + + # assert that an exception in first pass is handled correctly + a = np.array([ThrowsAfter(5)]*10) + assert_raises(ValueError, np.nonzero, a) + + # raise exception in second pass for 1-dimensional loop + a = np.array([ThrowsAfter(15)]*10) + assert_raises(ValueError, np.nonzero, a) + + # raise exception in second pass for n-dimensional loop + a = np.array([[ThrowsAfter(15)]]*10) + assert_raises(ValueError, np.nonzero, a) + class TestIndex(object): def test_boolean(self): @@ -1308,6 +1341,11 @@ class TestBinaryRepr(object): exp = '1' + (width - 1) * '0' assert_equal(np.binary_repr(num, width=width), exp) + def test_large_neg_int64(self): + # See gh-14289. + assert_equal(np.binary_repr(np.int64(-2**62), width=64), + '11' + '0'*62) + class TestBaseRepr(object): def test_base3(self): @@ -2529,6 +2567,11 @@ class TestCorrelate(object): z = np.correlate(y, x, mode='full') assert_array_almost_equal(z, r_z) + def test_zero_size(self): + with pytest.raises(ValueError): + np.correlate(np.array([]), np.ones(1000), mode='full') + with pytest.raises(ValueError): + np.correlate(np.ones(1000), np.array([]), mode='full') class TestConvolve(object): def test_object(self): @@ -2545,6 +2588,30 @@ class TestConvolve(object): class TestArgwhere(object): + + @pytest.mark.parametrize('nd', [0, 1, 2]) + def test_nd(self, nd): + # get an nd array with multiple elements in every dimension + x = np.empty((2,)*nd, bool) + + # none + x[...] = False + assert_equal(np.argwhere(x).shape, (0, nd)) + + # only one + x[...] = False + x.flat[0] = True + assert_equal(np.argwhere(x).shape, (1, nd)) + + # all but one + x[...] = True + x.flat[0] = False + assert_equal(np.argwhere(x).shape, (x.size - 1, nd)) + + # all + x[...] = True + assert_equal(np.argwhere(x).shape, (x.size, nd)) + def test_2D(self): x = np.arange(6).reshape((2, 3)) assert_array_equal(np.argwhere(x > 1), @@ -2886,7 +2953,7 @@ class TestIndices(object): assert_array_equal(x, np.array([[0], [1], [2], [3]])) assert_array_equal(y, np.array([[0, 1, 2]])) - @pytest.mark.parametrize("dtype", [np.int, np.float32, np.float64]) + @pytest.mark.parametrize("dtype", [np.int32, np.int64, np.float32, np.float64]) @pytest.mark.parametrize("dims", [(), (0,), (4, 3)]) def test_return_type(self, dtype, dims): inds = np.indices(dims, dtype=dtype) diff --git a/numpy/core/tests/test_numerictypes.py b/numpy/core/tests/test_numerictypes.py index d0ff5578a..387740e35 100644 --- a/numpy/core/tests/test_numerictypes.py +++ b/numpy/core/tests/test_numerictypes.py @@ -498,3 +498,32 @@ class TestDocStrings(object): assert_('int64' in np.int_.__doc__) elif np.int64 is np.longlong: assert_('int64' in np.longlong.__doc__) + + +class TestScalarTypeNames: + # gh-9799 + + numeric_types = [ + np.byte, np.short, np.intc, np.int_, np.longlong, + np.ubyte, np.ushort, np.uintc, np.uint, np.ulonglong, + np.half, np.single, np.double, np.longdouble, + np.csingle, np.cdouble, np.clongdouble, + ] + + def test_names_are_unique(self): + # none of the above may be aliases for each other + assert len(set(self.numeric_types)) == len(self.numeric_types) + + # names must be unique + names = [t.__name__ for t in self.numeric_types] + assert len(set(names)) == len(names) + + @pytest.mark.parametrize('t', numeric_types) + def test_names_reflect_attributes(self, t): + """ Test that names correspond to where the type is under ``np.`` """ + assert getattr(np, t.__name__) is t + + @pytest.mark.parametrize('t', numeric_types) + def test_names_are_undersood_by_dtype(self, t): + """ Test the dtype constructor maps names back to the type """ + assert np.dtype(t.__name__).type is t diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index cbd4ceafc..3880b1394 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -17,6 +17,7 @@ from numpy.testing import ( _assert_valid_refcount, HAS_REFCOUNT, ) from numpy.compat import asbytes, asunicode, long, pickle +from test.support import no_tracing try: RecursionError @@ -1316,6 +1317,7 @@ class TestRegression(object): assert_(pickle.loads( pickle.dumps(test_record, protocol=proto)) == test_record) + @no_tracing def test_blasdot_uninitialized_memory(self): # Ticket #950 for m in [0, 1, 2]: @@ -1511,7 +1513,7 @@ class TestRegression(object): min //= -1 with np.errstate(divide="ignore"): - for t in (np.int8, np.int16, np.int32, np.int64, int, np.long): + for t in (np.int8, np.int16, np.int32, np.int64, int, np.compat.long): test_type(t) def test_buffer_hashlib(self): @@ -1539,7 +1541,8 @@ class TestRegression(object): def test_fromstring_crash(self): # Ticket #1345: the following should not cause a crash - np.fromstring(b'aa, aa, 1.0', sep=',') + with assert_warns(DeprecationWarning): + np.fromstring(b'aa, aa, 1.0', sep=',') def test_ticket_1539(self): dtypes = [x for x in np.typeDict.values() @@ -2111,7 +2114,7 @@ class TestRegression(object): # Ticket #1578, the mismatch only showed up when running # python-debug for python versions >= 2.7, and then as # a core dump and error message. - a = np.array(['abc'], dtype=np.unicode)[0] + a = np.array(['abc'], dtype=np.unicode_)[0] del a def test_refcount_error_in_clip(self): @@ -2481,26 +2484,6 @@ class TestRegression(object): np.array([T()]) - def test_2d__array__shape(self): - class T(object): - def __array__(self): - return np.ndarray(shape=(0,0)) - - # Make sure __array__ is used instead of Sequence methods. - def __iter__(self): - return iter([]) - - def __getitem__(self, idx): - raise AssertionError("__getitem__ was called") - - def __len__(self): - return 0 - - - t = T() - #gh-13659, would raise in broadcasting [x=t for x in result] - np.array([t]) - @pytest.mark.skipif(sys.maxsize < 2 ** 31 + 1, reason='overflows 32-bit python') @pytest.mark.skipif(sys.platform == 'win32' and sys.version_info[:2] < (3, 8), reason='overflows on windows, fixed in bpo-16865') diff --git a/numpy/core/tests/test_scalarinherit.py b/numpy/core/tests/test_scalarinherit.py index 9e32cf624..6a5c4fde9 100644 --- a/numpy/core/tests/test_scalarinherit.py +++ b/numpy/core/tests/test_scalarinherit.py @@ -68,8 +68,7 @@ class TestCharacter(object): def test_char_repeat(self): np_s = np.string_('abc') np_u = np.unicode_('abc') - np_i = np.int(5) res_s = b'abc' * 5 res_u = u'abc' * 5 - assert_(np_s * np_i == res_s) - assert_(np_u * np_i == res_u) + assert_(np_s * 5 == res_s) + assert_(np_u * 5 == res_u) diff --git a/numpy/core/tests/test_scalarmath.py b/numpy/core/tests/test_scalarmath.py index ebba457e3..c84380cd9 100644 --- a/numpy/core/tests/test_scalarmath.py +++ b/numpy/core/tests/test_scalarmath.py @@ -11,7 +11,7 @@ import numpy as np from numpy.testing import ( assert_, assert_equal, assert_raises, assert_almost_equal, assert_array_equal, IS_PYPY, suppress_warnings, _gen_alignment_data, - assert_warns + assert_warns, assert_raises_regex, ) types = [np.bool_, np.byte, np.ubyte, np.short, np.ushort, np.intc, np.uintc, @@ -293,6 +293,16 @@ class TestModulus(object): rem = operator.mod(finf, fone) assert_(np.isnan(rem), 'dt: %s' % dt) + def test_inplace_floordiv_handling(self): + # issue gh-12927 + # this only applies to in-place floordiv //=, because the output type + # promotes to float which does not fit + a = np.array([1, 2], np.int64) + b = np.array([1, 2], np.uint64) + pattern = 'could not be coerced to provided output parameter' + with assert_raises_regex(TypeError, pattern): + a //= b + class TestComplexDivision(object): def test_zero_division(self): @@ -664,3 +674,31 @@ class TestAbs(object): def test_numpy_abs(self): self._test_abs_func(np.abs) + + +class TestBitShifts(object): + + @pytest.mark.parametrize('type_code', np.typecodes['AllInteger']) + @pytest.mark.parametrize('op', + [operator.rshift, operator.lshift], ids=['>>', '<<']) + def test_shift_all_bits(self, type_code, op): + """ Shifts where the shift amount is the width of the type or wider """ + # gh-2449 + dt = np.dtype(type_code) + nbits = dt.itemsize * 8 + for val in [5, -5]: + for shift in [nbits, nbits + 4]: + val_scl = dt.type(val) + shift_scl = dt.type(shift) + res_scl = op(val_scl, shift_scl) + if val_scl < 0 and op is operator.rshift: + # sign bit is preserved + assert_equal(res_scl, -1) + else: + assert_equal(res_scl, 0) + + # Result on scalars should be the same as on arrays + val_arr = np.array([val]*32, dtype=dt) + shift_arr = np.array([shift]*32, dtype=dt) + res_arr = op(val_arr, shift_arr) + assert_equal(res_arr, res_scl) diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 707c690dd..526925ece 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -980,7 +980,7 @@ class TestUfunc(object): assert_array_equal(out, mm_row_col_vec.squeeze()) def test_matrix_multiply(self): - self.compare_matrix_multiply_results(np.long) + self.compare_matrix_multiply_results(np.int64) self.compare_matrix_multiply_results(np.double) def test_matrix_multiply_umath_empty(self): @@ -1092,7 +1092,6 @@ class TestUfunc(object): arr0d = np.array(HasComparisons()) assert_equal(arr0d == arr0d, True) assert_equal(np.equal(arr0d, arr0d), True) # normal behavior is a cast - assert_equal(np.equal(arr0d, arr0d, dtype=object), '==') arr1d = np.array([HasComparisons()]) assert_equal(arr1d == arr1d, np.array([True])) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 1d93f9ac0..e892e81d2 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -75,11 +75,9 @@ class TestOut(object): assert_(r1 is o1) assert_(r2 is o2) - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', DeprecationWarning) + with assert_raises(TypeError): + # Out argument must be tuple, since there are multiple outputs. r1, r2 = np.frexp(d, out=o1, subok=subok) - assert_(r1 is o1) - assert_(w[0].category is DeprecationWarning) assert_raises(ValueError, np.add, a, 2, o, o, subok=subok) assert_raises(ValueError, np.add, a, 2, o, out=o, subok=subok) @@ -165,19 +163,14 @@ class TestOut(object): else: assert_(type(r1) == np.ndarray) - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', DeprecationWarning) + with assert_raises(TypeError): + # Out argument must be tuple, since there are multiple outputs. r1, r2 = np.frexp(d, out=o1, subok=subok) - if subok: - assert_(isinstance(r2, ArrayWrap)) - else: - assert_(type(r2) == np.ndarray) - assert_(w[0].category is DeprecationWarning) class TestComparisons(object): def test_ignore_object_identity_in_equal(self): - # Check error raised when comparing identical objects whose comparison + # Check comparing identical objects whose comparison # is not a simple boolean, e.g., arrays that are compared elementwise. a = np.array([np.array([1, 2, 3]), None], dtype=object) assert_raises(ValueError, np.equal, a, a) @@ -195,7 +188,7 @@ class TestComparisons(object): assert_equal(np.equal(a, a), [False]) def test_ignore_object_identity_in_not_equal(self): - # Check error raised when comparing identical objects whose comparison + # Check comparing identical objects whose comparison # is not a simple boolean, e.g., arrays that are compared elementwise. a = np.array([np.array([1, 2, 3]), None], dtype=object) assert_raises(ValueError, np.not_equal, a, a) @@ -678,30 +671,151 @@ class TestSpecialFloats(object): assert_raises(FloatingPointError, np.log, np.float32(-np.inf)) assert_raises(FloatingPointError, np.log, np.float32(-1.0)) -class TestExpLogFloat32(object): + def test_sincos_values(self): + with np.errstate(all='ignore'): + x = [np.nan, np.nan, np.nan, np.nan] + y = [np.nan, -np.nan, np.inf, -np.inf] + for dt in ['f', 'd', 'g']: + xf = np.array(x, dtype=dt) + yf = np.array(y, dtype=dt) + assert_equal(np.sin(yf), xf) + assert_equal(np.cos(yf), xf) + + with np.errstate(invalid='raise'): + assert_raises(FloatingPointError, np.sin, np.float32(-np.inf)) + assert_raises(FloatingPointError, np.sin, np.float32(np.inf)) + assert_raises(FloatingPointError, np.cos, np.float32(-np.inf)) + assert_raises(FloatingPointError, np.cos, np.float32(np.inf)) + + def test_sqrt_values(self): + with np.errstate(all='ignore'): + x = [np.nan, np.nan, np.inf, np.nan, 0.] + y = [np.nan, -np.nan, np.inf, -np.inf, 0.] + for dt in ['f', 'd', 'g']: + xf = np.array(x, dtype=dt) + yf = np.array(y, dtype=dt) + assert_equal(np.sqrt(yf), xf) + + #with np.errstate(invalid='raise'): + # for dt in ['f', 'd', 'g']: + # assert_raises(FloatingPointError, np.sqrt, np.array(-100., dtype=dt)) + + def test_abs_values(self): + x = [np.nan, np.nan, np.inf, np.inf, 0., 0., 1.0, 1.0] + y = [np.nan, -np.nan, np.inf, -np.inf, 0., -0., -1.0, 1.0] + for dt in ['f', 'd', 'g']: + xf = np.array(x, dtype=dt) + yf = np.array(y, dtype=dt) + assert_equal(np.abs(yf), xf) + + def test_square_values(self): + x = [np.nan, np.nan, np.inf, np.inf] + y = [np.nan, -np.nan, np.inf, -np.inf] + with np.errstate(all='ignore'): + for dt in ['f', 'd', 'g']: + xf = np.array(x, dtype=dt) + yf = np.array(y, dtype=dt) + assert_equal(np.square(yf), xf) + + with np.errstate(over='raise'): + assert_raises(FloatingPointError, np.square, np.array(1E32, dtype='f')) + assert_raises(FloatingPointError, np.square, np.array(1E200, dtype='d')) + + def test_reciprocal_values(self): + with np.errstate(all='ignore'): + x = [np.nan, np.nan, 0.0, -0.0, np.inf, -np.inf] + y = [np.nan, -np.nan, np.inf, -np.inf, 0., -0.] + for dt in ['f', 'd', 'g']: + xf = np.array(x, dtype=dt) + yf = np.array(y, dtype=dt) + assert_equal(np.reciprocal(yf), xf) + + with np.errstate(divide='raise'): + for dt in ['f', 'd', 'g']: + assert_raises(FloatingPointError, np.reciprocal, np.array(-0.0, dtype=dt)) + +# func : [maxulperror, low, high] +avx_ufuncs = {'sqrt' :[1, 0., 100.], + 'absolute' :[0, -100., 100.], + 'reciprocal' :[1, 1., 100.], + 'square' :[1, -100., 100.], + 'rint' :[0, -100., 100.], + 'floor' :[0, -100., 100.], + 'ceil' :[0, -100., 100.], + 'trunc' :[0, -100., 100.]} + +class TestAVXUfuncs(object): + def test_avx_based_ufunc(self): + strides = np.array([-4,-3,-2,-1,1,2,3,4]) + np.random.seed(42) + for func, prop in avx_ufuncs.items(): + maxulperr = prop[0] + minval = prop[1] + maxval = prop[2] + # various array sizes to ensure masking in AVX is tested + for size in range(1,32): + myfunc = getattr(np, func) + x_f32 = np.float32(np.random.uniform(low=minval, high=maxval, + size=size)) + x_f64 = np.float64(x_f32) + x_f128 = np.longdouble(x_f32) + y_true128 = myfunc(x_f128) + if maxulperr == 0: + assert_equal(myfunc(x_f32), np.float32(y_true128)) + assert_equal(myfunc(x_f64), np.float64(y_true128)) + else: + assert_array_max_ulp(myfunc(x_f32), np.float32(y_true128), + maxulp=maxulperr) + assert_array_max_ulp(myfunc(x_f64), np.float64(y_true128), + maxulp=maxulperr) + # various strides to test gather instruction + if size > 1: + y_true32 = myfunc(x_f32) + y_true64 = myfunc(x_f64) + for jj in strides: + assert_equal(myfunc(x_f64[::jj]), y_true64[::jj]) + assert_equal(myfunc(x_f32[::jj]), y_true32[::jj]) + +class TestAVXFloat32Transcendental(object): def test_exp_float32(self): np.random.seed(42) x_f32 = np.float32(np.random.uniform(low=0.0,high=88.1,size=1000000)) x_f64 = np.float64(x_f32) - assert_array_max_ulp(np.exp(x_f32), np.float32(np.exp(x_f64)), maxulp=2.6) + assert_array_max_ulp(np.exp(x_f32), np.float32(np.exp(x_f64)), maxulp=3) def test_log_float32(self): np.random.seed(42) x_f32 = np.float32(np.random.uniform(low=0.0,high=1000,size=1000000)) x_f64 = np.float64(x_f32) - assert_array_max_ulp(np.log(x_f32), np.float32(np.log(x_f64)), maxulp=3.9) + assert_array_max_ulp(np.log(x_f32), np.float32(np.log(x_f64)), maxulp=4) + + def test_sincos_float32(self): + np.random.seed(42) + N = 1000000 + M = np.int_(N/20) + index = np.random.randint(low=0, high=N, size=M) + x_f32 = np.float32(np.random.uniform(low=-100.,high=100.,size=N)) + # test coverage for elements > 117435.992f for which glibc is used + x_f32[index] = np.float32(10E+10*np.random.rand(M)) + x_f64 = np.float64(x_f32) + assert_array_max_ulp(np.sin(x_f32), np.float32(np.sin(x_f64)), maxulp=2) + assert_array_max_ulp(np.cos(x_f32), np.float32(np.cos(x_f64)), maxulp=2) - def test_strided_exp_log_float32(self): + def test_strided_float32(self): np.random.seed(42) - strides = np.random.randint(low=-100, high=100, size=100) - sizes = np.random.randint(low=1, high=2000, size=100) + strides = np.array([-4,-3,-2,-1,1,2,3,4]) + sizes = np.arange(2,100) for ii in sizes: x_f32 = np.float32(np.random.uniform(low=0.01,high=88.1,size=ii)) exp_true = np.exp(x_f32) log_true = np.log(x_f32) + sin_true = np.sin(x_f32) + cos_true = np.cos(x_f32) for jj in strides: assert_array_almost_equal_nulp(np.exp(x_f32[::jj]), exp_true[::jj], nulp=2) assert_array_almost_equal_nulp(np.log(x_f32[::jj]), log_true[::jj], nulp=2) + assert_array_almost_equal_nulp(np.sin(x_f32[::jj]), sin_true[::jj], nulp=2) + assert_array_almost_equal_nulp(np.cos(x_f32[::jj]), cos_true[::jj], nulp=2) class TestLogAddExp(_FilterInvalids): def test_logaddexp_values(self): @@ -1655,7 +1769,6 @@ class TestSpecialMethods(object): ok = np.empty(1).view(Ok) bad = np.empty(1).view(Bad) - # double-free (segfault) of "ok" if "bad" raises an exception for i in range(10): assert_raises(RuntimeError, ncu.frexp, 1, ok, bad) @@ -2129,10 +2242,9 @@ class TestSpecialMethods(object): assert_(np.modf(a, None) == {}) assert_(np.modf(a, None, None) == {}) assert_(np.modf(a, out=(None, None)) == {}) - with warnings.catch_warnings(record=True) as w: - warnings.filterwarnings('always', '', DeprecationWarning) - assert_(np.modf(a, out=None) == {}) - assert_(w[0].category is DeprecationWarning) + with assert_raises(TypeError): + # Out argument must be tuple, since there are multiple outputs. + np.modf(a, out=None) # don't give positional and output argument, or too many arguments. # wrong number of arguments in the tuple is an error too. diff --git a/numpy/core/tests/test_umath_accuracy.py b/numpy/core/tests/test_umath_accuracy.py index fcbed0dd3..fec180786 100644 --- a/numpy/core/tests/test_umath_accuracy.py +++ b/numpy/core/tests/test_umath_accuracy.py @@ -35,9 +35,10 @@ class TestAccuracy(object): for filename in files: data_dir = path.join(path.dirname(__file__), 'data') filepath = path.join(data_dir, filename) - file_without_comments = (r for r in open(filepath) if not r[0] in ('$', '#')) + with open(filepath) as fid: + file_without_comments = (r for r in fid if not r[0] in ('$', '#')) data = np.genfromtxt(file_without_comments, - dtype=('|S39','|S39','|S39',np.int), + dtype=('|S39','|S39','|S39',int), names=('type','input','output','ulperr'), delimiter=',', skip_header=1) diff --git a/numpy/ctypeslib.py b/numpy/ctypeslib.py index c9016cf8f..58f3ef9d3 100644 --- a/numpy/ctypeslib.py +++ b/numpy/ctypeslib.py @@ -120,7 +120,7 @@ else: """ if ctypes.__version__ < '1.0.1': import warnings - warnings.warn("All features of ctypes interface may not work " \ + warnings.warn("All features of ctypes interface may not work " "with ctypes < 1.0.1", stacklevel=2) ext = os.path.splitext(libname)[1] diff --git a/numpy/distutils/__init__.py b/numpy/distutils/__init__.py index 55514750e..8dbb63b28 100644 --- a/numpy/distutils/__init__.py +++ b/numpy/distutils/__init__.py @@ -1,12 +1,31 @@ +""" +An enhanced distutils, providing support for Fortran compilers, for BLAS, +LAPACK and other common libraries for numerical computing, and more. + +Public submodules are:: + + misc_util + system_info + cpu_info + log + exec_command + +For details, please see the *Packaging* and *NumPy Distutils User Guide* +sections of the NumPy Reference Guide. + +For configuring the preference for and location of libraries like BLAS and +LAPACK, and for setting include paths and similar build options, please see +``site.cfg.example`` in the root of the NumPy repository or sdist. + +""" + from __future__ import division, absolute_import, print_function -from .__version__ import version as __version__ # Must import local ccompiler ASAP in order to get # customized CCompiler.spawn effective. from . import ccompiler from . import unixccompiler -from .info import __doc__ from .npy_pkg_config import * # If numpy is installed, add distutils.test() @@ -28,7 +47,7 @@ def customized_fcompiler(plat=None, compiler=None): c.customize() return c -def customized_ccompiler(plat=None, compiler=None): - c = ccompiler.new_compiler(plat=plat, compiler=compiler) +def customized_ccompiler(plat=None, compiler=None, verbose=1): + c = ccompiler.new_compiler(plat=plat, compiler=compiler, verbose=verbose) c.customize('') return c diff --git a/numpy/distutils/__version__.py b/numpy/distutils/__version__.py deleted file mode 100644 index 969decbba..000000000 --- a/numpy/distutils/__version__.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import division, absolute_import, print_function - -major = 0 -minor = 4 -micro = 0 -version = '%(major)d.%(minor)d.%(micro)d' % (locals()) diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py index 14451fa66..684c7535b 100644 --- a/numpy/distutils/ccompiler.py +++ b/numpy/distutils/ccompiler.py @@ -140,7 +140,10 @@ def CCompiler_spawn(self, cmd, display=None): display = ' '.join(list(display)) log.info(display) try: - subprocess.check_output(cmd) + if self.verbose: + subprocess.check_output(cmd) + else: + subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: o = exc.output s = exc.returncode @@ -162,7 +165,8 @@ def CCompiler_spawn(self, cmd, display=None): if is_sequence(cmd): cmd = ' '.join(list(cmd)) - forward_bytes_to_stdout(o) + if self.verbose: + forward_bytes_to_stdout(o) if re.search(b'Too many open files', o): msg = '\nTry rerunning setup command until build succeeds.' @@ -528,6 +532,11 @@ def CCompiler_customize(self, dist, need_cxx=0): 'g++' in self.compiler[0] or 'clang' in self.compiler[0]): self._auto_depends = True + if 'gcc' in self.compiler[0]: + # add std=c99 flag for gcc + # TODO: does this need to be more specific? + self.compiler.append('-std=c99') + self.compiler_so.append('-std=c99') elif os.name == 'posix': import tempfile import shutil @@ -727,10 +736,12 @@ if sys.platform == 'win32': _distutils_new_compiler = new_compiler def new_compiler (plat=None, compiler=None, - verbose=0, + verbose=None, dry_run=0, force=0): # Try first C compilers from numpy.distutils. + if verbose is None: + verbose = log.get_threshold() <= log.INFO if plat is None: plat = os.name try: @@ -763,6 +774,7 @@ def new_compiler (plat=None, raise DistutilsModuleError(("can't compile C/C++ code: unable to find class '%s' " + "in module '%s'") % (class_name, module_name)) compiler = klass(None, dry_run, force) + compiler.verbose = verbose log.debug('new_compiler returns %s' % (klass)) return compiler diff --git a/numpy/distutils/command/build.py b/numpy/distutils/command/build.py index 3d7101582..5a9da1217 100644 --- a/numpy/distutils/command/build.py +++ b/numpy/distutils/command/build.py @@ -16,8 +16,8 @@ class build(old_build): user_options = old_build.user_options + [ ('fcompiler=', None, "specify the Fortran compiler type"), - ('parallel=', 'j', - "number of parallel jobs"), + ('warn-error', None, + "turn all warnings into errors (-Werror)"), ] help_options = old_build.help_options + [ @@ -28,17 +28,12 @@ class build(old_build): def initialize_options(self): old_build.initialize_options(self) self.fcompiler = None - self.parallel = None + self.warn_error = False def finalize_options(self): - if self.parallel: - try: - self.parallel = int(self.parallel) - except ValueError: - raise ValueError("--parallel/-j argument must be an integer") build_scripts = self.build_scripts old_build.finalize_options(self) - plat_specifier = ".%s-%s" % (get_platform(), sys.version[0:3]) + plat_specifier = ".{}-{}.{}".format(get_platform(), *sys.version_info[:2]) if build_scripts is None: self.build_scripts = os.path.join(self.build_base, 'scripts' + plat_specifier) diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py index 910493a77..13edf0717 100644 --- a/numpy/distutils/command/build_clib.py +++ b/numpy/distutils/command/build_clib.py @@ -33,15 +33,18 @@ class build_clib(old_build_clib): ('inplace', 'i', 'Build in-place'), ('parallel=', 'j', "number of parallel jobs"), + ('warn-error', None, + "turn all warnings into errors (-Werror)"), ] - boolean_options = old_build_clib.boolean_options + ['inplace'] + boolean_options = old_build_clib.boolean_options + ['inplace', 'warn-error'] def initialize_options(self): old_build_clib.initialize_options(self) self.fcompiler = None self.inplace = 0 self.parallel = None + self.warn_error = None def finalize_options(self): if self.parallel: @@ -50,7 +53,10 @@ class build_clib(old_build_clib): except ValueError: raise ValueError("--parallel/-j argument must be an integer") old_build_clib.finalize_options(self) - self.set_undefined_options('build', ('parallel', 'parallel')) + self.set_undefined_options('build', + ('parallel', 'parallel'), + ('warn_error', 'warn_error'), + ) def have_f_sources(self): for (lib_name, build_info) in self.libraries: @@ -86,6 +92,10 @@ class build_clib(old_build_clib): self.compiler.customize(self.distribution, need_cxx=self.have_cxx_sources()) + if self.warn_error: + self.compiler.compiler.append('-Werror') + self.compiler.compiler_so.append('-Werror') + libraries = self.libraries self.libraries = None self.compiler.customize_cmd(self) diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index ef54fb25e..cd9b1c6f1 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -33,6 +33,8 @@ class build_ext (old_build_ext): "specify the Fortran compiler type"), ('parallel=', 'j', "number of parallel jobs"), + ('warn-error', None, + "turn all warnings into errors (-Werror)"), ] help_options = old_build_ext.help_options + [ @@ -40,10 +42,13 @@ class build_ext (old_build_ext): show_fortran_compilers), ] + boolean_options = old_build_ext.boolean_options + ['warn-error'] + def initialize_options(self): old_build_ext.initialize_options(self) self.fcompiler = None self.parallel = None + self.warn_error = None def finalize_options(self): if self.parallel: @@ -69,7 +74,10 @@ class build_ext (old_build_ext): self.include_dirs.extend(incl_dirs) old_build_ext.finalize_options(self) - self.set_undefined_options('build', ('parallel', 'parallel')) + self.set_undefined_options('build', + ('parallel', 'parallel'), + ('warn_error', 'warn_error'), + ) def run(self): if not self.extensions: @@ -116,6 +124,11 @@ class build_ext (old_build_ext): force=self.force) self.compiler.customize(self.distribution) self.compiler.customize_cmd(self) + + if self.warn_error: + self.compiler.compiler.append('-Werror') + self.compiler.compiler_so.append('-Werror') + self.compiler.show_customization() # Setup directory for storing generated extra DLL files on Windows diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index 6d5d305d2..3e0522c5f 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -53,9 +53,12 @@ class build_src(build_ext.build_ext): ('inplace', 'i', "ignore build-lib and put compiled extensions into the source " + "directory alongside your pure Python modules"), + ('verbose-cfg', None, + "change logging level from WARN to INFO which will show all " + + "compiler output") ] - boolean_options = ['force', 'inplace'] + boolean_options = ['force', 'inplace', 'verbose-cfg'] help_options = [] @@ -76,6 +79,7 @@ class build_src(build_ext.build_ext): self.swig_opts = None self.swig_cpp = None self.swig = None + self.verbose_cfg = None def finalize_options(self): self.set_undefined_options('build', @@ -90,7 +94,7 @@ class build_src(build_ext.build_ext): self.data_files = self.distribution.data_files or [] if self.build_src is None: - plat_specifier = ".%s-%s" % (get_platform(), sys.version[0:3]) + plat_specifier = ".{}-{}.{}".format(get_platform(), *sys.version_info[:2]) self.build_src = os.path.join(self.build_base, 'src'+plat_specifier) # py_modules_dict is used in build_py.find_package_modules @@ -362,9 +366,16 @@ class build_src(build_ext.build_ext): # incl_dirs = extension.include_dirs #if self.build_src not in incl_dirs: # incl_dirs.append(self.build_src) - build_dir = os.path.join(*([self.build_src]\ + build_dir = os.path.join(*([self.build_src] +name.split('.')[:-1])) self.mkpath(build_dir) + + if self.verbose_cfg: + new_level = log.INFO + else: + new_level = log.WARN + old_level = log.set_threshold(new_level) + for func in func_sources: source = func(extension, build_dir) if not source: @@ -375,7 +386,7 @@ class build_src(build_ext.build_ext): else: log.info(" adding '%s' to sources." % (source,)) new_sources.append(source) - + log.set_threshold(old_level) return new_sources def filter_py_files(self, sources): @@ -540,7 +551,7 @@ class build_src(build_ext.build_ext): if is_sequence(extension): name = extension[0] else: name = extension.name - target_dir = os.path.join(*([self.build_src]\ + target_dir = os.path.join(*([self.build_src] +name.split('.')[:-1])) target_file = os.path.join(target_dir, ext_name + 'module.c') new_sources.append(target_file) diff --git a/numpy/distutils/command/config.py b/numpy/distutils/command/config.py index e9ca7bea8..b9f2fa76e 100644 --- a/numpy/distutils/command/config.py +++ b/numpy/distutils/command/config.py @@ -442,10 +442,10 @@ class config(old_config): of the program and its output. """ # 2008-11-16, RemoveMe - warnings.warn("\n+++++++++++++++++++++++++++++++++++++++++++++++++\n" \ - "Usage of get_output is deprecated: please do not \n" \ - "use it anymore, and avoid configuration checks \n" \ - "involving running executable on the target machine.\n" \ + warnings.warn("\n+++++++++++++++++++++++++++++++++++++++++++++++++\n" + "Usage of get_output is deprecated: please do not \n" + "use it anymore, and avoid configuration checks \n" + "involving running executable on the target machine.\n" "+++++++++++++++++++++++++++++++++++++++++++++++++\n", DeprecationWarning, stacklevel=2) self._check_compiler() diff --git a/numpy/distutils/cpuinfo.py b/numpy/distutils/cpuinfo.py index 580299347..bc9728335 100644 --- a/numpy/distutils/cpuinfo.py +++ b/numpy/distutils/cpuinfo.py @@ -242,16 +242,16 @@ class LinuxCPUInfo(CPUInfoBase): return self.is_PentiumIV() and self.has_sse3() def _is_Nocona(self): - return self.is_Intel() \ - and (self.info[0]['cpu family'] == '6' \ - or self.info[0]['cpu family'] == '15' ) \ - and (self.has_sse3() and not self.has_ssse3())\ - and re.match(r'.*?\blm\b', self.info[0]['flags']) is not None + return (self.is_Intel() + and (self.info[0]['cpu family'] == '6' + or self.info[0]['cpu family'] == '15') + and (self.has_sse3() and not self.has_ssse3()) + and re.match(r'.*?\blm\b', self.info[0]['flags']) is not None) def _is_Core2(self): - return self.is_64bit() and self.is_Intel() and \ - re.match(r'.*?Core\(TM\)2\b', \ - self.info[0]['model name']) is not None + return (self.is_64bit() and self.is_Intel() and + re.match(r'.*?Core\(TM\)2\b', + self.info[0]['model name']) is not None) def _is_Itanium(self): return re.match(r'.*?Itanium\b', @@ -632,13 +632,13 @@ class Win32CPUInfo(CPUInfoBase): def _has_sse(self): if self.is_Intel(): - return (self.info[0]['Family']==6 and \ - self.info[0]['Model'] in [7, 8, 9, 10, 11]) \ - or self.info[0]['Family']==15 + return ((self.info[0]['Family']==6 and + self.info[0]['Model'] in [7, 8, 9, 10, 11]) + or self.info[0]['Family']==15) elif self.is_AMD(): - return (self.info[0]['Family']==6 and \ - self.info[0]['Model'] in [6, 7, 8, 10]) \ - or self.info[0]['Family']==15 + return ((self.info[0]['Family']==6 and + self.info[0]['Model'] in [6, 7, 8, 10]) + or self.info[0]['Family']==15) else: return False diff --git a/numpy/distutils/exec_command.py b/numpy/distutils/exec_command.py index 2e7b9e463..712f22666 100644 --- a/numpy/distutils/exec_command.py +++ b/numpy/distutils/exec_command.py @@ -217,7 +217,7 @@ def exec_command(command, execute_in='', use_shell=None, use_tee=None, # 2019-01-30, 1.17 warnings.warn('exec_command is deprecated since NumPy v1.17, use ' 'subprocess.Popen instead', DeprecationWarning, stacklevel=1) - log.debug('exec_command(%r,%s)' % (command,\ + log.debug('exec_command(%r,%s)' % (command, ','.join(['%s=%r'%kv for kv in env.items()]))) if use_tee is None: diff --git a/numpy/distutils/extension.py b/numpy/distutils/extension.py index 935f3eec9..872bd5362 100644 --- a/numpy/distutils/extension.py +++ b/numpy/distutils/extension.py @@ -19,8 +19,24 @@ if sys.version_info[0] >= 3: cxx_ext_re = re.compile(r'.*[.](cpp|cxx|cc)\Z', re.I).match fortran_pyf_ext_re = re.compile(r'.*[.](f90|f95|f77|for|ftn|f|pyf)\Z', re.I).match + class Extension(old_Extension): - def __init__ ( + """ + Parameters + ---------- + name : str + Extension name. + sources : list of str + List of source file locations relative to the top directory of + the package. + extra_compile_args : list of str + Extra command line arguments to pass to the compiler. + extra_f77_compile_args : list of str + Extra command line arguments to pass to the fortran77 compiler. + extra_f90_compile_args : list of str + Extra command line arguments to pass to the fortran90 compiler. + """ + def __init__( self, name, sources, include_dirs=None, define_macros=None, diff --git a/numpy/distutils/fcompiler/compaq.py b/numpy/distutils/fcompiler/compaq.py index 07d502706..671b3a55f 100644 --- a/numpy/distutils/fcompiler/compaq.py +++ b/numpy/distutils/fcompiler/compaq.py @@ -95,7 +95,7 @@ class CompaqVisualFCompiler(FCompiler): raise e except ValueError: e = get_exception() - if not "path']" in str(e): + if not "'path'" in str(e): print("Unexpected ValueError in", __file__) raise e diff --git a/numpy/distutils/fcompiler/environment.py b/numpy/distutils/fcompiler/environment.py index 73a5e98e1..bb362d483 100644 --- a/numpy/distutils/fcompiler/environment.py +++ b/numpy/distutils/fcompiler/environment.py @@ -59,17 +59,13 @@ class EnvironmentConfig(object): if envvar_contents is not None: envvar_contents = convert(envvar_contents) if var and append: - if os.environ.get('NPY_DISTUTILS_APPEND_FLAGS', '0') == '1': + if os.environ.get('NPY_DISTUTILS_APPEND_FLAGS', '1') == '1': var.extend(envvar_contents) else: + # NPY_DISTUTILS_APPEND_FLAGS was explicitly set to 0 + # to keep old (overwrite flags rather than append to + # them) behavior var = envvar_contents - if 'NPY_DISTUTILS_APPEND_FLAGS' not in os.environ.keys(): - msg = "{} is used as is, not appended ".format(envvar) + \ - "to flags already defined " + \ - "by numpy.distutils! Use NPY_DISTUTILS_APPEND_FLAGS=1 " + \ - "to obtain appending behavior instead (this " + \ - "behavior will become default in a future release)." - warnings.warn(msg, UserWarning, stacklevel=3) else: var = envvar_contents if confvar is not None and self._conf: diff --git a/numpy/distutils/info.py b/numpy/distutils/info.py deleted file mode 100644 index 2f5310665..000000000 --- a/numpy/distutils/info.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Enhanced distutils with Fortran compilers support and more. -""" -from __future__ import division, absolute_import, print_function - -postpone_import = True diff --git a/numpy/distutils/log.py b/numpy/distutils/log.py index 37f9fe5dd..ff7de86b1 100644 --- a/numpy/distutils/log.py +++ b/numpy/distutils/log.py @@ -67,6 +67,8 @@ def set_threshold(level, force=False): ' %s to %s' % (prev_level, level)) return prev_level +def get_threshold(): + return _global_log.threshold def set_verbosity(v, force=False): prev_level = _global_log.threshold diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index 2a6472b95..d46ff8981 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -859,7 +859,7 @@ class Configuration(object): print(message) def warn(self, message): - sys.stderr.write('Warning: %s' % (message,)) + sys.stderr.write('Warning: %s\n' % (message,)) def set_options(self, **options): """ @@ -1687,6 +1687,41 @@ class Configuration(object): and will be installed as foo.ini in the 'lib' subpath. + When cross-compiling with numpy distutils, it might be necessary to + use modified npy-pkg-config files. Using the default/generated files + will link with the host libraries (i.e. libnpymath.a). For + cross-compilation you of-course need to link with target libraries, + while using the host Python installation. + + You can copy out the numpy/core/lib/npy-pkg-config directory, add a + pkgdir value to the .ini files and set NPY_PKG_CONFIG_PATH environment + variable to point to the directory with the modified npy-pkg-config + files. + + Example npymath.ini modified for cross-compilation:: + + [meta] + Name=npymath + Description=Portable, core math library implementing C99 standard + Version=0.1 + + [variables] + pkgname=numpy.core + pkgdir=/build/arm-linux-gnueabi/sysroot/usr/lib/python3.7/site-packages/numpy/core + prefix=${pkgdir} + libdir=${prefix}/lib + includedir=${prefix}/include + + [default] + Libs=-L${libdir} -lnpymath + Cflags=-I${includedir} + Requires=mlib + + [msvc] + Libs=/LIBPATH:${libdir} npymath.lib + Cflags=/INCLUDE:${includedir} + Requires=mlib + """ if subst_dict is None: subst_dict = {} @@ -2092,9 +2127,22 @@ def get_numpy_include_dirs(): return include_dirs def get_npy_pkg_dir(): - """Return the path where to find the npy-pkg-config directory.""" + """Return the path where to find the npy-pkg-config directory. + + If the NPY_PKG_CONFIG_PATH environment variable is set, the value of that + is returned. Otherwise, a path inside the location of the numpy module is + returned. + + The NPY_PKG_CONFIG_PATH can be useful when cross-compiling, maintaining + customized npy-pkg-config .ini files for the cross-compilation + environment, and using them when cross-compiling. + + """ # XXX: import here for bootstrapping reasons import numpy + d = os.environ.get('NPY_PKG_CONFIG_PATH') + if d is not None: + return d d = os.path.join(os.path.dirname(numpy.__file__), 'core', 'lib', 'npy-pkg-config') return d diff --git a/numpy/distutils/npy_pkg_config.py b/numpy/distutils/npy_pkg_config.py index bfe8b9f77..48584b4c4 100644 --- a/numpy/distutils/npy_pkg_config.py +++ b/numpy/distutils/npy_pkg_config.py @@ -426,7 +426,7 @@ if __name__ == '__main__': if options.define_variable: m = re.search(r'([\S]+)=([\S]+)', options.define_variable) if not m: - raise ValueError("--define-variable option should be of " \ + raise ValueError("--define-variable option should be of " "the form --define-variable=foo=bar") else: name = m.group(1) diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index ba2b1f46c..4786b3a0c 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -21,9 +21,17 @@ classes are available: blas_info lapack_info openblas_info + openblas64__info + openblas_ilp64_info blis_info blas_opt_info # usage recommended lapack_opt_info # usage recommended + blas_ilp64_opt_info # usage recommended (general ILP64 BLAS) + lapack_ilp64_opt_info # usage recommended (general ILP64 LAPACK) + blas_ilp64_plain_opt_info # usage recommended (general ILP64 BLAS, no symbol suffix) + lapack_ilp64_plain_opt_info # usage recommended (general ILP64 LAPACK, no symbol suffix) + blas64__opt_info # usage recommended (general ILP64 BLAS, 64_ symbol suffix) + lapack64__opt_info # usage recommended (general ILP64 LAPACK, 64_ symbol suffix) fftw_info,dfftw_info,sfftw_info fftw_threads_info,dfftw_threads_info,sfftw_threads_info djbfft_info @@ -146,7 +154,7 @@ else: from distutils.errors import DistutilsError from distutils.dist import Distribution import distutils.sysconfig -from distutils import log +from numpy.distutils import log from distutils.util import get_platform from numpy.distutils.exec_command import ( @@ -156,7 +164,7 @@ from numpy.distutils.misc_util import (is_sequence, is_string, get_shared_lib_extension) from numpy.distutils.command.config import config as cmd_config from numpy.distutils.compat import get_exception -from numpy.distutils import customized_ccompiler +from numpy.distutils import customized_ccompiler as _customized_ccompiler from numpy.distutils import _shell_utils import distutils.ccompiler import tempfile @@ -169,6 +177,15 @@ _bits = {'32bit': 32, '64bit': 64} platform_bits = _bits[platform.architecture()[0]] +global_compiler = None + +def customized_ccompiler(): + global global_compiler + if not global_compiler: + global_compiler = _customized_ccompiler() + return global_compiler + + def _c_string_literal(s): """ Convert a python string into a literal suitable for inclusion into C code @@ -397,6 +414,10 @@ def get_info(name, notfound_action=0): 'lapack_mkl': lapack_mkl_info, # use lapack_opt instead 'blas_mkl': blas_mkl_info, # use blas_opt instead 'accelerate': accelerate_info, # use blas_opt instead + 'openblas64_': openblas64__info, + 'openblas64__lapack': openblas64__lapack_info, + 'openblas_ilp64': openblas_ilp64_info, + 'openblas_ilp64_lapack': openblas_ilp64_lapack_info, 'x11': x11_info, 'fft_opt': fft_opt_info, 'fftw': fftw_info, @@ -419,7 +440,13 @@ def get_info(name, notfound_action=0): 'numarray': numarray_info, 'numerix': numerix_info, 'lapack_opt': lapack_opt_info, + 'lapack_ilp64_opt': lapack_ilp64_opt_info, + 'lapack_ilp64_plain_opt': lapack_ilp64_plain_opt_info, + 'lapack64__opt': lapack64__opt_info, 'blas_opt': blas_opt_info, + 'blas_ilp64_opt': blas_ilp64_opt_info, + 'blas_ilp64_plain_opt': blas_ilp64_plain_opt_info, + 'blas64__opt': blas64__opt_info, 'boost_python': boost_python_info, 'agg2': agg2_info, 'wx': wx_info, @@ -456,7 +483,7 @@ class AliasedOptionError(DistutilsError): class AtlasNotFoundError(NotFoundError): """ - Atlas (http://math-atlas.sourceforge.net/) libraries not found. + Atlas (http://github.com/math-atlas/math-atlas) libraries not found. Directories to search for the libraries can be specified in the numpy/distutils/site.cfg file (section [atlas]) or by setting the ATLAS environment variable.""" @@ -485,6 +512,13 @@ class LapackSrcNotFoundError(LapackNotFoundError): the LAPACK_SRC environment variable.""" +class LapackILP64NotFoundError(NotFoundError): + """ + 64-bit Lapack libraries not found. + Known libraries in numpy/distutils/site.cfg file are: + openblas64_, openblas_ilp64 + """ + class BlasOptNotFoundError(NotFoundError): """ Optimized (vendor) Blas libraries are not found. @@ -499,6 +533,12 @@ class BlasNotFoundError(NotFoundError): numpy/distutils/site.cfg file (section [blas]) or by setting the BLAS environment variable.""" +class BlasILP64NotFoundError(NotFoundError): + """ + 64-bit Blas libraries not found. + Known libraries in numpy/distutils/site.cfg file are: + openblas64_, openblas_ilp64 + """ class BlasSrcNotFoundError(BlasNotFoundError): """ @@ -550,7 +590,6 @@ class system_info(object): dir_env_var = None search_static_first = 0 # XXX: disabled by default, may disappear in # future unless it is proved to be useful. - verbosity = 1 saved_results = {} notfounderror = NotFoundError @@ -558,7 +597,6 @@ class system_info(object): def __init__(self, default_lib_dirs=default_lib_dirs, default_include_dirs=default_include_dirs, - verbosity=1, ): self.__class__.info = {} self.local_prefixes = [] @@ -704,7 +742,7 @@ class system_info(object): log.info(' FOUND:') res = self.saved_results.get(self.__class__.__name__) - if self.verbosity > 0 and flag: + if log.get_threshold() <= log.INFO and flag: for k, v in res.items(): v = str(v) if k in ['sources', 'libraries'] and len(v) > 270: @@ -914,7 +952,7 @@ class system_info(object): """Return a list of existing paths composed by all combinations of items from the arguments. """ - return combine_paths(*args, **{'verbosity': self.verbosity}) + return combine_paths(*args) class fft_opt_info(system_info): @@ -1531,12 +1569,12 @@ def get_atlas_version(**config): try: s, o = c.get_output(atlas_version_c_text, libraries=libraries, library_dirs=library_dirs, - use_tee=(system_info.verbosity > 0)) + ) if s and re.search(r'undefined reference to `_gfortran', o, re.M): s, o = c.get_output(atlas_version_c_text, libraries=libraries + ['gfortran'], library_dirs=library_dirs, - use_tee=(system_info.verbosity > 0)) + ) if not s: warnings.warn(textwrap.dedent(""" ***************************************************** @@ -1582,7 +1620,7 @@ def get_atlas_version(**config): log.info('Status: %d', s) log.info('Output: %s', o) - if atlas_version == '3.2.1_pre3.3.6': + elif atlas_version == '3.2.1_pre3.3.6': dict_append(info, define_macros=[('NO_ATLAS_INFO', -2)]) else: dict_append(info, define_macros=[( @@ -1593,10 +1631,10 @@ def get_atlas_version(**config): class lapack_opt_info(system_info): - notfounderror = LapackNotFoundError - # Default order of LAPACK checks + # List of all known BLAS libraries, in the default order lapack_order = ['mkl', 'openblas', 'flame', 'atlas', 'accelerate', 'lapack'] + order_env_var_name = 'NPY_LAPACK_ORDER' def _calc_info_mkl(self): info = get_info('lapack_mkl') @@ -1688,8 +1726,11 @@ class lapack_opt_info(system_info): return True return False + def _calc_info(self, name): + return getattr(self, '_calc_info_{}'.format(name))() + def calc_info(self): - user_order = os.environ.get('NPY_LAPACK_ORDER', None) + user_order = os.environ.get(self.order_env_var_name, None) if user_order is None: lapack_order = self.lapack_order else: @@ -1709,7 +1750,7 @@ class lapack_opt_info(system_info): "values: {}".format(non_existing)) for lapack in lapack_order: - if getattr(self, '_calc_info_{}'.format(lapack))(): + if self._calc_info(lapack): return if 'lapack' not in lapack_order: @@ -1719,11 +1760,53 @@ class lapack_opt_info(system_info): warnings.warn(LapackSrcNotFoundError.__doc__ or '', stacklevel=2) -class blas_opt_info(system_info): +class _ilp64_opt_info_mixin: + symbol_suffix = None + symbol_prefix = None + + def _check_info(self, info): + macros = dict(info.get('define_macros', [])) + prefix = macros.get('BLAS_SYMBOL_PREFIX', '') + suffix = macros.get('BLAS_SYMBOL_SUFFIX', '') + + if self.symbol_prefix not in (None, prefix): + return False + + if self.symbol_suffix not in (None, suffix): + return False + + return bool(info) + + +class lapack_ilp64_opt_info(lapack_opt_info, _ilp64_opt_info_mixin): + notfounderror = LapackILP64NotFoundError + lapack_order = ['openblas64_', 'openblas_ilp64'] + order_env_var_name = 'NPY_LAPACK_ILP64_ORDER' + + def _calc_info(self, name): + info = get_info(name + '_lapack') + if self._check_info(info): + self.set_info(**info) + return True + return False + + +class lapack_ilp64_plain_opt_info(lapack_ilp64_opt_info): + # Same as lapack_ilp64_opt_info, but fix symbol names + symbol_prefix = '' + symbol_suffix = '' + + +class lapack64__opt_info(lapack_ilp64_opt_info): + symbol_prefix = '' + symbol_suffix = '64_' + +class blas_opt_info(system_info): notfounderror = BlasNotFoundError - # Default order of BLAS checks + # List of all known BLAS libraries, in the default order blas_order = ['mkl', 'blis', 'openblas', 'atlas', 'accelerate', 'blas'] + order_env_var_name = 'NPY_BLAS_ORDER' def _calc_info_mkl(self): info = get_info('blas_mkl') @@ -1788,8 +1871,11 @@ class blas_opt_info(system_info): self.set_info(**info) return True + def _calc_info(self, name): + return getattr(self, '_calc_info_{}'.format(name))() + def calc_info(self): - user_order = os.environ.get('NPY_BLAS_ORDER', None) + user_order = os.environ.get(self.order_env_var_name, None) if user_order is None: blas_order = self.blas_order else: @@ -1807,7 +1893,7 @@ class blas_opt_info(system_info): raise ValueError("blas_opt_info user defined BLAS order has unacceptable values: {}".format(non_existing)) for blas in blas_order: - if getattr(self, '_calc_info_{}'.format(blas))(): + if self._calc_info(blas): return if 'blas' not in blas_order: @@ -1817,6 +1903,29 @@ class blas_opt_info(system_info): warnings.warn(BlasSrcNotFoundError.__doc__ or '', stacklevel=2) +class blas_ilp64_opt_info(blas_opt_info, _ilp64_opt_info_mixin): + notfounderror = BlasILP64NotFoundError + blas_order = ['openblas64_', 'openblas_ilp64'] + order_env_var_name = 'NPY_BLAS_ILP64_ORDER' + + def _calc_info(self, name): + info = get_info(name) + if self._check_info(info): + self.set_info(**info) + return True + return False + + +class blas_ilp64_plain_opt_info(blas_ilp64_opt_info): + symbol_prefix = '' + symbol_suffix = '' + + +class blas64__opt_info(blas_ilp64_opt_info): + symbol_prefix = '' + symbol_suffix = '64_' + + class blas_info(system_info): section = 'blas' dir_env_var = 'BLAS' @@ -1905,8 +2014,6 @@ class blas_info(system_info): library_dirs=info['library_dirs'], extra_postargs=info.get('extra_link_args', [])) return libs - # This breaks the for loop - break except distutils.ccompiler.LinkError: pass finally: @@ -1918,12 +2025,24 @@ class openblas_info(blas_info): section = 'openblas' dir_env_var = 'OPENBLAS' _lib_names = ['openblas'] + _require_symbols = [] notfounderror = BlasNotFoundError - def check_embedded_lapack(self, info): - return True + @property + def symbol_prefix(self): + try: + return self.cp.get(self.section, 'symbol_prefix') + except NoOptionError: + return '' - def calc_info(self): + @property + def symbol_suffix(self): + try: + return self.cp.get(self.section, 'symbol_suffix') + except NoOptionError: + return '' + + def _calc_info(self): c = customized_ccompiler() lib_dirs = self.get_lib_dirs() @@ -1941,23 +2060,33 @@ class openblas_info(blas_info): # Try gfortran-compatible library files info = self.check_msvc_gfortran_libs(lib_dirs, openblas_libs) # Skip lapack check, we'd need build_ext to do it - assume_lapack = True + skip_symbol_check = True elif info: - assume_lapack = False + skip_symbol_check = False info['language'] = 'c' if info is None: - return + return None # Add extra info for OpenBLAS extra_info = self.calc_extra_info() dict_append(info, **extra_info) - if not (assume_lapack or self.check_embedded_lapack(info)): - return + if not (skip_symbol_check or self.check_symbols(info)): + return None info['define_macros'] = [('HAVE_CBLAS', None)] - self.set_info(**info) + if self.symbol_prefix: + info['define_macros'] += [('BLAS_SYMBOL_PREFIX', self.symbol_prefix)] + if self.symbol_suffix: + info['define_macros'] += [('BLAS_SYMBOL_SUFFIX', self.symbol_suffix)] + + return info + + def calc_info(self): + info = self._calc_info() + if info is not None: + self.set_info(**info) def check_msvc_gfortran_libs(self, library_dirs, libraries): # First, find the full path to each library directory @@ -1990,24 +2119,27 @@ class openblas_info(blas_info): return info -class openblas_lapack_info(openblas_info): - section = 'openblas' - dir_env_var = 'OPENBLAS' - _lib_names = ['openblas'] - notfounderror = BlasNotFoundError - - def check_embedded_lapack(self, info): + def check_symbols(self, info): res = False c = customized_ccompiler() tmpdir = tempfile.mkdtemp() + + prototypes = "\n".join("void %s%s%s();" % (self.symbol_prefix, + symbol_name, + self.symbol_suffix) + for symbol_name in self._require_symbols) + calls = "\n".join("%s%s%s();" % (self.symbol_prefix, + symbol_name, + self.symbol_suffix) + for symbol_name in self._require_symbols) s = textwrap.dedent("""\ - void zungqr_(); + %(prototypes)s int main(int argc, const char *argv[]) { - zungqr_(); + %(calls)s return 0; - }""") + }""") % dict(prototypes=prototypes, calls=calls) src = os.path.join(tmpdir, 'source.c') out = os.path.join(tmpdir, 'a.out') # Add the additional "extra" arguments @@ -2032,9 +2164,49 @@ class openblas_lapack_info(openblas_info): shutil.rmtree(tmpdir) return res +class openblas_lapack_info(openblas_info): + section = 'openblas' + dir_env_var = 'OPENBLAS' + _lib_names = ['openblas'] + _require_symbols = ['zungqr_'] + notfounderror = BlasNotFoundError + class openblas_clapack_info(openblas_lapack_info): _lib_names = ['openblas', 'lapack'] +class openblas_ilp64_info(openblas_info): + section = 'openblas_ilp64' + dir_env_var = 'OPENBLAS_ILP64' + _lib_names = ['openblas64'] + _require_symbols = ['dgemm_', 'cblas_dgemm'] + notfounderror = BlasILP64NotFoundError + + def _calc_info(self): + info = super()._calc_info() + if info is not None: + info['define_macros'] += [('HAVE_BLAS_ILP64', None)] + return info + +class openblas_ilp64_lapack_info(openblas_ilp64_info): + _require_symbols = ['dgemm_', 'cblas_dgemm', 'zungqr_', 'LAPACKE_zungqr'] + + def _calc_info(self): + info = super()._calc_info() + if info: + info['define_macros'] += [('HAVE_LAPACKE', None)] + return info + +class openblas64__info(openblas_ilp64_info): + # ILP64 Openblas, with default symbol suffix + section = 'openblas64_' + dir_env_var = 'OPENBLAS64_' + _lib_names = ['openblas64_'] + symbol_suffix = '64_' + symbol_prefix = '' + +class openblas64__lapack_info(openblas_ilp64_lapack_info, openblas64__info): + pass + class blis_info(blas_info): section = 'blis' dir_env_var = 'BLIS' diff --git a/numpy/distutils/tests/test_fcompiler.py b/numpy/distutils/tests/test_fcompiler.py index ba19a97ea..6d245fbd4 100644 --- a/numpy/distutils/tests/test_fcompiler.py +++ b/numpy/distutils/tests/test_fcompiler.py @@ -45,37 +45,3 @@ def test_fcompiler_flags(monkeypatch): else: assert_(new_flags == prev_flags + [new_flag]) - -def test_fcompiler_flags_append_warning(monkeypatch): - # Test to check that the warning for append behavior changing in future - # is triggered. Need to use a real compiler instance so that we have - # non-empty flags to start with (otherwise the "if var and append" check - # will always be false). - try: - with suppress_warnings() as sup: - sup.record() - fc = numpy.distutils.fcompiler.new_fcompiler(compiler='gnu95') - fc.customize() - except numpy.distutils.fcompiler.CompilerNotFound: - pytest.skip("gfortran not found, so can't execute this test") - - # Ensure NPY_DISTUTILS_APPEND_FLAGS not defined - monkeypatch.delenv('NPY_DISTUTILS_APPEND_FLAGS', raising=False) - - for opt, envvar in customizable_flags: - new_flag = '-dummy-{}-flag'.format(opt) - with suppress_warnings() as sup: - sup.record() - prev_flags = getattr(fc.flag_vars, opt) - - monkeypatch.setenv(envvar, new_flag) - with suppress_warnings() as sup: - sup.record() - new_flags = getattr(fc.flag_vars, opt) - if prev_flags: - # Check that warning was issued - assert len(sup.log) == 1 - - monkeypatch.delenv(envvar) - assert_(new_flags == [new_flag]) - diff --git a/numpy/doc/basics.py b/numpy/doc/basics.py index 7946c6432..c05f347a1 100644 --- a/numpy/doc/basics.py +++ b/numpy/doc/basics.py @@ -18,7 +18,7 @@ The primitive types supported are tied closely to those in C: - C type - Description - * - `np.bool` + * - `np.bool_` - ``bool`` - Boolean (True or False) stored as a byte @@ -283,7 +283,7 @@ NumPy provides `numpy.iinfo` and `numpy.finfo` to verify the minimum or maximum values of NumPy integer and floating point values respectively :: - >>> np.iinfo(np.int) # Bounds of the default integer on this system. + >>> np.iinfo(int) # Bounds of the default integer on this system. iinfo(min=-9223372036854775808, max=9223372036854775807, dtype=int64) >>> np.iinfo(np.int32) # Bounds of a 32-bit integer iinfo(min=-2147483648, max=2147483647, dtype=int32) @@ -314,8 +314,8 @@ compiler's ``long double`` available as ``np.longdouble`` (and ``np.clongdouble`` for the complex numbers). You can find out what your numpy provides with ``np.finfo(np.longdouble)``. -NumPy does not provide a dtype with more precision than C -``long double``\\s; in particular, the 128-bit IEEE quad precision +NumPy does not provide a dtype with more precision than C's +``long double``\\; in particular, the 128-bit IEEE quad precision data type (FORTRAN's ``REAL*16``\\) is not available. For efficient memory alignment, ``np.longdouble`` is usually stored diff --git a/numpy/doc/broadcasting.py b/numpy/doc/broadcasting.py index 0bdb6ae7d..cb548a0d0 100644 --- a/numpy/doc/broadcasting.py +++ b/numpy/doc/broadcasting.py @@ -42,7 +42,7 @@ We can think of the scalar ``b`` being *stretched* during the arithmetic operation into an array with the same shape as ``a``. The new elements in ``b`` are simply copies of the original scalar. The stretching analogy is only conceptual. NumPy is smart enough to use the original scalar value -without actually making copies, so that broadcasting operations are as +without actually making copies so that broadcasting operations are as memory and computationally efficient as possible. The code in the second example is more efficient than that in the first @@ -52,7 +52,7 @@ because broadcasting moves less memory around during the multiplication General Broadcasting Rules ========================== When operating on two arrays, NumPy compares their shapes element-wise. -It starts with the trailing dimensions, and works its way forward. Two +It starts with the trailing dimensions and works its way forward. Two dimensions are compatible when 1) they are equal, or @@ -61,8 +61,7 @@ dimensions are compatible when If these conditions are not met, a ``ValueError: operands could not be broadcast together`` exception is thrown, indicating that the arrays have incompatible shapes. The size of -the resulting array is the maximum size along each dimension of the input -arrays. +the resulting array is the size that is not 1 along each axis of the inputs. Arrays do not need to have the same *number* of dimensions. For example, if you have a ``256x256x3`` array of RGB values, and you want to scale diff --git a/numpy/doc/dispatch.py b/numpy/doc/dispatch.py index 15e9fff6a..c9029941b 100644 --- a/numpy/doc/dispatch.py +++ b/numpy/doc/dispatch.py @@ -72,7 +72,7 @@ The ``__array_ufunc__`` receives: - ``inputs``, which could be a mixture of different types - ``kwargs``, keyword arguments passed to the function -For this example we will only handle the method ``'__call__``. +For this example we will only handle the method ``__call__``. >>> from numbers import Number >>> class DiagonalArray: @@ -218,7 +218,7 @@ For completeness, to support the usage ``arr.sum()`` add a method ``sum`` that calls ``numpy.sum(self)``, and the same for ``mean``. >>> @implements(np.sum) -... def sum(a): +... def sum(arr): ... "Implementation of np.sum for DiagonalArray objects" ... return arr._i * arr._N ... diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index 4b983893a..d0685328e 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -118,7 +118,8 @@ For example, consider the following Python code: def __new__(cls, *args): print('Cls in __new__:', cls) print('Args in __new__:', args) - return object.__new__(cls, *args) + # The `object` type __new__ method takes a single argument. + return object.__new__(cls) def __init__(self, *args): print('type(self) in __init__:', type(self)) diff --git a/numpy/doc/ufuncs.py b/numpy/doc/ufuncs.py index a112e559c..df2c455ec 100644 --- a/numpy/doc/ufuncs.py +++ b/numpy/doc/ufuncs.py @@ -13,9 +13,9 @@ example is the addition operator: :: >>> np.array([0,2,3,4]) + np.array([1,1,-1,2]) array([1, 3, 2, 6]) -The unfunc module lists all the available ufuncs in numpy. Documentation on +The ufunc module lists all the available ufuncs in numpy. Documentation on the specific ufuncs may be found in those modules. This documentation is -intended to address the more general aspects of unfuncs common to most of +intended to address the more general aspects of ufuncs common to most of them. All of the ufuncs that make use of Python operators (e.g., +, -, etc.) have equivalent functions defined (e.g. add() for +) diff --git a/numpy/f2py/__init__.py b/numpy/f2py/__init__.py index d146739bb..42e3632fd 100644 --- a/numpy/f2py/__init__.py +++ b/numpy/f2py/__init__.py @@ -109,6 +109,7 @@ def compile(source, output = '' else: status = 0 + output = output.decode() if verbose: print(output) finally: diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index c41dd77c6..ce79f680f 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -179,17 +179,29 @@ f2cmap_all = {'real': {'': 'float', '4': 'float', '8': 'double', 'character': {'': 'string'} } -if os.path.isfile('.f2py_f2cmap'): +f2cmap_default = copy.deepcopy(f2cmap_all) + + +def load_f2cmap_file(f2cmap_file): + global f2cmap_all + + f2cmap_all = copy.deepcopy(f2cmap_default) + + if f2cmap_file is None: + # Default value + f2cmap_file = '.f2py_f2cmap' + if not os.path.isfile(f2cmap_file): + return + # User defined additions to f2cmap_all. - # .f2py_f2cmap must contain a dictionary of dictionaries, only. For + # f2cmap_file must contain a dictionary of dictionaries, only. For # example, {'real':{'low':'float'}} means that Fortran 'real(low)' is # interpreted as C 'float'. This feature is useful for F90/95 users if # they use PARAMETERSs in type specifications. try: - outmess('Reading .f2py_f2cmap ...\n') - f = open('.f2py_f2cmap', 'r') - d = eval(f.read(), {}, {}) - f.close() + outmess('Reading f2cmap from {!r} ...\n'.format(f2cmap_file)) + with open(f2cmap_file, 'r') as f: + d = eval(f.read(), {}, {}) for k, d1 in list(d.items()): for k1 in list(d1.keys()): d1[k1.lower()] = d1[k1] @@ -208,10 +220,10 @@ if os.path.isfile('.f2py_f2cmap'): else: errmess("\tIgnoring map {'%s':{'%s':'%s'}}: '%s' must be in %s\n" % ( k, k1, d[k][k1], d[k][k1], list(c2py_map.keys()))) - outmess('Successfully applied user defined changes from .f2py_f2cmap\n') + outmess('Successfully applied user defined f2cmap changes\n') except Exception as msg: errmess( - 'Failed to apply user defined changes from .f2py_f2cmap: %s. Skipping.\n' % (msg)) + 'Failed to apply user defined f2cmap changes: %s. Skipping.\n' % (msg)) cformat_map = {'double': '%g', 'float': '%g', diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index d59b6301c..ccb7b3a32 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -542,7 +542,7 @@ cppmacros[ 'ARRSIZE'] = '#define ARRSIZE(dims,rank) (_PyArray_multiply_list(dims,rank))' cppmacros['OLDPYNUM'] = """\ #ifdef OLDPYNUM -#error You need to install Numeric Python version 13 or higher. Get it from http:/sourceforge.net/project/?group_id=1369 +#error You need to install NumPy version 13 or higher. See https://scipy.org/install.html #endif """ ################# C functions ############### @@ -1049,8 +1049,10 @@ static int create_cb_arglist(PyObject* fun,PyTupleObject* xa,const int maxnofarg CFUNCSMESS(\"create_cb_arglist\\n\"); tot=opt=ext=siz=0; /* Get the total number of arguments */ - if (PyFunction_Check(fun)) + if (PyFunction_Check(fun)) { tmp_fun = fun; + Py_INCREF(tmp_fun); + } else { di = 1; if (PyObject_HasAttrString(fun,\"im_func\")) { @@ -1062,6 +1064,7 @@ static int create_cb_arglist(PyObject* fun,PyTupleObject* xa,const int maxnofarg tmp_fun = PyObject_GetAttrString(tmp,\"im_func\"); else { tmp_fun = fun; /* built-in function */ + Py_INCREF(tmp_fun); tot = maxnofargs; if (xa != NULL) tot += PyTuple_Size((PyObject *)xa); @@ -1073,6 +1076,7 @@ static int create_cb_arglist(PyObject* fun,PyTupleObject* xa,const int maxnofarg if (xa != NULL) tot += PyTuple_Size((PyObject *)xa); tmp_fun = fun; + Py_INCREF(tmp_fun); } else if (F2PyCapsule_Check(fun)) { tot = maxnofargs; @@ -1083,6 +1087,7 @@ static int create_cb_arglist(PyObject* fun,PyTupleObject* xa,const int maxnofarg goto capi_fail; } tmp_fun = fun; + Py_INCREF(tmp_fun); } } if (tmp_fun==NULL) { @@ -1091,13 +1096,19 @@ goto capi_fail; } #if PY_VERSION_HEX >= 0x03000000 if (PyObject_HasAttrString(tmp_fun,\"__code__\")) { - if (PyObject_HasAttrString(tmp = PyObject_GetAttrString(tmp_fun,\"__code__\"),\"co_argcount\")) + if (PyObject_HasAttrString(tmp = PyObject_GetAttrString(tmp_fun,\"__code__\"),\"co_argcount\")) { #else if (PyObject_HasAttrString(tmp_fun,\"func_code\")) { - if (PyObject_HasAttrString(tmp = PyObject_GetAttrString(tmp_fun,\"func_code\"),\"co_argcount\")) + if (PyObject_HasAttrString(tmp = PyObject_GetAttrString(tmp_fun,\"func_code\"),\"co_argcount\")) { #endif - tot = PyInt_AsLong(PyObject_GetAttrString(tmp,\"co_argcount\")) - di; - Py_XDECREF(tmp); + PyObject *tmp_argcount = PyObject_GetAttrString(tmp,\"co_argcount\"); + Py_DECREF(tmp); + if (tmp_argcount == NULL) { + goto capi_fail; + } + tot = PyInt_AsLong(tmp_argcount) - di; + Py_DECREF(tmp_argcount); + } } /* Get the number of optional arguments */ #if PY_VERSION_HEX >= 0x03000000 @@ -1136,10 +1147,12 @@ goto capi_fail; PyTuple_SET_ITEM(*args,i,tmp); } CFUNCSMESS(\"create_cb_arglist-end\\n\"); + Py_DECREF(tmp_fun); return 1; capi_fail: if ((PyErr_Occurred())==NULL) PyErr_SetString(#modulename#_error,errmess); + Py_XDECREF(tmp_fun); return 0; } """ diff --git a/numpy/f2py/common_rules.py b/numpy/f2py/common_rules.py index 62c1ba207..f61d8810a 100644 --- a/numpy/f2py/common_rules.py +++ b/numpy/f2py/common_rules.py @@ -124,8 +124,9 @@ def buildhooks(m): cadd('\t%s(f2pyinit%s,F2PYINIT%s)(f2py_setup_%s);' % (F_FUNC, lower_name, name.upper(), name)) cadd('}\n') - iadd('\tF2PyDict_SetItemString(d, \"%s\", PyFortranObject_New(f2py_%s_def,f2py_init_%s));' % ( - name, name, name)) + iadd('\ttmp = PyFortranObject_New(f2py_%s_def,f2py_init_%s);' % (name, name)) + iadd('\tF2PyDict_SetItemString(d, \"%s\", tmp);' % name) + iadd('\tPy_DECREF(tmp);') tname = name.replace('_', '\\_') dadd('\\subsection{Common block \\texttt{%s}}\n' % (tname)) dadd('\\begin{description}') diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 110337f92..d03eff9e3 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -28,6 +28,7 @@ from . import auxfuncs from . import cfuncs from . import f90mod_rules from . import __version__ +from . import capi_maps f2py_version = __version__.version errmess = sys.stderr.write @@ -118,6 +119,9 @@ Options: --link-<resource> switch below. [..] is optional list of resources names. E.g. try 'f2py --help-link lapack_opt'. + --f2cmap <filename> Load Fortran-to-Python KIND specification from the given + file. Default: .f2py_f2cmap in current directory. + --quiet Run quietly. --verbose Run with extra verbosity. -v Print f2py version ID and exit. @@ -175,7 +179,7 @@ http://cens.ioc.ee/projects/f2py2e/""" % (f2py_version, numpy_version) def scaninputline(inputline): files, skipfuncs, onlyfuncs, debug = [], [], [], [] - f, f2, f3, f5, f6, f7, f8, f9 = 1, 0, 0, 0, 0, 0, 0, 0 + f, f2, f3, f5, f6, f7, f8, f9, f10 = 1, 0, 0, 0, 0, 0, 0, 0, 0 verbose = 1 dolc = -1 dolatexdoc = 0 @@ -226,6 +230,8 @@ def scaninputline(inputline): f8 = 1 elif l == '--f2py-wrapper-output': f9 = 1 + elif l == '--f2cmap': + f10 = 1 elif l == '--overwrite-signature': options['h-overwrite'] = 1 elif l == '-h': @@ -267,6 +273,9 @@ def scaninputline(inputline): elif f9: f9 = 0 options["f2py_wrapper_output"] = l + elif f10: + f10 = 0 + options["f2cmap_file"] = l elif f == 1: try: with open(l): @@ -312,6 +321,7 @@ def scaninputline(inputline): options['wrapfuncs'] = wrapfuncs options['buildpath'] = buildpath options['include_paths'] = include_paths + options.setdefault('f2cmap_file', None) return files, options @@ -422,6 +432,7 @@ def run_main(comline_list): fobjcsrc = os.path.join(f2pydir, 'src', 'fortranobject.c') files, options = scaninputline(comline_list) auxfuncs.options = options + capi_maps.load_f2cmap_file(options['f2cmap_file']) postlist = callcrackfortran(files, options) isusedby = {} for i in range(len(postlist)): @@ -574,7 +585,7 @@ def run_compile(): modulename = 'untitled' sources = sys.argv[1:] - for optname in ['--include_paths', '--include-paths']: + for optname in ['--include_paths', '--include-paths', '--f2cmap']: if optname in sys.argv: i = sys.argv.index(optname) f2py_flags.extend(sys.argv[i:i + 2]) diff --git a/numpy/f2py/info.py b/numpy/f2py/info.py deleted file mode 100644 index c895c5de2..000000000 --- a/numpy/f2py/info.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Fortran to Python Interface Generator. - -""" -from __future__ import division, absolute_import, print_function - -postpone_import = True diff --git a/numpy/f2py/rules.py b/numpy/f2py/rules.py index 1b41498ea..f2f713bde 100755 --- a/numpy/f2py/rules.py +++ b/numpy/f2py/rules.py @@ -215,6 +215,7 @@ PyMODINIT_FUNC init#modulename#(void) { \td = PyModule_GetDict(m); \ts = PyString_FromString(\"$R""" + """evision: $\"); \tPyDict_SetItemString(d, \"__version__\", s); +\tPy_DECREF(s); #if PY_VERSION_HEX >= 0x03000000 \ts = PyUnicode_FromString( #else @@ -222,8 +223,14 @@ PyMODINIT_FUNC init#modulename#(void) { #endif \t\t\"This module '#modulename#' is auto-generated with f2py (version:#f2py_version#).\\nFunctions:\\n\"\n#docs#\".\"); \tPyDict_SetItemString(d, \"__doc__\", s); -\t#modulename#_error = PyErr_NewException (\"#modulename#.error\", NULL, NULL); \tPy_DECREF(s); +\t#modulename#_error = PyErr_NewException (\"#modulename#.error\", NULL, NULL); +\t/* +\t * Store the error object inside the dict, so that it could get deallocated. +\t * (in practice, this is a module, so it likely will not and cannot.) +\t */ +\tPyDict_SetItemString(d, \"_#modulename#_error\", #modulename#_error); +\tPy_DECREF(#modulename#_error); \tfor(i=0;f2py_routine_defs[i].name!=NULL;i++) { \t\ttmp = PyFortranObject_NewAsAttr(&f2py_routine_defs[i]); \t\tPyDict_SetItemString(d, f2py_routine_defs[i].name, tmp); @@ -238,7 +245,6 @@ PyMODINIT_FUNC init#modulename#(void) { \tif (! PyErr_Occurred()) \t\ton_exit(f2py_report_on_exit,(void*)\"#modulename#\"); #endif - \treturn RETVAL; } #ifdef __cplusplus @@ -439,12 +445,16 @@ rout_rules = [ { extern #ctype# #F_FUNC#(#name_lower#,#NAME#)(void); PyObject* o = PyDict_GetItemString(d,"#name#"); - PyObject_SetAttrString(o,"_cpointer", F2PyCapsule_FromVoidPtr((void*)#F_FUNC#(#name_lower#,#NAME#),NULL)); + tmp = F2PyCapsule_FromVoidPtr((void*)#F_FUNC#(#name_lower#,#NAME#),NULL); + PyObject_SetAttrString(o,"_cpointer", tmp); + Py_DECREF(tmp); #if PY_VERSION_HEX >= 0x03000000 - PyObject_SetAttrString(o,"__name__", PyUnicode_FromString("#name#")); + s = PyUnicode_FromString("#name#"); #else - PyObject_SetAttrString(o,"__name__", PyString_FromString("#name#")); + s = PyString_FromString("#name#"); #endif + PyObject_SetAttrString(o,"__name__", s); + Py_DECREF(s); } '''}, 'need': {l_not(l_or(ismoduleroutine, isdummyroutine)): ['F_WRAPPEDFUNC', 'F_FUNC']}, @@ -477,12 +487,16 @@ rout_rules = [ { extern void #F_FUNC#(#name_lower#,#NAME#)(void); PyObject* o = PyDict_GetItemString(d,"#name#"); - PyObject_SetAttrString(o,"_cpointer", F2PyCapsule_FromVoidPtr((void*)#F_FUNC#(#name_lower#,#NAME#),NULL)); + tmp = F2PyCapsule_FromVoidPtr((void*)#F_FUNC#(#name_lower#,#NAME#),NULL); + PyObject_SetAttrString(o,"_cpointer", tmp); + Py_DECREF(tmp); #if PY_VERSION_HEX >= 0x03000000 - PyObject_SetAttrString(o,"__name__", PyUnicode_FromString("#name#")); + s = PyUnicode_FromString("#name#"); #else - PyObject_SetAttrString(o,"__name__", PyString_FromString("#name#")); + s = PyString_FromString("#name#"); #endif + PyObject_SetAttrString(o,"__name__", s); + Py_DECREF(s); } '''}, 'need': {l_not(l_or(ismoduleroutine, isdummyroutine)): ['F_WRAPPEDFUNC', 'F_FUNC']}, @@ -794,10 +808,13 @@ if (#varname#_capi==Py_None) { if (#varname#_xa_capi==NULL) { if (PyObject_HasAttrString(#modulename#_module,\"#varname#_extra_args\")) { PyObject* capi_tmp = PyObject_GetAttrString(#modulename#_module,\"#varname#_extra_args\"); - if (capi_tmp) + if (capi_tmp) { #varname#_xa_capi = (PyTupleObject *)PySequence_Tuple(capi_tmp); - else + Py_DECREF(capi_tmp); + } + else { #varname#_xa_capi = (PyTupleObject *)Py_BuildValue(\"()\"); + } if (#varname#_xa_capi==NULL) { PyErr_SetString(#modulename#_error,\"Failed to convert #modulename#.#varname#_extra_args to tuple.\\n\"); return NULL; diff --git a/numpy/f2py/setup.py b/numpy/f2py/setup.py index c0c50ce54..a8c1401aa 100644 --- a/numpy/f2py/setup.py +++ b/numpy/f2py/setup.py @@ -3,7 +3,7 @@ setup.py for installing F2PY Usage: - python setup.py install + pip install . Copyright 2001-2005 Pearu Peterson all rights reserved, Pearu Peterson <pearu@cens.ioc.ee> diff --git a/numpy/f2py/src/fortranobject.c b/numpy/f2py/src/fortranobject.c index b55385b50..8aa55555d 100644 --- a/numpy/f2py/src/fortranobject.c +++ b/numpy/f2py/src/fortranobject.c @@ -39,19 +39,33 @@ PyFortranObject_New(FortranDataDef* defs, f2py_void_func init) { int i; PyFortranObject *fp = NULL; PyObject *v = NULL; - if (init!=NULL) /* Initialize F90 module objects */ + if (init!=NULL) { /* Initialize F90 module objects */ (*(init))(); - if ((fp = PyObject_New(PyFortranObject, &PyFortran_Type))==NULL) return NULL; - if ((fp->dict = PyDict_New())==NULL) return NULL; + } + fp = PyObject_New(PyFortranObject, &PyFortran_Type); + if (fp == NULL) { + return NULL; + } + if ((fp->dict = PyDict_New()) == NULL) { + Py_DECREF(fp); + return NULL; + } fp->len = 0; - while (defs[fp->len].name != NULL) fp->len++; - if (fp->len == 0) goto fail; + while (defs[fp->len].name != NULL) { + fp->len++; + } + if (fp->len == 0) { + goto fail; + } fp->defs = defs; - for (i=0;i<fp->len;i++) + for (i=0;i<fp->len;i++) { if (fp->defs[i].rank == -1) { /* Is Fortran routine */ v = PyFortranObject_NewAsAttr(&(fp->defs[i])); - if (v==NULL) return NULL; + if (v==NULL) { + goto fail; + } PyDict_SetItemString(fp->dict,fp->defs[i].name,v); + Py_XDECREF(v); } else if ((fp->defs[i].data)!=NULL) { /* Is Fortran variable or array (not allocatable) */ if (fp->defs[i].type == NPY_STRING) { @@ -65,13 +79,16 @@ PyFortranObject_New(FortranDataDef* defs, f2py_void_func init) { fp->defs[i].type, NULL, fp->defs[i].data, 0, NPY_ARRAY_FARRAY, NULL); } - if (v==NULL) return NULL; + if (v==NULL) { + goto fail; + } PyDict_SetItemString(fp->dict,fp->defs[i].name,v); + Py_XDECREF(v); } - Py_XDECREF(v); + } return (PyObject *)fp; fail: - Py_XDECREF(v); + Py_XDECREF(fp); return NULL; } diff --git a/numpy/f2py/src/test/foomodule.c b/numpy/f2py/src/test/foomodule.c index 733fab0be..caf3590d4 100644 --- a/numpy/f2py/src/test/foomodule.c +++ b/numpy/f2py/src/test/foomodule.c @@ -115,7 +115,7 @@ static PyMethodDef foo_module_methods[] = { void initfoo() { int i; - PyObject *m, *d, *s; + PyObject *m, *d, *s, *tmp; import_array(); m = Py_InitModule("foo", foo_module_methods); @@ -125,11 +125,17 @@ void initfoo() { PyDict_SetItemString(d, "__doc__", s); /* Fortran objects: */ - PyDict_SetItemString(d, "mod", PyFortranObject_New(f2py_mod_def,f2py_init_mod)); - PyDict_SetItemString(d, "foodata", PyFortranObject_New(f2py_foodata_def,f2py_init_foodata)); - for(i=0;f2py_routines_def[i].name!=NULL;i++) - PyDict_SetItemString(d, f2py_routines_def[i].name, - PyFortranObject_NewAsAttr(&f2py_routines_def[i])); + tmp = PyFortranObject_New(f2py_mod_def,f2py_init_mod); + PyDict_SetItemString(d, "mod", tmp); + Py_DECREF(tmp); + tmp = PyFortranObject_New(f2py_foodata_def,f2py_init_foodata); + PyDict_SetItemString(d, "foodata", tmp); + Py_DECREF(tmp); + for(i=0;f2py_routines_def[i].name!=NULL;i++) { + tmp = PyFortranObject_NewAsAttr(&f2py_routines_def[i]); + PyDict_SetItemString(d, f2py_routines_def[i].name, tmp); + Py_DECREF(tmp); + } Py_DECREF(s); diff --git a/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c b/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c index 7f46303b0..978db4e69 100644 --- a/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c +++ b/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c @@ -49,9 +49,18 @@ static PyObject *f2py_rout_wrap_call(PyObject *capi_self, return NULL; rank = PySequence_Length(dims_capi); dims = malloc(rank*sizeof(npy_intp)); - for (i=0;i<rank;++i) - dims[i] = (npy_intp)PyInt_AsLong(PySequence_GetItem(dims_capi,i)); - + for (i=0;i<rank;++i) { + PyObject *tmp; + tmp = PySequence_GetItem(dims_capi, i); + if (tmp == NULL) { + goto fail; + } + dims[i] = (npy_intp)PyInt_AsLong(tmp); + Py_DECREF(tmp); + if (dims[i] == -1 && PyErr_Occurred()) { + goto fail; + } + } capi_arr_tmp = array_from_pyobj(type_num,dims,rank,intent|F2PY_INTENT_OUT,arr_capi); if (capi_arr_tmp == NULL) { free(dims); @@ -60,6 +69,10 @@ static PyObject *f2py_rout_wrap_call(PyObject *capi_self, capi_buildvalue = Py_BuildValue("N",capi_arr_tmp); free(dims); return capi_buildvalue; + +fail: + free(dims); + return NULL; } static char doc_f2py_rout_wrap_attrs[] = "\ @@ -97,7 +110,7 @@ static PyObject *f2py_rout_wrap_attrs(PyObject *capi_self, PyTuple_SetItem(dimensions,i,PyInt_FromLong(PyArray_DIM(arr,i))); PyTuple_SetItem(strides,i,PyInt_FromLong(PyArray_STRIDE(arr,i))); } - return Py_BuildValue("siOOO(cciii)ii",s,PyArray_NDIM(arr), + return Py_BuildValue("siNNO(cciii)ii",s,PyArray_NDIM(arr), dimensions,strides, (PyArray_BASE(arr)==NULL?Py_None:PyArray_BASE(arr)), PyArray_DESCR(arr)->kind, @@ -154,61 +167,69 @@ PyMODINIT_FUNC inittest_array_from_pyobj_ext(void) { PyDict_SetItemString(d, "__doc__", s); wrap_error = PyErr_NewException ("wrap.error", NULL, NULL); Py_DECREF(s); - PyDict_SetItemString(d, "F2PY_INTENT_IN", PyInt_FromLong(F2PY_INTENT_IN)); - PyDict_SetItemString(d, "F2PY_INTENT_INOUT", PyInt_FromLong(F2PY_INTENT_INOUT)); - PyDict_SetItemString(d, "F2PY_INTENT_OUT", PyInt_FromLong(F2PY_INTENT_OUT)); - PyDict_SetItemString(d, "F2PY_INTENT_HIDE", PyInt_FromLong(F2PY_INTENT_HIDE)); - PyDict_SetItemString(d, "F2PY_INTENT_CACHE", PyInt_FromLong(F2PY_INTENT_CACHE)); - PyDict_SetItemString(d, "F2PY_INTENT_COPY", PyInt_FromLong(F2PY_INTENT_COPY)); - PyDict_SetItemString(d, "F2PY_INTENT_C", PyInt_FromLong(F2PY_INTENT_C)); - PyDict_SetItemString(d, "F2PY_OPTIONAL", PyInt_FromLong(F2PY_OPTIONAL)); - PyDict_SetItemString(d, "F2PY_INTENT_INPLACE", PyInt_FromLong(F2PY_INTENT_INPLACE)); - PyDict_SetItemString(d, "NPY_BOOL", PyInt_FromLong(NPY_BOOL)); - PyDict_SetItemString(d, "NPY_BYTE", PyInt_FromLong(NPY_BYTE)); - PyDict_SetItemString(d, "NPY_UBYTE", PyInt_FromLong(NPY_UBYTE)); - PyDict_SetItemString(d, "NPY_SHORT", PyInt_FromLong(NPY_SHORT)); - PyDict_SetItemString(d, "NPY_USHORT", PyInt_FromLong(NPY_USHORT)); - PyDict_SetItemString(d, "NPY_INT", PyInt_FromLong(NPY_INT)); - PyDict_SetItemString(d, "NPY_UINT", PyInt_FromLong(NPY_UINT)); - PyDict_SetItemString(d, "NPY_INTP", PyInt_FromLong(NPY_INTP)); - PyDict_SetItemString(d, "NPY_UINTP", PyInt_FromLong(NPY_UINTP)); - PyDict_SetItemString(d, "NPY_LONG", PyInt_FromLong(NPY_LONG)); - PyDict_SetItemString(d, "NPY_ULONG", PyInt_FromLong(NPY_ULONG)); - PyDict_SetItemString(d, "NPY_LONGLONG", PyInt_FromLong(NPY_LONGLONG)); - PyDict_SetItemString(d, "NPY_ULONGLONG", PyInt_FromLong(NPY_ULONGLONG)); - PyDict_SetItemString(d, "NPY_FLOAT", PyInt_FromLong(NPY_FLOAT)); - PyDict_SetItemString(d, "NPY_DOUBLE", PyInt_FromLong(NPY_DOUBLE)); - PyDict_SetItemString(d, "NPY_LONGDOUBLE", PyInt_FromLong(NPY_LONGDOUBLE)); - PyDict_SetItemString(d, "NPY_CFLOAT", PyInt_FromLong(NPY_CFLOAT)); - PyDict_SetItemString(d, "NPY_CDOUBLE", PyInt_FromLong(NPY_CDOUBLE)); - PyDict_SetItemString(d, "NPY_CLONGDOUBLE", PyInt_FromLong(NPY_CLONGDOUBLE)); - PyDict_SetItemString(d, "NPY_OBJECT", PyInt_FromLong(NPY_OBJECT)); - PyDict_SetItemString(d, "NPY_STRING", PyInt_FromLong(NPY_STRING)); - PyDict_SetItemString(d, "NPY_UNICODE", PyInt_FromLong(NPY_UNICODE)); - PyDict_SetItemString(d, "NPY_VOID", PyInt_FromLong(NPY_VOID)); - PyDict_SetItemString(d, "NPY_NTYPES", PyInt_FromLong(NPY_NTYPES)); - PyDict_SetItemString(d, "NPY_NOTYPE", PyInt_FromLong(NPY_NOTYPE)); - PyDict_SetItemString(d, "NPY_USERDEF", PyInt_FromLong(NPY_USERDEF)); - - PyDict_SetItemString(d, "CONTIGUOUS", PyInt_FromLong(NPY_ARRAY_C_CONTIGUOUS)); - PyDict_SetItemString(d, "FORTRAN", PyInt_FromLong(NPY_ARRAY_F_CONTIGUOUS)); - PyDict_SetItemString(d, "OWNDATA", PyInt_FromLong(NPY_ARRAY_OWNDATA)); - PyDict_SetItemString(d, "FORCECAST", PyInt_FromLong(NPY_ARRAY_FORCECAST)); - PyDict_SetItemString(d, "ENSURECOPY", PyInt_FromLong(NPY_ARRAY_ENSURECOPY)); - PyDict_SetItemString(d, "ENSUREARRAY", PyInt_FromLong(NPY_ARRAY_ENSUREARRAY)); - PyDict_SetItemString(d, "ALIGNED", PyInt_FromLong(NPY_ARRAY_ALIGNED)); - PyDict_SetItemString(d, "WRITEABLE", PyInt_FromLong(NPY_ARRAY_WRITEABLE)); - PyDict_SetItemString(d, "UPDATEIFCOPY", PyInt_FromLong(NPY_ARRAY_UPDATEIFCOPY)); - PyDict_SetItemString(d, "WRITEBACKIFCOPY", PyInt_FromLong(NPY_ARRAY_WRITEBACKIFCOPY)); - - PyDict_SetItemString(d, "BEHAVED", PyInt_FromLong(NPY_ARRAY_BEHAVED)); - PyDict_SetItemString(d, "BEHAVED_NS", PyInt_FromLong(NPY_ARRAY_BEHAVED_NS)); - PyDict_SetItemString(d, "CARRAY", PyInt_FromLong(NPY_ARRAY_CARRAY)); - PyDict_SetItemString(d, "FARRAY", PyInt_FromLong(NPY_ARRAY_FARRAY)); - PyDict_SetItemString(d, "CARRAY_RO", PyInt_FromLong(NPY_ARRAY_CARRAY_RO)); - PyDict_SetItemString(d, "FARRAY_RO", PyInt_FromLong(NPY_ARRAY_FARRAY_RO)); - PyDict_SetItemString(d, "DEFAULT", PyInt_FromLong(NPY_ARRAY_DEFAULT)); - PyDict_SetItemString(d, "UPDATE_ALL", PyInt_FromLong(NPY_ARRAY_UPDATE_ALL)); + +#define ADDCONST(NAME, CONST) \ + s = PyInt_FromLong(CONST); \ + PyDict_SetItemString(d, NAME, s); \ + Py_DECREF(s) + + ADDCONST("F2PY_INTENT_IN", F2PY_INTENT_IN); + ADDCONST("F2PY_INTENT_INOUT", F2PY_INTENT_INOUT); + ADDCONST("F2PY_INTENT_OUT", F2PY_INTENT_OUT); + ADDCONST("F2PY_INTENT_HIDE", F2PY_INTENT_HIDE); + ADDCONST("F2PY_INTENT_CACHE", F2PY_INTENT_CACHE); + ADDCONST("F2PY_INTENT_COPY", F2PY_INTENT_COPY); + ADDCONST("F2PY_INTENT_C", F2PY_INTENT_C); + ADDCONST("F2PY_OPTIONAL", F2PY_OPTIONAL); + ADDCONST("F2PY_INTENT_INPLACE", F2PY_INTENT_INPLACE); + ADDCONST("NPY_BOOL", NPY_BOOL); + ADDCONST("NPY_BYTE", NPY_BYTE); + ADDCONST("NPY_UBYTE", NPY_UBYTE); + ADDCONST("NPY_SHORT", NPY_SHORT); + ADDCONST("NPY_USHORT", NPY_USHORT); + ADDCONST("NPY_INT", NPY_INT); + ADDCONST("NPY_UINT", NPY_UINT); + ADDCONST("NPY_INTP", NPY_INTP); + ADDCONST("NPY_UINTP", NPY_UINTP); + ADDCONST("NPY_LONG", NPY_LONG); + ADDCONST("NPY_ULONG", NPY_ULONG); + ADDCONST("NPY_LONGLONG", NPY_LONGLONG); + ADDCONST("NPY_ULONGLONG", NPY_ULONGLONG); + ADDCONST("NPY_FLOAT", NPY_FLOAT); + ADDCONST("NPY_DOUBLE", NPY_DOUBLE); + ADDCONST("NPY_LONGDOUBLE", NPY_LONGDOUBLE); + ADDCONST("NPY_CFLOAT", NPY_CFLOAT); + ADDCONST("NPY_CDOUBLE", NPY_CDOUBLE); + ADDCONST("NPY_CLONGDOUBLE", NPY_CLONGDOUBLE); + ADDCONST("NPY_OBJECT", NPY_OBJECT); + ADDCONST("NPY_STRING", NPY_STRING); + ADDCONST("NPY_UNICODE", NPY_UNICODE); + ADDCONST("NPY_VOID", NPY_VOID); + ADDCONST("NPY_NTYPES", NPY_NTYPES); + ADDCONST("NPY_NOTYPE", NPY_NOTYPE); + ADDCONST("NPY_USERDEF", NPY_USERDEF); + + ADDCONST("CONTIGUOUS", NPY_ARRAY_C_CONTIGUOUS); + ADDCONST("FORTRAN", NPY_ARRAY_F_CONTIGUOUS); + ADDCONST("OWNDATA", NPY_ARRAY_OWNDATA); + ADDCONST("FORCECAST", NPY_ARRAY_FORCECAST); + ADDCONST("ENSURECOPY", NPY_ARRAY_ENSURECOPY); + ADDCONST("ENSUREARRAY", NPY_ARRAY_ENSUREARRAY); + ADDCONST("ALIGNED", NPY_ARRAY_ALIGNED); + ADDCONST("WRITEABLE", NPY_ARRAY_WRITEABLE); + ADDCONST("UPDATEIFCOPY", NPY_ARRAY_UPDATEIFCOPY); + ADDCONST("WRITEBACKIFCOPY", NPY_ARRAY_WRITEBACKIFCOPY); + + ADDCONST("BEHAVED", NPY_ARRAY_BEHAVED); + ADDCONST("BEHAVED_NS", NPY_ARRAY_BEHAVED_NS); + ADDCONST("CARRAY", NPY_ARRAY_CARRAY); + ADDCONST("FARRAY", NPY_ARRAY_FARRAY); + ADDCONST("CARRAY_RO", NPY_ARRAY_CARRAY_RO); + ADDCONST("FARRAY_RO", NPY_ARRAY_FARRAY_RO); + ADDCONST("DEFAULT", NPY_ARRAY_DEFAULT); + ADDCONST("UPDATE_ALL", NPY_ARRAY_UPDATE_ALL); + +#undef ADDCONST( if (PyErr_Occurred()) Py_FatalError("can't initialize module wrap"); diff --git a/numpy/f2py/tests/test_assumed_shape.py b/numpy/f2py/tests/test_assumed_shape.py index 460afd68d..e5695a61c 100644 --- a/numpy/f2py/tests/test_assumed_shape.py +++ b/numpy/f2py/tests/test_assumed_shape.py @@ -2,6 +2,7 @@ from __future__ import division, absolute_import, print_function import os import pytest +import tempfile from numpy.testing import assert_ from . import util @@ -16,6 +17,7 @@ class TestAssumedShapeSumExample(util.F2PyTest): _path('src', 'assumed_shape', 'foo_use.f90'), _path('src', 'assumed_shape', 'precision.f90'), _path('src', 'assumed_shape', 'foo_mod.f90'), + _path('src', 'assumed_shape', '.f2py_f2cmap'), ] @pytest.mark.slow @@ -31,3 +33,23 @@ class TestAssumedShapeSumExample(util.F2PyTest): assert_(r == 3, repr(r)) r = self.module.mod.fsum([1, 2]) assert_(r == 3, repr(r)) + + +class TestF2cmapOption(TestAssumedShapeSumExample): + def setup(self): + # Use a custom file name for .f2py_f2cmap + self.sources = list(self.sources) + f2cmap_src = self.sources.pop(-1) + + self.f2cmap_file = tempfile.NamedTemporaryFile(delete=False) + with open(f2cmap_src, 'rb') as f: + self.f2cmap_file.write(f.read()) + self.f2cmap_file.close() + + self.sources.append(self.f2cmap_file.name) + self.options = ["--f2cmap", self.f2cmap_file.name] + + super(TestF2cmapOption, self).setup() + + def teardown(self): + os.unlink(self.f2cmap_file.name) diff --git a/numpy/f2py/tests/test_compile_function.py b/numpy/f2py/tests/test_compile_function.py index 36abf05f9..40ea7997f 100644 --- a/numpy/f2py/tests/test_compile_function.py +++ b/numpy/f2py/tests/test_compile_function.py @@ -29,6 +29,7 @@ def setup_module(): @pytest.mark.parametrize( "extra_args", [['--noopt', '--debug'], '--noopt --debug', ''] ) +@pytest.mark.leaks_references(reason="Imported module seems never deleted.") def test_f2py_init_compile(extra_args): # flush through the f2py __init__ compile() function code path as a # crude test for input handling following migration from @@ -81,6 +82,9 @@ def test_f2py_init_compile(extra_args): return_check = import_module(modname) calc_result = return_check.foo() assert_equal(calc_result, 15) + # Removal from sys.modules, is not as such necessary. Even with + # removal, the module (dict) stays alive. + del sys.modules[modname] def test_f2py_init_compile_failure(): diff --git a/numpy/f2py/tests/util.py b/numpy/f2py/tests/util.py index d20dc5908..bf005df88 100644 --- a/numpy/f2py/tests/util.py +++ b/numpy/f2py/tests/util.py @@ -31,6 +31,7 @@ except ImportError: # _module_dir = None +_module_num = 5403 def _cleanup(): @@ -59,13 +60,14 @@ def get_module_dir(): def get_temp_module_name(): # Assume single-threaded, and the module dir usable only by this thread + global _module_num d = get_module_dir() - for j in range(5403, 9999999): - name = "_test_ext_module_%d" % j - fn = os.path.join(d, name) - if name not in sys.modules and not os.path.isfile(fn + '.py'): - return name - raise RuntimeError("Failed to create a temporary module name") + name = "_test_ext_module_%d" % _module_num + _module_num += 1 + if name in sys.modules: + # this should not be possible, but check anyway + raise RuntimeError("Temporary module name already in use.") + return name def _memoize(func): @@ -105,6 +107,7 @@ def build_module(source_files, options=[], skip=[], only=[], module_name=None): # Copy files dst_sources = [] + f2py_sources = [] for fn in source_files: if not os.path.isfile(fn): raise RuntimeError("%s is not a file" % fn) @@ -112,16 +115,14 @@ def build_module(source_files, options=[], skip=[], only=[], module_name=None): shutil.copyfile(fn, dst) dst_sources.append(dst) - fn = os.path.join(os.path.dirname(fn), '.f2py_f2cmap') - if os.path.isfile(fn): - dst = os.path.join(d, os.path.basename(fn)) - if not os.path.isfile(dst): - shutil.copyfile(fn, dst) + base, ext = os.path.splitext(dst) + if ext in ('.f90', '.f', '.c', '.pyf'): + f2py_sources.append(dst) # Prepare options if module_name is None: module_name = get_temp_module_name() - f2py_opts = ['-c', '-m', module_name] + options + dst_sources + f2py_opts = ['-c', '-m', module_name] + options + f2py_sources if skip: f2py_opts += ['skip:'] + skip if only: @@ -203,14 +204,20 @@ def _get_compiler_status(): """) code = code % dict(syspath=repr(sys.path)) - with temppath(suffix='.py') as script: + tmpdir = tempfile.mkdtemp() + try: + script = os.path.join(tmpdir, 'setup.py') + with open(script, 'w') as f: f.write(code) - cmd = [sys.executable, script, 'config'] + cmd = [sys.executable, 'setup.py', 'config'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + cwd=tmpdir) out, err = p.communicate() + finally: + shutil.rmtree(tmpdir) m = re.search(br'COMPILERS:(\d+),(\d+),(\d+)', out) if m: diff --git a/numpy/fft/__init__.py b/numpy/fft/__init__.py index fe95d8b17..37b3f0da6 100644 --- a/numpy/fft/__init__.py +++ b/numpy/fft/__init__.py @@ -118,8 +118,16 @@ The inverse DFT is defined as It differs from the forward transform by the sign of the exponential argument and the default normalization by :math:`1/n`. +Type Promotion +-------------- + +`numpy.fft` promotes ``float32`` and ``complex64`` arrays to ``float64`` and +``complex128`` arrays respectively. For an FFT implementation that does not +promote input arrays, see `scipy.fftpack`. + Normalization ------------- + The default normalization has the direct transforms unscaled and the inverse transforms are scaled by :math:`1/n`. It is possible to obtain unitary transforms by setting the keyword argument ``norm`` to ``"ortho"`` (default is diff --git a/numpy/fft/tests/test_pocketfft.py b/numpy/fft/tests/test_pocketfft.py index 7a06d4c45..453e964fa 100644 --- a/numpy/fft/tests/test_pocketfft.py +++ b/numpy/fft/tests/test_pocketfft.py @@ -45,12 +45,16 @@ class TestFFT1D(object): assert_allclose(fft1(x) / np.sqrt(30), np.fft.fft(x, norm="ortho"), atol=1e-6) - def test_ifft(self): + @pytest.mark.parametrize('norm', (None, 'ortho')) + def test_ifft(self, norm): x = random(30) + 1j*random(30) - assert_allclose(x, np.fft.ifft(np.fft.fft(x)), atol=1e-6) assert_allclose( - x, np.fft.ifft(np.fft.fft(x, norm="ortho"), norm="ortho"), + x, np.fft.ifft(np.fft.fft(x, norm=norm), norm=norm), atol=1e-6) + # Ensure we get the correct error message + with pytest.raises(ValueError, + match='Invalid number of FFT data points'): + np.fft.ifft([], norm=norm) def test_fft2(self): x = random((30, 20)) + 1j*random((30, 20)) diff --git a/numpy/lib/__init__.py b/numpy/lib/__init__.py index c1757150e..2db12d9a4 100644 --- a/numpy/lib/__init__.py +++ b/numpy/lib/__init__.py @@ -1,14 +1,31 @@ +""" +**Note:** almost all functions in the ``numpy.lib`` namespace +are also present in the main ``numpy`` namespace. Please use the +functions as ``np.<funcname>`` where possible. + +``numpy.lib`` is mostly a space for implementing functions that don't +belong in core or in another NumPy submodule with a clear purpose +(e.g. ``random``, ``fft``, ``linalg``, ``ma``). + +Most contains basic functions that are used by several submodules and are +useful to have in the main name-space. + +""" from __future__ import division, absolute_import, print_function import math -from .info import __doc__ from numpy.version import version as __version__ +# Public submodules +# Note: recfunctions and (maybe) format are public too, but not imported +from . import mixins +from . import scimath as emath + +# Private submodules from .type_check import * from .index_tricks import * from .function_base import * -from .mixins import * from .nanfunctions import * from .shape_base import * from .stride_tricks import * @@ -16,9 +33,7 @@ from .twodim_base import * from .ufunclike import * from .histograms import * -from . import scimath as emath from .polynomial import * -#import convertcode from .utils import * from .arraysetops import * from .npyio import * @@ -28,11 +43,10 @@ from .arraypad import * from ._version import * from numpy.core._multiarray_umath import tracemalloc_domain -__all__ = ['emath', 'math', 'tracemalloc_domain'] +__all__ = ['emath', 'math', 'tracemalloc_domain', 'Arrayterator'] __all__ += type_check.__all__ __all__ += index_tricks.__all__ __all__ += function_base.__all__ -__all__ += mixins.__all__ __all__ += shape_base.__all__ __all__ += stride_tricks.__all__ __all__ += twodim_base.__all__ diff --git a/numpy/lib/arraypad.py b/numpy/lib/arraypad.py index 600a32dde..33e64708d 100644 --- a/numpy/lib/arraypad.py +++ b/numpy/lib/arraypad.py @@ -271,6 +271,12 @@ def _get_stats(padded, axis, width_pair, length_pair, stat_func): if right_length is None or max_length < right_length: right_length = max_length + if (left_length == 0 or right_length == 0) \ + and stat_func in {np.amax, np.amin}: + # amax and amin can't operate on an emtpy array, + # raise a more descriptive warning here instead of the default one + raise ValueError("stat_length of 0 yields no value for padding") + # Calculate statistic for the left side left_slice = _slice_at_axis( slice(left_index, left_index + left_length), axis) @@ -288,6 +294,7 @@ def _get_stats(padded, axis, width_pair, length_pair, stat_func): right_chunk = padded[right_slice] right_stat = stat_func(right_chunk, axis=axis, keepdims=True) _round_if_needed(right_stat, padded.dtype) + return left_stat, right_stat @@ -783,7 +790,7 @@ def pad(array, pad_width, mode='constant', **kwargs): raise ValueError("unsupported keyword arguments for mode '{}': {}" .format(mode, unsupported_kwargs)) - stat_functions = {"maximum": np.max, "minimum": np.min, + stat_functions = {"maximum": np.amax, "minimum": np.amin, "mean": np.mean, "median": np.median} # Create array with final shape and original values diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index f3f4bc17e..2309f7e42 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -213,6 +213,7 @@ def unique(ar, return_index=False, return_inverse=False, ----- When an axis is specified the subarrays indexed by the axis are sorted. This is done by making the specified axis the first dimension of the array + (move the axis to the first dimension to keep the order of the other axes) and then flattening the subarrays in C order. The flattened subarrays are then viewed as a structured type with each element given a label, with the effect that we end up with a 1-D array of structured types that can be @@ -264,7 +265,7 @@ def unique(ar, return_index=False, return_inverse=False, # axis was specified and not None try: - ar = np.swapaxes(ar, axis, 0) + ar = np.moveaxis(ar, axis, 0) except np.AxisError: # this removes the "axis1" or "axis2" prefix from the error message raise np.AxisError(axis, ar.ndim) @@ -285,7 +286,7 @@ def unique(ar, return_index=False, return_inverse=False, def reshape_uniq(uniq): uniq = uniq.view(orig_dtype) uniq = uniq.reshape(-1, *orig_shape[1:]) - uniq = np.swapaxes(uniq, 0, axis) + uniq = np.moveaxis(uniq, 0, axis) return uniq output = _unique1d(consolidated, return_index, diff --git a/numpy/lib/financial.py b/numpy/lib/financial.py index 216687475..a011e52a9 100644 --- a/numpy/lib/financial.py +++ b/numpy/lib/financial.py @@ -12,6 +12,7 @@ otherwise stated. """ from __future__ import division, absolute_import, print_function +import warnings from decimal import Decimal import functools @@ -19,6 +20,10 @@ import numpy as np from numpy.core import overrides +_depmsg = ("numpy.{name} is deprecated and will be removed from NumPy 1.20. " + "Use numpy_financial.{name} instead " + "(https://pypi.org/project/numpy-financial/).") + array_function_dispatch = functools.partial( overrides.array_function_dispatch, module='numpy') @@ -45,6 +50,8 @@ def _convert_when(when): def _fv_dispatcher(rate, nper, pmt, pv, when=None): + warnings.warn(_depmsg.format(name='fv'), + DeprecationWarning, stacklevel=3) return (rate, nper, pmt, pv) @@ -53,6 +60,12 @@ def fv(rate, nper, pmt, pv, when='end'): """ Compute the future value. + .. deprecated:: 1.18 + + `fv` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Given: * a present value, `pv` * an interest `rate` compounded once per period, of which @@ -100,7 +113,9 @@ def fv(rate, nper, pmt, pv, when='end'): References ---------- - .. [WRW] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). Open Document Format for Office Applications (OpenDocument)v1.2, Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, Pre-Draft 12. Organization for the Advancement of Structured Information @@ -109,6 +124,7 @@ def fv(rate, nper, pmt, pv, when='end'): http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula OpenDocument-formula-20090508.odt + Examples -------- What is the future value after 10 years of saving $100 now, with @@ -139,6 +155,8 @@ def fv(rate, nper, pmt, pv, when='end'): def _pmt_dispatcher(rate, nper, pv, fv=None, when=None): + warnings.warn(_depmsg.format(name='pmt'), + DeprecationWarning, stacklevel=3) return (rate, nper, pv, fv) @@ -147,6 +165,12 @@ def pmt(rate, nper, pv, fv=0, when='end'): """ Compute the payment against loan principal plus interest. + .. deprecated:: 1.18 + + `pmt` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Given: * a present value, `pv` (e.g., an amount borrowed) * a future value, `fv` (e.g., 0) @@ -204,7 +228,9 @@ def pmt(rate, nper, pv, fv=0, when='end'): References ---------- - .. [WRW] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). Open Document Format for Office Applications (OpenDocument)v1.2, Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, Pre-Draft 12. Organization for the Advancement of Structured Information @@ -237,6 +263,8 @@ def pmt(rate, nper, pv, fv=0, when='end'): def _nper_dispatcher(rate, pmt, pv, fv=None, when=None): + warnings.warn(_depmsg.format(name='nper'), + DeprecationWarning, stacklevel=3) return (rate, pmt, pv, fv) @@ -245,6 +273,12 @@ def nper(rate, pmt, pv, fv=0, when='end'): """ Compute the number of periodic payments. + .. deprecated:: 1.18 + + `nper` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + :class:`decimal.Decimal` type is not supported. Parameters @@ -270,6 +304,11 @@ def nper(rate, pmt, pv, fv=0, when='end'): fv + pv + pmt*nper = 0 + References + ---------- + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + Examples -------- If you only had $150/month to pay towards the loan, how long would it take @@ -311,6 +350,8 @@ def nper(rate, pmt, pv, fv=0, when='end'): def _ipmt_dispatcher(rate, per, nper, pv, fv=None, when=None): + warnings.warn(_depmsg.format(name='ipmt'), + DeprecationWarning, stacklevel=3) return (rate, per, nper, pv, fv) @@ -319,6 +360,12 @@ def ipmt(rate, per, nper, pv, fv=0, when='end'): """ Compute the interest portion of a payment. + .. deprecated:: 1.18 + + `ipmt` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Parameters ---------- rate : scalar or array_like of shape(M, ) @@ -354,6 +401,11 @@ def ipmt(rate, per, nper, pv, fv=0, when='end'): ``pmt = ppmt + ipmt`` + References + ---------- + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + Examples -------- What is the amortization schedule for a 1 year loan of $2500 at @@ -422,6 +474,8 @@ def _rbl(rate, per, pmt, pv, when): def _ppmt_dispatcher(rate, per, nper, pv, fv=None, when=None): + warnings.warn(_depmsg.format(name='ppmt'), + DeprecationWarning, stacklevel=3) return (rate, per, nper, pv, fv) @@ -430,6 +484,12 @@ def ppmt(rate, per, nper, pv, fv=0, when='end'): """ Compute the payment against loan principal. + .. deprecated:: 1.18 + + `ppmt` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Parameters ---------- rate : array_like @@ -450,12 +510,19 @@ def ppmt(rate, per, nper, pv, fv=0, when='end'): -------- pmt, pv, ipmt + References + ---------- + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + """ total = pmt(rate, nper, pv, fv, when) return total - ipmt(rate, per, nper, pv, fv, when) def _pv_dispatcher(rate, nper, pmt, fv=None, when=None): + warnings.warn(_depmsg.format(name='pv'), + DeprecationWarning, stacklevel=3) return (rate, nper, nper, pv, fv) @@ -464,6 +531,12 @@ def pv(rate, nper, pmt, fv=0, when='end'): """ Compute the present value. + .. deprecated:: 1.18 + + `pv` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Given: * a future value, `fv` * an interest `rate` compounded once per period, of which @@ -510,7 +583,9 @@ def pv(rate, nper, pmt, fv=0, when='end'): References ---------- - .. [WRW] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). Open Document Format for Office Applications (OpenDocument)v1.2, Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, Pre-Draft 12. Organization for the Advancement of Structured Information @@ -567,6 +642,8 @@ def _g_div_gp(r, n, p, x, y, w): def _rate_dispatcher(nper, pmt, pv, fv, when=None, guess=None, tol=None, maxiter=None): + warnings.warn(_depmsg.format(name='rate'), + DeprecationWarning, stacklevel=3) return (nper, pmt, pv, fv) @@ -582,6 +659,12 @@ def rate(nper, pmt, pv, fv, when='end', guess=None, tol=None, maxiter=100): """ Compute the rate of interest per period. + .. deprecated:: 1.18 + + `rate` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Parameters ---------- nper : array_like @@ -612,13 +695,16 @@ def rate(nper, pmt, pv, fv, when='end', guess=None, tol=None, maxiter=100): References ---------- - Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). Open Document - Format for Office Applications (OpenDocument)v1.2, Part 2: Recalculated - Formula (OpenFormula) Format - Annotated Version, Pre-Draft 12. - Organization for the Advancement of Structured Information Standards - (OASIS). Billerica, MA, USA. [ODT Document]. Available: - http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula - OpenDocument-formula-20090508.odt + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). + Open Document Format for Office Applications (OpenDocument)v1.2, + Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, + Pre-Draft 12. Organization for the Advancement of Structured Information + Standards (OASIS). Billerica, MA, USA. [ODT Document]. + Available: + http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula + OpenDocument-formula-20090508.odt """ when = _convert_when(when) @@ -651,6 +737,8 @@ def rate(nper, pmt, pv, fv, when='end', guess=None, tol=None, maxiter=100): def _irr_dispatcher(values): + warnings.warn(_depmsg.format(name='irr'), + DeprecationWarning, stacklevel=3) return (values,) @@ -659,6 +747,12 @@ def irr(values): """ Return the Internal Rate of Return (IRR). + .. deprecated:: 1.18 + + `irr` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + This is the "average" periodically compounded rate of return that gives a net present value of 0.0; for a more complete explanation, see Notes below. @@ -693,13 +787,15 @@ def irr(values): + \\frac{55}{(1+r)^3} + \\frac{20}{(1+r)^4} = 0 In general, for `values` :math:`= [v_0, v_1, ... v_M]`, - irr is the solution of the equation: [G]_ + irr is the solution of the equation: [2]_ .. math:: \\sum_{t=0}^M{\\frac{v_t}{(1+irr)^{t}}} = 0 References ---------- - .. [G] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., Addison-Wesley, 2003, pg. 348. Examples @@ -715,8 +811,6 @@ def irr(values): >>> round(np.irr([-5, 10.5, 1, -8, 1]), 5) 0.0886 - (Compare with the Example given for numpy.lib.financial.npv) - """ # `np.roots` call is why this function does not support Decimal type. # @@ -736,6 +830,8 @@ def irr(values): def _npv_dispatcher(rate, values): + warnings.warn(_depmsg.format(name='npv'), + DeprecationWarning, stacklevel=3) return (values,) @@ -744,6 +840,12 @@ def npv(rate, values): """ Returns the NPV (Net Present Value) of a cash flow series. + .. deprecated:: 1.18 + + `npv` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Parameters ---------- rate : scalar @@ -763,23 +865,48 @@ def npv(rate, values): The NPV of the input cash flow series `values` at the discount `rate`. + Warnings + -------- + ``npv`` considers a series of cashflows starting in the present (t = 0). + NPV can also be defined with a series of future cashflows, paid at the + end, rather than the start, of each period. If future cashflows are used, + the first cashflow `values[0]` must be zeroed and added to the net + present value of the future cashflows. This is demonstrated in the + examples. + Notes ----- - Returns the result of: [G]_ + Returns the result of: [2]_ .. math :: \\sum_{t=0}^{M-1}{\\frac{values_t}{(1+rate)^{t}}} References ---------- - .. [G] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html + .. [2] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., Addison-Wesley, 2003, pg. 346. Examples -------- - >>> np.npv(0.281,[-100, 39, 59, 55, 20]) - -0.0084785916384548798 # may vary + Consider a potential project with an initial investment of $40 000 and + projected cashflows of $5 000, $8 000, $12 000 and $30 000 at the end of + each period discounted at a rate of 8% per period. To find the project's + net present value: + + >>> rate, cashflows = 0.08, [-40_000, 5_000, 8_000, 12_000, 30_000] + >>> np.npv(rate, cashflows).round(5) + 3065.22267 - (Compare with the Example given for numpy.lib.financial.irr) + It may be preferable to split the projected cashflow into an initial + investment and expected future cashflows. In this case, the value of + the initial cashflow is zero and the initial investment is later added + to the future cashflows net present value: + + >>> initial_cashflow = cashflows[0] + >>> cashflows[0] = 0 + >>> np.round(np.npv(rate, cashflows) + initial_cashflow, 5) + 3065.22267 """ values = np.asarray(values) @@ -787,6 +914,8 @@ def npv(rate, values): def _mirr_dispatcher(values, finance_rate, reinvest_rate): + warnings.warn(_depmsg.format(name='mirr'), + DeprecationWarning, stacklevel=3) return (values,) @@ -795,6 +924,12 @@ def mirr(values, finance_rate, reinvest_rate): """ Modified internal rate of return. + .. deprecated:: 1.18 + + `mirr` is deprecated; for details, see NEP 32 [1]_. + Use the corresponding function in the numpy-financial library, + https://pypi.org/project/numpy-financial. + Parameters ---------- values : array_like @@ -811,6 +946,10 @@ def mirr(values, finance_rate, reinvest_rate): out : float Modified internal rate of return + References + ---------- + .. [1] NumPy Enhancement Proposal (NEP) 32, + https://numpy.org/neps/nep-0032-remove-financial-functions.html """ values = np.asarray(values) n = values.size diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 93bdbce97..20e2e9c72 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -173,6 +173,9 @@ from numpy.compat import ( ) +__all__ = [] + + MAGIC_PREFIX = b'\x93NUMPY' MAGIC_LEN = len(MAGIC_PREFIX) + 2 ARRAY_ALIGN = 64 # plausible values are powers of 2 between 16 and 4096 @@ -239,6 +242,16 @@ def read_magic(fp): major, minor = magic_str[-2:] return major, minor +def _has_metadata(dt): + if dt.metadata is not None: + return True + elif dt.names is not None: + return any(_has_metadata(dt[k]) for k in dt.names) + elif dt.subdtype is not None: + return _has_metadata(dt.base) + else: + return False + def dtype_to_descr(dtype): """ Get a serializable descriptor from the dtype. @@ -262,6 +275,10 @@ def dtype_to_descr(dtype): replicate the input dtype. """ + if _has_metadata(dtype): + warnings.warn("metadata on a dtype may be saved or ignored, but will " + "raise if saved when read. Use another form of storage.", + UserWarning, stacklevel=2) if dtype.names is not None: # This is a record array. The .descr is fine. XXX: parts of the # record array with an empty name, like padding bytes, still get @@ -384,7 +401,7 @@ def _wrap_header_guess_version(header): return ret header = _wrap_header(header, (3, 0)) - warnings.warn("Stored array in format 3.0. It can only be" + warnings.warn("Stored array in format 3.0. It can only be " "read by NumPy >= 1.17", UserWarning, stacklevel=2) return header diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index cf7246402..499120630 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -316,14 +316,17 @@ def average(a, axis=None, weights=None, returned=False): The weights array can either be 1-D (in which case its length must be the size of `a` along the given axis) or of the same shape as `a`. If `weights=None`, then all data in `a` are assumed to have a - weight equal to one. + weight equal to one. The 1-D calculation is:: + + avg = sum(a * weights) / sum(weights) + + The only constraint on `weights` is that `sum(weights)` must not be 0. returned : bool, optional Default is `False`. If `True`, the tuple (`average`, `sum_of_weights`) is returned, otherwise only the average is returned. If `weights=None`, `sum_of_weights` is equivalent to the number of elements over which the average is taken. - Returns ------- retval, [sum_of_weights] : array_type or double @@ -358,12 +361,12 @@ def average(a, axis=None, weights=None, returned=False): Examples -------- - >>> data = list(range(1,5)) + >>> data = np.arange(1, 5) >>> data - [1, 2, 3, 4] + array([1, 2, 3, 4]) >>> np.average(data) 2.5 - >>> np.average(range(1,11), weights=range(10,0,-1)) + >>> np.average(np.arange(1, 11), weights=np.arange(10, 0, -1)) 4.0 >>> data = np.arange(6).reshape((3,2)) @@ -679,11 +682,7 @@ def select(condlist, choicelist, default=0): # Now that the dtype is known, handle the deprecated select([], []) case if len(condlist) == 0: - # 2014-02-24, 1.9 - warnings.warn("select with an empty condition list is not possible" - "and will be deprecated", - DeprecationWarning, stacklevel=3) - return np.asarray(default)[()] + raise ValueError("select with an empty condition list is not possible") choicelist = [np.asarray(choice) for choice in choicelist] choicelist.append(np.asarray(default)) @@ -699,25 +698,11 @@ def select(condlist, choicelist, default=0): choicelist = np.broadcast_arrays(*choicelist) # If cond array is not an ndarray in boolean format or scalar bool, abort. - deprecated_ints = False for i in range(len(condlist)): cond = condlist[i] if cond.dtype.type is not np.bool_: - if np.issubdtype(cond.dtype, np.integer): - # A previous implementation accepted int ndarrays accidentally. - # Supported here deliberately, but deprecated. - condlist[i] = condlist[i].astype(bool) - deprecated_ints = True - else: - raise ValueError( - 'invalid entry {} in condlist: should be boolean ndarray'.format(i)) - - if deprecated_ints: - # 2014-02-24, 1.9 - msg = "select condlists containing integer ndarrays is deprecated " \ - "and will be removed in the future. Use `.astype(bool)` to " \ - "convert to bools." - warnings.warn(msg, DeprecationWarning, stacklevel=3) + raise TypeError( + 'invalid entry {} in condlist: should be boolean ndarray'.format(i)) if choicelist[0].ndim == 0: # This may be common, so avoid the call. @@ -1164,11 +1149,13 @@ def diff(a, n=1, axis=-1, prepend=np._NoValue, append=np._NoValue): The axis along which the difference is taken, default is the last axis. prepend, append : array_like, optional - Values to prepend or append to "a" along axis prior to + Values to prepend or append to `a` along axis prior to performing the difference. Scalar values are expanded to arrays with length 1 in the direction of axis and the shape of the input array in along all other axes. Otherwise the - dimension and shape must match "a" except along axis. + dimension and shape must match `a` except along axis. + + .. versionadded:: 1.16.0 Returns ------- @@ -1327,9 +1314,13 @@ def interp(x, xp, fp, left=None, right=None, period=None): Notes ----- - Does not check that the x-coordinate sequence `xp` is increasing. - If `xp` is not increasing, the results are nonsense. - A simple check for increasing is:: + The x-coordinate sequence is expected to be increasing, but this is not + explicitly enforced. However, if the sequence `xp` is non-increasing, + interpolation results are meaningless. + + Note that, since NaN is unsortable, `xp` also cannot contain NaNs. + + A simple check for `xp` being strictly increasing is:: np.all(np.diff(xp) > 0) @@ -1902,7 +1893,7 @@ class vectorize(object): typecode characters or a list of data type specifiers. There should be one data type specifier for each output. doc : str, optional - The docstring for the function. If `None`, the docstring will be the + The docstring for the function. If None, the docstring will be the ``pyfunc.__doc__``. excluded : set, optional Set of strings or integers representing the positional or keyword @@ -2314,7 +2305,7 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None, >>> m = np.arange(10, dtype=np.float64) >>> f = np.arange(10) * 2 >>> a = np.arange(10) ** 2. - >>> ddof = 9 # N - 1 + >>> ddof = 1 >>> w = f * a >>> v1 = np.sum(w) >>> v2 = np.sum(w * a) @@ -3310,13 +3301,6 @@ def sinc(x): Text(0.5, 0, 'X') >>> plt.show() - It works in 2-D as well: - - >>> x = np.linspace(-4, 4, 401) - >>> xx = np.outer(x, x) - >>> plt.imshow(np.sinc(xx)) - <matplotlib.image.AxesImage object at 0x...> - """ x = np.asanyarray(x) y = pi * where(x == 0, 1.0e-20, x) diff --git a/numpy/lib/info.py b/numpy/lib/info.py deleted file mode 100644 index 8815a52f0..000000000 --- a/numpy/lib/info.py +++ /dev/null @@ -1,160 +0,0 @@ -""" -Basic functions used by several sub-packages and -useful to have in the main name-space. - -Type Handling -------------- -================ =================== -iscomplexobj Test for complex object, scalar result -isrealobj Test for real object, scalar result -iscomplex Test for complex elements, array result -isreal Test for real elements, array result -imag Imaginary part -real Real part -real_if_close Turns complex number with tiny imaginary part to real -isneginf Tests for negative infinity, array result -isposinf Tests for positive infinity, array result -isnan Tests for nans, array result -isinf Tests for infinity, array result -isfinite Tests for finite numbers, array result -isscalar True if argument is a scalar -nan_to_num Replaces NaN's with 0 and infinities with large numbers -cast Dictionary of functions to force cast to each type -common_type Determine the minimum common type code for a group - of arrays -mintypecode Return minimal allowed common typecode. -================ =================== - -Index Tricks ------------- -================ =================== -mgrid Method which allows easy construction of N-d - 'mesh-grids' -``r_`` Append and construct arrays: turns slice objects into - ranges and concatenates them, for 2d arrays appends rows. -index_exp Konrad Hinsen's index_expression class instance which - can be useful for building complicated slicing syntax. -================ =================== - -Useful Functions ----------------- -================ =================== -select Extension of where to multiple conditions and choices -extract Extract 1d array from flattened array according to mask -insert Insert 1d array of values into Nd array according to mask -linspace Evenly spaced samples in linear space -logspace Evenly spaced samples in logarithmic space -fix Round x to nearest integer towards zero -mod Modulo mod(x,y) = x % y except keeps sign of y -amax Array maximum along axis -amin Array minimum along axis -ptp Array max-min along axis -cumsum Cumulative sum along axis -prod Product of elements along axis -cumprod Cumluative product along axis -diff Discrete differences along axis -angle Returns angle of complex argument -unwrap Unwrap phase along given axis (1-d algorithm) -sort_complex Sort a complex-array (based on real, then imaginary) -trim_zeros Trim the leading and trailing zeros from 1D array. -vectorize A class that wraps a Python function taking scalar - arguments into a generalized function which can handle - arrays of arguments using the broadcast rules of - numerix Python. -================ =================== - -Shape Manipulation ------------------- -================ =================== -squeeze Return a with length-one dimensions removed. -atleast_1d Force arrays to be >= 1D -atleast_2d Force arrays to be >= 2D -atleast_3d Force arrays to be >= 3D -vstack Stack arrays vertically (row on row) -hstack Stack arrays horizontally (column on column) -column_stack Stack 1D arrays as columns into 2D array -dstack Stack arrays depthwise (along third dimension) -stack Stack arrays along a new axis -split Divide array into a list of sub-arrays -hsplit Split into columns -vsplit Split into rows -dsplit Split along third dimension -================ =================== - -Matrix (2D Array) Manipulations -------------------------------- -================ =================== -fliplr 2D array with columns flipped -flipud 2D array with rows flipped -rot90 Rotate a 2D array a multiple of 90 degrees -eye Return a 2D array with ones down a given diagonal -diag Construct a 2D array from a vector, or return a given - diagonal from a 2D array. -mat Construct a Matrix -bmat Build a Matrix from blocks -================ =================== - -Polynomials ------------ -================ =================== -poly1d A one-dimensional polynomial class -poly Return polynomial coefficients from roots -roots Find roots of polynomial given coefficients -polyint Integrate polynomial -polyder Differentiate polynomial -polyadd Add polynomials -polysub Subtract polynomials -polymul Multiply polynomials -polydiv Divide polynomials -polyval Evaluate polynomial at given argument -================ =================== - -Iterators ---------- -================ =================== -Arrayterator A buffered iterator for big arrays. -================ =================== - -Import Tricks -------------- -================ =================== -ppimport Postpone module import until trying to use it -ppimport_attr Postpone module import until trying to use its attribute -ppresolve Import postponed module and return it. -================ =================== - -Machine Arithmetics -------------------- -================ =================== -machar_single Single precision floating point arithmetic parameters -machar_double Double precision floating point arithmetic parameters -================ =================== - -Threading Tricks ----------------- -================ =================== -ParallelExec Execute commands in parallel thread. -================ =================== - -Array Set Operations ------------------------ -Set operations for numeric arrays based on sort() function. - -================ =================== -unique Unique elements of an array. -isin Test whether each element of an ND array is present - anywhere within a second array. -ediff1d Array difference (auxiliary function). -intersect1d Intersection of 1D arrays with unique elements. -setxor1d Set exclusive-or of 1D arrays with unique elements. -in1d Test whether elements in a 1D array are also present in - another array. -union1d Union of 1D arrays with unique elements. -setdiff1d Set difference of 1D arrays with unique elements. -================ =================== - -""" -from __future__ import division, absolute_import, print_function - -depends = ['core', 'testing'] -global_symbols = ['*'] diff --git a/numpy/lib/mixins.py b/numpy/lib/mixins.py index 52ad45b68..f974a7724 100644 --- a/numpy/lib/mixins.py +++ b/numpy/lib/mixins.py @@ -5,8 +5,8 @@ import sys from numpy.core import umath as um -# Nothing should be exposed in the top-level NumPy module. -__all__ = [] + +__all__ = ['NDArrayOperatorsMixin'] def _disables_array_ufunc(obj): diff --git a/numpy/lib/nanfunctions.py b/numpy/lib/nanfunctions.py index 9a03d0b39..8e2a34e70 100644 --- a/numpy/lib/nanfunctions.py +++ b/numpy/lib/nanfunctions.py @@ -95,17 +95,18 @@ def _replace_nan(a, val): NaNs, otherwise return None. """ - a = np.array(a, subok=True, copy=True) + a = np.asanyarray(a) if a.dtype == np.object_: # object arrays do not support `isnan` (gh-9009), so make a guess - mask = a != a + mask = np.not_equal(a, a, dtype=bool) elif issubclass(a.dtype.type, np.inexact): mask = np.isnan(a) else: mask = None if mask is not None: + a = np.array(a, subok=True, copy=True) np.copyto(a, val, where=mask) return a, mask @@ -244,8 +245,8 @@ def nanmin(a, axis=None, out=None, keepdims=np._NoValue): out : ndarray, optional Alternate output array in which to place the result. The default is ``None``; if provided, it must have the same shape as the - expected output, but the type will be cast if necessary. See - `doc.ufuncs` for details. + expected output, but the type will be cast if necessary. See + `ufuncs-output-type` for more details. .. versionadded:: 1.8.0 keepdims : bool, optional @@ -359,8 +360,8 @@ def nanmax(a, axis=None, out=None, keepdims=np._NoValue): out : ndarray, optional Alternate output array in which to place the result. The default is ``None``; if provided, it must have the same shape as the - expected output, but the type will be cast if necessary. See - `doc.ufuncs` for details. + expected output, but the type will be cast if necessary. See + `ufuncs-output-type` for more details. .. versionadded:: 1.8.0 keepdims : bool, optional @@ -585,8 +586,8 @@ def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): Alternate output array in which to place the result. The default is ``None``. If provided, it must have the same shape as the expected output, but the type will be cast if necessary. See - `doc.ufuncs` for details. The casting of NaN to integer can yield - unexpected results. + `ufuncs-output-type` for more details. The casting of NaN to integer + can yield unexpected results. .. versionadded:: 1.8.0 keepdims : bool, optional @@ -681,9 +682,9 @@ def nanprod(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): out : ndarray, optional Alternate output array in which to place the result. The default is ``None``. If provided, it must have the same shape as the - expected output, but the type will be cast if necessary. See - `doc.ufuncs` for details. The casting of NaN to integer can yield - unexpected results. + expected output, but the type will be cast if necessary. See + `ufuncs-output-type` for more details. The casting of NaN to integer + can yield unexpected results. keepdims : bool, optional If True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will @@ -750,8 +751,8 @@ def nancumsum(a, axis=None, dtype=None, out=None): out : ndarray, optional Alternative output array in which to place the result. It must have the same shape and buffer length as the expected output - but the type will be cast if necessary. See `doc.ufuncs` - (Section "Output arguments") for more details. + but the type will be cast if necessary. See `ufuncs-output-type` for + more details. Returns ------- @@ -888,8 +889,8 @@ def nanmean(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): out : ndarray, optional Alternate output array in which to place the result. The default is ``None``; if provided, it must have the same shape as the - expected output, but the type will be cast if necessary. See - `doc.ufuncs` for details. + expected output, but the type will be cast if necessary. See + `ufuncs-output-type` for more details. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -1443,7 +1444,7 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): the variance of the flattened array. dtype : data-type, optional Type to use in computing the variance. For arrays of integer type - the default is `float32`; for arrays of float types it is the same as + the default is `float64`; for arrays of float types it is the same as the array type. out : ndarray, optional Alternate output array in which to place the result. It must have @@ -1473,7 +1474,7 @@ def nanvar(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): mean : Average var : Variance while not ignoring NaNs nanstd, nanmean - numpy.doc.ufuncs : Section "Output arguments" + ufuncs-output-type Notes ----- @@ -1625,7 +1626,7 @@ def nanstd(a, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): -------- var, mean, std nanvar, nanmean - numpy.doc.ufuncs : Section "Output arguments" + ufuncs-output-type Notes ----- diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 633ef9ca2..3e54ff10c 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -480,7 +480,7 @@ def save(file, arr, allow_pickle=True, fix_imports=True): file : file, str, or pathlib.Path File or filename to which the data is saved. If file is a file-object, then the filename is unchanged. If file is a string or Path, a ``.npy`` - extension will be appended to the file name if it does not already + extension will be appended to the filename if it does not already have one. arr : array_like Array data to be saved. @@ -507,6 +507,8 @@ def save(file, arr, allow_pickle=True, fix_imports=True): ----- For a description of the ``.npy`` format, see :py:mod:`numpy.lib.format`. + Any data saved to the file is appended to the end of the file. + Examples -------- >>> from tempfile import TemporaryFile @@ -519,6 +521,15 @@ def save(file, arr, allow_pickle=True, fix_imports=True): >>> np.load(outfile) array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + + >>> with open('test.npy', 'wb') as f: + ... np.save(f, np.array([1, 2])) + ... np.save(f, np.array([1, 3])) + >>> with open('test.npy', 'rb') as f: + ... a = np.load(f) + ... b = np.load(f) + >>> print(a, b) + # [1 2] [1 3] """ own_fid = False if hasattr(file, 'write'): @@ -554,8 +565,7 @@ def _savez_dispatcher(file, *args, **kwds): @array_function_dispatch(_savez_dispatcher) def savez(file, *args, **kwds): - """ - Save several arrays into a single file in uncompressed ``.npz`` format. + """Save several arrays into a single file in uncompressed ``.npz`` format. If arguments are passed in with no keywords, the corresponding variable names, in the ``.npz`` file, are 'arr_0', 'arr_1', etc. If keyword @@ -565,9 +575,9 @@ def savez(file, *args, **kwds): Parameters ---------- file : str or file - Either the file name (string) or an open file (file-like object) + Either the filename (string) or an open file (file-like object) where the data will be saved. If file is a string or a Path, the - ``.npz`` extension will be appended to the file name if it is not + ``.npz`` extension will be appended to the filename if it is not already there. args : Arguments, optional Arrays to save to the file. Since it is not possible for Python to @@ -600,6 +610,10 @@ def savez(file, *args, **kwds): its list of arrays (with the ``.files`` attribute), and for the arrays themselves. + When saving dictionaries, the dictionary keys become filenames + inside the ZIP archive. Therefore, keys should be valid filenames. + E.g., avoid keys that begin with ``/`` or contain ``.``. + Examples -------- >>> from tempfile import TemporaryFile @@ -627,7 +641,6 @@ def savez(file, *args, **kwds): ['x', 'y'] >>> npzfile['x'] array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - """ _savez(file, args, kwds, False) @@ -645,15 +658,15 @@ def savez_compressed(file, *args, **kwds): Save several arrays into a single file in compressed ``.npz`` format. If keyword arguments are given, then filenames are taken from the keywords. - If arguments are passed in with no keywords, then stored file names are + If arguments are passed in with no keywords, then stored filenames are arr_0, arr_1, etc. Parameters ---------- file : str or file - Either the file name (string) or an open file (file-like object) + Either the filename (string) or an open file (file-like object) where the data will be saved. If file is a string or a Path, the - ``.npz`` extension will be appended to the file name if it is not + ``.npz`` extension will be appended to the filename if it is not already there. args : Arguments, optional Arrays to save to the file. Since it is not possible for Python to @@ -820,7 +833,7 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, fname : file, str, or pathlib.Path File, filename, or generator to read. If the filename extension is ``.gz`` or ``.bz2``, the file is first decompressed. Note that - generators should return byte strings for Python 3k. + generators should return byte strings. dtype : data-type, optional Data-type of the resulting array; default: float. If this is a structured data-type, the resulting array will be 1-dimensional, and @@ -1458,7 +1471,7 @@ def fromregex(file, regexp, dtype, encoding=None): Parameters ---------- file : str or file - File name or file object to read. + Filename or file object to read. regexp : str or regexp Regular expression used to parse the file. Groups in the regular expression correspond to fields in the dtype. @@ -1516,9 +1529,9 @@ def fromregex(file, regexp, dtype, encoding=None): dtype = np.dtype(dtype) content = file.read() - if isinstance(content, bytes) and isinstance(regexp, np.unicode): + if isinstance(content, bytes) and isinstance(regexp, np.compat.unicode): regexp = asbytes(regexp) - elif isinstance(content, np.unicode) and isinstance(regexp, bytes): + elif isinstance(content, np.compat.unicode) and isinstance(regexp, bytes): regexp = asstr(regexp) if not hasattr(regexp, 'match'): @@ -1565,7 +1578,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, fname : file, str, pathlib.Path, list of str, generator File, filename, list, or generator to read. If the filename extension is `.gz` or `.bz2`, the file is first decompressed. Note - that generators must return byte strings in Python 3k. The strings + that generators must return byte strings. The strings in a list or produced by a generator are treated as lines. dtype : dtype, optional Data type of the resulting array. @@ -1776,12 +1789,13 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, replace_space=replace_space) # Skip the first `skip_header` rows - for i in range(skip_header): - next(fhd) - - # Keep on until we find the first valid values - first_values = None try: + for i in range(skip_header): + next(fhd) + + # Keep on until we find the first valid values + first_values = None + while not first_values: first_line = _decode_line(next(fhd), encoding) if (names is True) and (comments is not None): diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index 2c72f623c..3d07a0de4 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -479,10 +479,10 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None, cov=False): coefficients for `k`-th data set are in ``p[:,k]``. residuals, rank, singular_values, rcond - Present only if `full` = True. Residuals of the least-squares fit, - the effective rank of the scaled Vandermonde coefficient matrix, - its singular values, and the specified value of `rcond`. For more - details, see `linalg.lstsq`. + Present only if `full` = True. Residuals is sum of squared residuals + of the least-squares fit, the effective rank of the scaled Vandermonde + coefficient matrix, its singular values, and the specified value of + `rcond`. For more details, see `linalg.lstsq`. V : ndarray, shape (M,M) or (M,M,K) Present only if `full` = False and `cov`=True. The covariance diff --git a/numpy/lib/recfunctions.py b/numpy/lib/recfunctions.py index 40060b41a..927161ddb 100644 --- a/numpy/lib/recfunctions.py +++ b/numpy/lib/recfunctions.py @@ -200,7 +200,7 @@ def flatten_descr(ndtype): descr = [] for field in names: (typ, _) = ndtype.fields[field] - if typ.names: + if typ.names is not None: descr.extend(flatten_descr(typ)) else: descr.append((field, typ)) @@ -527,6 +527,10 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): Nested fields are supported. + ..versionchanged: 1.18.0 + `drop_fields` returns an array with 0 fields if all fields are dropped, + rather than returning ``None`` as it did previously. + Parameters ---------- base : array @@ -566,7 +570,7 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): current = ndtype[name] if name in drop_names: continue - if current.names: + if current.names is not None: descr = _drop_descr(current, drop_names) if descr: newdtype.append((name, descr)) @@ -575,8 +579,6 @@ def drop_fields(base, drop_names, usemask=True, asrecarray=False): return newdtype newdtype = _drop_descr(base.dtype, drop_names) - if not newdtype: - return None output = np.empty(base.shape, dtype=newdtype) output = recursive_fill_fields(base, output) diff --git a/numpy/lib/shape_base.py b/numpy/lib/shape_base.py index a5d0040aa..dbb61c225 100644 --- a/numpy/lib/shape_base.py +++ b/numpy/lib/shape_base.py @@ -1,7 +1,6 @@ from __future__ import division, absolute_import, print_function import functools -import warnings import numpy.core.numeric as _nx from numpy.core.numeric import ( @@ -11,6 +10,7 @@ from numpy.core.fromnumeric import reshape, transpose from numpy.core.multiarray import normalize_axis_index from numpy.core import overrides from numpy.core import vstack, atleast_3d +from numpy.core.numeric import normalize_axis_tuple from numpy.core.shape_base import _arrays_for_stack_dispatcher from numpy.lib.index_tricks import ndindex from numpy.matrixlib.defmatrix import matrix # this raises all the right alarm bells @@ -29,7 +29,7 @@ array_function_dispatch = functools.partial( def _make_along_axis_idx(arr_shape, indices, axis): - # compute dimensions to iterate over + # compute dimensions to iterate over if not _nx.issubdtype(indices.dtype, _nx.integer): raise IndexError('`indices` must be an integer array') if len(arr_shape) != indices.ndim: @@ -517,22 +517,26 @@ def expand_dims(a, axis): Insert a new axis that will appear at the `axis` position in the expanded array shape. - .. note:: Previous to NumPy 1.13.0, neither ``axis < -a.ndim - 1`` nor - ``axis > a.ndim`` raised errors or put the new axis where documented. - Those axis values are now deprecated and will raise an AxisError in the - future. - Parameters ---------- a : array_like Input array. - axis : int - Position in the expanded axes where the new axis is placed. + axis : int or tuple of ints + Position in the expanded axes where the new axis (or axes) is placed. + + .. deprecated:: 1.13.0 + Passing an axis where ``axis > a.ndim`` will be treated as + ``axis == a.ndim``, and passing ``axis < -a.ndim - 1`` will + be treated as ``axis == 0``. This behavior is deprecated. + + .. versionchanged:: 1.18.0 + A tuple of axes is now supported. Out of range axes as + described above are now forbidden and raise an `AxisError`. Returns ------- - res : ndarray - View of `a` with the number of dimensions increased by one. + result : ndarray + View of `a` with the number of dimensions increased. See Also -------- @@ -542,11 +546,11 @@ def expand_dims(a, axis): Examples -------- - >>> x = np.array([1,2]) + >>> x = np.array([1, 2]) >>> x.shape (2,) - The following is equivalent to ``x[np.newaxis,:]`` or ``x[np.newaxis]``: + The following is equivalent to ``x[np.newaxis, :]`` or ``x[np.newaxis]``: >>> y = np.expand_dims(x, axis=0) >>> y @@ -554,13 +558,26 @@ def expand_dims(a, axis): >>> y.shape (1, 2) - >>> y = np.expand_dims(x, axis=1) # Equivalent to x[:,np.newaxis] + The following is equivalent to ``x[:, np.newaxis]``: + + >>> y = np.expand_dims(x, axis=1) >>> y array([[1], [2]]) >>> y.shape (2, 1) + ``axis`` may also be a tuple: + + >>> y = np.expand_dims(x, axis=(0, 1)) + >>> y + array([[[1, 2]]]) + + >>> y = np.expand_dims(x, axis=(2, 0)) + >>> y + array([[[1], + [2]]]) + Note that some examples may use ``None`` instead of ``np.newaxis``. These are the same objects: @@ -573,18 +590,16 @@ def expand_dims(a, axis): else: a = asanyarray(a) - shape = a.shape - if axis > a.ndim or axis < -a.ndim - 1: - # 2017-05-17, 1.13.0 - warnings.warn("Both axis > a.ndim and axis < -a.ndim - 1 are " - "deprecated and will raise an AxisError in the future.", - DeprecationWarning, stacklevel=3) - # When the deprecation period expires, delete this if block, - if axis < 0: - axis = axis + a.ndim + 1 - # and uncomment the following line. - # axis = normalize_axis_index(axis, a.ndim + 1) - return a.reshape(shape[:axis] + (1,) + shape[axis:]) + if type(axis) not in (tuple, list): + axis = (axis,) + + out_ndim = len(axis) + a.ndim + axis = normalize_axis_tuple(axis, out_ndim) + + shape_it = iter(a.shape) + shape = [1 if ax in axis else next(shape_it) for ax in range(out_ndim)] + + return a.reshape(shape) row_stack = vstack @@ -782,7 +797,7 @@ def _split_dispatcher(ary, indices_or_sections, axis=None): @array_function_dispatch(_split_dispatcher) def split(ary, indices_or_sections, axis=0): """ - Split an array into multiple sub-arrays. + Split an array into multiple sub-arrays as views into `ary`. Parameters ---------- @@ -809,7 +824,7 @@ def split(ary, indices_or_sections, axis=0): Returns ------- sub-arrays : list of ndarrays - A list of sub-arrays. + A list of sub-arrays as views into `ary`. Raises ------ @@ -854,8 +869,7 @@ def split(ary, indices_or_sections, axis=0): if N % sections: raise ValueError( 'array split does not result in an equal division') - res = array_split(ary, indices_or_sections, axis) - return res + return array_split(ary, indices_or_sections, axis) def _hvdsplit_dispatcher(ary, indices_or_sections): diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index e04fdc808..15cd3ad9d 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -204,14 +204,18 @@ class TestStringConverter(object): def test_upgrademapper(self): "Tests updatemapper" dateparser = _bytes_to_date - StringConverter.upgrade_mapper(dateparser, date(2000, 1, 1)) - convert = StringConverter(dateparser, date(2000, 1, 1)) - test = convert('2001-01-01') - assert_equal(test, date(2001, 1, 1)) - test = convert('2009-01-01') - assert_equal(test, date(2009, 1, 1)) - test = convert('') - assert_equal(test, date(2000, 1, 1)) + _original_mapper = StringConverter._mapper[:] + try: + StringConverter.upgrade_mapper(dateparser, date(2000, 1, 1)) + convert = StringConverter(dateparser, date(2000, 1, 1)) + test = convert('2001-01-01') + assert_equal(test, date(2001, 1, 1)) + test = convert('2009-01-01') + assert_equal(test, date(2009, 1, 1)) + test = convert('') + assert_equal(test, date(2000, 1, 1)) + finally: + StringConverter._mapper = _original_mapper def test_string_to_object(self): "Make sure that string-to-object functions are properly recognized" diff --git a/numpy/lib/tests/test_arraypad.py b/numpy/lib/tests/test_arraypad.py index 4e1f3bcaa..65593dd29 100644 --- a/numpy/lib/tests/test_arraypad.py +++ b/numpy/lib/tests/test_arraypad.py @@ -474,6 +474,29 @@ class TestStatistic(object): ) assert_array_equal(a, b) + @pytest.mark.filterwarnings("ignore:Mean of empty slice:RuntimeWarning") + @pytest.mark.filterwarnings( + "ignore:invalid value encountered in (true_divide|double_scalars):" + "RuntimeWarning" + ) + @pytest.mark.parametrize("mode", ["mean", "median"]) + def test_zero_stat_length_valid(self, mode): + arr = np.pad([1., 2.], (1, 2), mode, stat_length=0) + expected = np.array([np.nan, 1., 2., np.nan, np.nan]) + assert_equal(arr, expected) + + @pytest.mark.parametrize("mode", ["minimum", "maximum"]) + def test_zero_stat_length_invalid(self, mode): + match = "stat_length of 0 yields no value for padding" + with pytest.raises(ValueError, match=match): + np.pad([1., 2.], 0, mode, stat_length=0) + with pytest.raises(ValueError, match=match): + np.pad([1., 2.], 0, mode, stat_length=(1, 0)) + with pytest.raises(ValueError, match=match): + np.pad([1., 2.], 1, mode, stat_length=0) + with pytest.raises(ValueError, match=match): + np.pad([1., 2.], 1, mode, stat_length=(1, 0)) + class TestConstant(object): def test_check_constant(self): diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index 93d4b279f..fd21a7f76 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -422,41 +422,41 @@ class TestUnique(object): assert_array_equal(v, b, msg) msg = base_msg.format('return_index', dt) - v, j = unique(a, 1, 0, 0) + v, j = unique(a, True, False, False) assert_array_equal(v, b, msg) assert_array_equal(j, i1, msg) msg = base_msg.format('return_inverse', dt) - v, j = unique(a, 0, 1, 0) + v, j = unique(a, False, True, False) assert_array_equal(v, b, msg) assert_array_equal(j, i2, msg) msg = base_msg.format('return_counts', dt) - v, j = unique(a, 0, 0, 1) + v, j = unique(a, False, False, True) assert_array_equal(v, b, msg) assert_array_equal(j, c, msg) msg = base_msg.format('return_index and return_inverse', dt) - v, j1, j2 = unique(a, 1, 1, 0) + v, j1, j2 = unique(a, True, True, False) assert_array_equal(v, b, msg) assert_array_equal(j1, i1, msg) assert_array_equal(j2, i2, msg) msg = base_msg.format('return_index and return_counts', dt) - v, j1, j2 = unique(a, 1, 0, 1) + v, j1, j2 = unique(a, True, False, True) assert_array_equal(v, b, msg) assert_array_equal(j1, i1, msg) assert_array_equal(j2, c, msg) msg = base_msg.format('return_inverse and return_counts', dt) - v, j1, j2 = unique(a, 0, 1, 1) + v, j1, j2 = unique(a, False, True, True) assert_array_equal(v, b, msg) assert_array_equal(j1, i2, msg) assert_array_equal(j2, c, msg) msg = base_msg.format(('return_index, return_inverse ' 'and return_counts'), dt) - v, j1, j2, j3 = unique(a, 1, 1, 1) + v, j1, j2, j3 = unique(a, True, True, True) assert_array_equal(v, b, msg) assert_array_equal(j1, i1, msg) assert_array_equal(j2, i2, msg) @@ -600,8 +600,11 @@ class TestUnique(object): assert_array_equal(unique(data, axis=1), result.astype(dtype), msg) msg = 'Unique with 3d array and axis=2 failed' - data3d = np.dstack([data] * 3) - result = data3d[..., :1] + data3d = np.array([[[1, 1], + [1, 0]], + [[0, 1], + [0, 0]]]).astype(dtype) + result = np.take(data3d, [1, 0], axis=2) assert_array_equal(unique(data3d, axis=2), result, msg) uniq, idx, inv, cnt = unique(data, axis=0, return_index=True, diff --git a/numpy/lib/tests/test_financial.py b/numpy/lib/tests/test_financial.py index 524915041..cb67f7c0f 100644 --- a/numpy/lib/tests/test_financial.py +++ b/numpy/lib/tests/test_financial.py @@ -1,5 +1,6 @@ from __future__ import division, absolute_import, print_function +import warnings from decimal import Decimal import numpy as np @@ -8,16 +9,35 @@ from numpy.testing import ( ) +def filter_deprecation(func): + def newfunc(*args, **kwargs): + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings('always', category=DeprecationWarning) + func(*args, **kwargs) + assert_(all(w.category is DeprecationWarning for w in ws)) + return newfunc + + class TestFinancial(object): + @filter_deprecation + def test_npv_irr_congruence(self): + # IRR is defined as the rate required for the present value of a + # a series of cashflows to be zero i.e. NPV(IRR(x), x) = 0 + cashflows = np.array([-40000, 5000, 8000, 12000, 30000]) + assert_allclose(np.npv(np.irr(cashflows), cashflows), 0, atol=1e-10, rtol=0) + + @filter_deprecation def test_rate(self): assert_almost_equal( np.rate(10, 0, -3500, 10000), 0.1107, 4) + @filter_deprecation def test_rate_decimal(self): rate = np.rate(Decimal('10'), Decimal('0'), Decimal('-3500'), Decimal('10000')) assert_equal(Decimal('0.1106908537142689284704528100'), rate) + @filter_deprecation def test_irr(self): v = [-150000, 15000, 25000, 35000, 45000, 60000] assert_almost_equal(np.irr(v), 0.0524, 2) @@ -37,20 +57,25 @@ class TestFinancial(object): v = [-1, -2, -3] assert_equal(np.irr(v), np.nan) + @filter_deprecation def test_pv(self): assert_almost_equal(np.pv(0.07, 20, 12000, 0), -127128.17, 2) + @filter_deprecation def test_pv_decimal(self): assert_equal(np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0')), Decimal('-127128.1709461939327295222005')) + @filter_deprecation def test_fv(self): assert_equal(np.fv(0.075, 20, -2000, 0, 0), 86609.362673042924) + @filter_deprecation def test_fv_decimal(self): assert_equal(np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), 0, 0), Decimal('86609.36267304300040536731624')) + @filter_deprecation def test_pmt(self): res = np.pmt(0.08 / 12, 5 * 12, 15000) tgt = -304.145914 @@ -65,6 +90,7 @@ class TestFinancial(object): tgt = np.array([[-166.66667, -19311.258], [-626.90814, -19311.258]]) assert_allclose(res, tgt) + @filter_deprecation def test_pmt_decimal(self): res = np.pmt(Decimal('0.08') / Decimal('12'), 5 * 12, 15000) tgt = Decimal('-304.1459143262052370338701494') @@ -88,18 +114,22 @@ class TestFinancial(object): assert_equal(res[1][0], tgt[1][0]) assert_equal(res[1][1], tgt[1][1]) + @filter_deprecation def test_ppmt(self): assert_equal(np.round(np.ppmt(0.1 / 12, 1, 60, 55000), 2), -710.25) + @filter_deprecation def test_ppmt_decimal(self): assert_equal(np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000')), Decimal('-710.2541257864217612489830917')) # Two tests showing how Decimal is actually getting at a more exact result # .23 / 12 does not come out nicely as a float but does as a decimal + @filter_deprecation def test_ppmt_special_rate(self): assert_equal(np.round(np.ppmt(0.23 / 12, 1, 60, 10000000000), 8), -90238044.232277036) + @filter_deprecation def test_ppmt_special_rate_decimal(self): # When rounded out to 8 decimal places like the float based test, this should not equal the same value # as the float, substituted for the decimal @@ -112,31 +142,38 @@ class TestFinancial(object): assert_equal(np.ppmt(Decimal('0.23') / Decimal('12'), 1, 60, Decimal('10000000000')), Decimal('-90238044.2322778884413969909')) + @filter_deprecation def test_ipmt(self): assert_almost_equal(np.round(np.ipmt(0.1 / 12, 1, 24, 2000), 2), -16.67) + @filter_deprecation def test_ipmt_decimal(self): result = np.ipmt(Decimal('0.1') / Decimal('12'), 1, 24, 2000) assert_equal(result.flat[0], Decimal('-16.66666666666666666666666667')) + @filter_deprecation def test_nper(self): assert_almost_equal(np.nper(0.075, -2000, 0, 100000.), 21.54, 2) + @filter_deprecation def test_nper2(self): assert_almost_equal(np.nper(0.0, -2000, 0, 100000.), 50.0, 1) + @filter_deprecation def test_npv(self): assert_almost_equal( np.npv(0.05, [-15000, 1500, 2500, 3500, 4500, 6000]), 122.89, 2) + @filter_deprecation def test_npv_decimal(self): assert_equal( np.npv(Decimal('0.05'), [-15000, 1500, 2500, 3500, 4500, 6000]), Decimal('122.894854950942692161628715')) + @filter_deprecation def test_mirr(self): val = [-4500, -800, 800, 800, 600, 600, 800, 800, 700, 3000] assert_almost_equal(np.mirr(val, 0.08, 0.055), 0.0666, 4) @@ -150,6 +187,7 @@ class TestFinancial(object): val = [39000, 30000, 21000, 37000, 46000] assert_(np.isnan(np.mirr(val, 0.10, 0.12))) + @filter_deprecation def test_mirr_decimal(self): val = [Decimal('-4500'), Decimal('-800'), Decimal('800'), Decimal('800'), Decimal('600'), Decimal('600'), Decimal('800'), Decimal('800'), @@ -168,6 +206,7 @@ class TestFinancial(object): val = [Decimal('39000'), Decimal('30000'), Decimal('21000'), Decimal('37000'), Decimal('46000')] assert_(np.isnan(np.mirr(val, Decimal('0.10'), Decimal('0.12')))) + @filter_deprecation def test_when(self): # begin assert_equal(np.rate(10, 20, -3500, 10000, 1), @@ -232,6 +271,7 @@ class TestFinancial(object): assert_equal(np.nper(0.075, -2000, 0, 100000., 0), np.nper(0.075, -2000, 0, 100000., 'end')) + @filter_deprecation def test_decimal_with_when(self): """Test that decimals are still supported if the when argument is passed""" # begin @@ -306,6 +346,7 @@ class TestFinancial(object): np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), Decimal('0'), 'end').flat[0]) + @filter_deprecation def test_broadcast(self): assert_almost_equal(np.nper(0.075, -2000, 0, 100000., [0, 1]), [21.5449442, 20.76156441], 4) @@ -323,6 +364,7 @@ class TestFinancial(object): [-74.998201, -75.62318601, -75.62318601, -76.88882405, -76.88882405], 4) + @filter_deprecation def test_broadcast_decimal(self): # Use almost equal because precision is tested in the explicit tests, this test is to ensure # broadcast with Decimal is not broken. diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 062c21725..0592e0b12 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -963,3 +963,33 @@ def test_unicode_field_names(): with open(fname, 'wb') as f: with assert_warns(UserWarning): format.write_array(f, arr, version=None) + + +@pytest.mark.parametrize('dt, fail', [ + (np.dtype({'names': ['a', 'b'], 'formats': [float, np.dtype('S3', + metadata={'some': 'stuff'})]}), True), + (np.dtype(int, metadata={'some': 'stuff'}), False), + (np.dtype([('subarray', (int, (2,)))], metadata={'some': 'stuff'}), False), + # recursive: metadata on the field of a dtype + (np.dtype({'names': ['a', 'b'], 'formats': [ + float, np.dtype({'names': ['c'], 'formats': [np.dtype(int, metadata={})]}) + ]}), False) + ]) +def test_metadata_dtype(dt, fail): + # gh-14142 + arr = np.ones(10, dtype=dt) + buf = BytesIO() + with assert_warns(UserWarning): + np.save(buf, arr) + buf.seek(0) + if fail: + with assert_raises(ValueError): + np.load(buf) + else: + arr2 = np.load(buf) + # BUG: assert_array_equal does not check metadata + from numpy.lib.format import _has_metadata + assert_array_equal(arr, arr2) + assert _has_metadata(arr.dtype) + assert not _has_metadata(arr2.dtype) + diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 3061d3a84..9075ff538 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -423,27 +423,17 @@ class TestSelect(object): assert_equal(select([m], [d]), [0, 0, 0, np.nan, 0, 0]) def test_deprecated_empty(self): - with warnings.catch_warnings(record=True): - warnings.simplefilter("always") - assert_equal(select([], [], 3j), 3j) - - with warnings.catch_warnings(): - warnings.simplefilter("always") - assert_warns(DeprecationWarning, select, [], []) - warnings.simplefilter("error") - assert_raises(DeprecationWarning, select, [], []) + assert_raises(ValueError, select, [], [], 3j) + assert_raises(ValueError, select, [], []) def test_non_bool_deprecation(self): choices = self.choices conditions = self.conditions[:] - with warnings.catch_warnings(): - warnings.filterwarnings("always") - conditions[0] = conditions[0].astype(np.int_) - assert_warns(DeprecationWarning, select, conditions, choices) - conditions[0] = conditions[0].astype(np.uint8) - assert_warns(DeprecationWarning, select, conditions, choices) - warnings.filterwarnings("error") - assert_raises(DeprecationWarning, select, conditions, choices) + conditions[0] = conditions[0].astype(np.int_) + assert_raises(TypeError, select, conditions, choices) + conditions[0] = conditions[0].astype(np.uint8) + assert_raises(TypeError, select, conditions, choices) + assert_raises(TypeError, select, conditions, choices) def test_many_arguments(self): # This used to be limited by NPY_MAXARGS == 32 @@ -1105,7 +1095,7 @@ class TestAngle(object): np.arctan(3.0 / 1.0), np.arctan(1.0), 0, np.pi / 2, np.pi, -np.pi / 2.0, -np.arctan(3.0 / 1.0), np.pi - np.arctan(3.0 / 1.0)] - z = angle(x, deg=1) + z = angle(x, deg=True) zo = np.array(yo) * 180 / np.pi assert_array_almost_equal(y, yo, 11) assert_array_almost_equal(z, zo, 11) @@ -1920,9 +1910,9 @@ class TestCov(object): [-np.inf, np.inf]])) def test_1D_rowvar(self): - assert_allclose(cov(self.x3), cov(self.x3, rowvar=0)) + assert_allclose(cov(self.x3), cov(self.x3, rowvar=False)) y = np.array([0.0780, 0.3107, 0.2111, 0.0334, 0.8501]) - assert_allclose(cov(self.x3, y), cov(self.x3, y, rowvar=0)) + assert_allclose(cov(self.x3, y), cov(self.x3, y, rowvar=False)) def test_1D_variance(self): assert_allclose(cov(self.x3, ddof=1), np.var(self.x3, ddof=1)) @@ -2533,7 +2523,7 @@ class TestPercentile(object): assert_equal(np.percentile(x, 0, interpolation='nearest'), np.nan) def test_fraction(self): - x = [Fraction(i, 2) for i in np.arange(8)] + x = [Fraction(i, 2) for i in range(8)] p = np.percentile(x, Fraction(0)) assert_equal(p, Fraction(0)) @@ -2953,7 +2943,7 @@ class TestQuantile(object): def test_fraction(self): # fractional input, integral quantile - x = [Fraction(i, 2) for i in np.arange(8)] + x = [Fraction(i, 2) for i in range(8)] q = np.quantile(x, 0) assert_equal(q, 0) diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py index 2f7e97831..dbe445c2c 100644 --- a/numpy/lib/tests/test_index_tricks.py +++ b/numpy/lib/tests/test_index_tricks.py @@ -175,6 +175,24 @@ class TestRavelUnravelIndex(object): assert_raises_regex( ValueError, "out of bounds", np.unravel_index, [1], ()) + @pytest.mark.parametrize("mode", ["clip", "wrap", "raise"]) + def test_empty_array_ravel(self, mode): + res = np.ravel_multi_index( + np.zeros((3, 0), dtype=np.intp), (2, 1, 0), mode=mode) + assert(res.shape == (0,)) + + with assert_raises(ValueError): + np.ravel_multi_index( + np.zeros((3, 1), dtype=np.intp), (2, 1, 0), mode=mode) + + def test_empty_array_unravel(self): + res = np.unravel_index(np.zeros(0, dtype=np.intp), (2, 1, 0)) + # res is a tuple of three empty arrays + assert(len(res) == 3) + assert(all(a.shape == (0,) for a in res)) + + with assert_raises(ValueError): + np.unravel_index([1], (2, 1, 0)) class TestGrid(object): def test_basic(self): @@ -190,7 +208,7 @@ class TestGrid(object): assert_almost_equal(a[1]-a[0], 2.0/9.0, 11) def test_linspace_equivalence(self): - y, st = np.linspace(2, 10, retstep=1) + y, st = np.linspace(2, 10, retstep=True) assert_almost_equal(st, 8/49.0) assert_array_almost_equal(y, mgrid[2:10:50j], 13) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index c2d3365a3..a095e250a 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -24,6 +24,7 @@ from numpy.testing import ( assert_allclose, assert_array_equal, temppath, tempdir, IS_PYPY, HAS_REFCOUNT, suppress_warnings, assert_no_gc_cycles, assert_no_warnings ) +from numpy.testing._private.utils import requires_memory class TextIO(BytesIO): @@ -518,7 +519,7 @@ class TestSaveTxt(object): def test_unicode(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) with tempdir() as tmpdir: # set encoding as on windows it may not be unicode even on py3 np.savetxt(os.path.join(tmpdir, 'test.csv'), a, fmt=['%s'], @@ -526,7 +527,7 @@ class TestSaveTxt(object): def test_unicode_roundtrip(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) # our gz wrapper support encoding suffixes = ['', '.gz'] # stdlib 2 versions do not support encoding @@ -540,12 +541,12 @@ class TestSaveTxt(object): np.savetxt(os.path.join(tmpdir, 'test.csv' + suffix), a, fmt=['%s'], encoding='UTF-16-LE') b = np.loadtxt(os.path.join(tmpdir, 'test.csv' + suffix), - encoding='UTF-16-LE', dtype=np.unicode) + encoding='UTF-16-LE', dtype=np.unicode_) assert_array_equal(a, b) def test_unicode_bytestream(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) s = BytesIO() np.savetxt(s, a, fmt=['%s'], encoding='UTF-8') s.seek(0) @@ -553,7 +554,7 @@ class TestSaveTxt(object): def test_unicode_stringstream(self): utf8 = b'\xcf\x96'.decode('UTF-8') - a = np.array([utf8], dtype=np.unicode) + a = np.array([utf8], dtype=np.unicode_) s = StringIO() np.savetxt(s, a, fmt=['%s'], encoding='UTF-8') s.seek(0) @@ -575,13 +576,9 @@ class TestSaveTxt(object): @pytest.mark.skipif(sys.platform=='win32', reason="large files cause problems") @pytest.mark.slow + @requires_memory(free_bytes=7e9) def test_large_zip(self): # The test takes at least 6GB of memory, writes a file larger than 4GB - try: - a = 'a' * 6 * 1024 * 1024 * 1024 - del a - except (MemoryError, OverflowError): - pytest.skip("Cannot allocate enough memory for test") test_data = np.asarray([np.random.rand(np.random.randint(50,100),4) for i in range(800000)]) with tempdir() as tmpdir: @@ -632,12 +629,12 @@ class LoadTxtBase(object): with temppath() as path: with open(path, "wb") as f: f.write(nonascii.encode("UTF-16")) - x = self.loadfunc(path, encoding="UTF-16", dtype=np.unicode) + x = self.loadfunc(path, encoding="UTF-16", dtype=np.unicode_) assert_array_equal(x, nonascii) def test_binary_decode(self): utf16 = b'\xff\xfeh\x04 \x00i\x04 \x00j\x04' - v = self.loadfunc(BytesIO(utf16), dtype=np.unicode, encoding='UTF-16') + v = self.loadfunc(BytesIO(utf16), dtype=np.unicode_, encoding='UTF-16') assert_array_equal(v, np.array(utf16.decode('UTF-16').split())) def test_converters_decode(self): @@ -645,7 +642,7 @@ class LoadTxtBase(object): c = TextIO() c.write(b'\xcf\x96') c.seek(0) - x = self.loadfunc(c, dtype=np.unicode, + x = self.loadfunc(c, dtype=np.unicode_, converters={0: lambda x: x.decode('UTF-8')}) a = np.array([b'\xcf\x96'.decode('UTF-8')]) assert_array_equal(x, a) @@ -656,7 +653,7 @@ class LoadTxtBase(object): with temppath() as path: with io.open(path, 'wt', encoding='UTF-8') as f: f.write(utf8) - x = self.loadfunc(path, dtype=np.unicode, + x = self.loadfunc(path, dtype=np.unicode_, converters={0: lambda x: x + 't'}, encoding='UTF-8') a = np.array([utf8 + 't']) @@ -1104,7 +1101,7 @@ class TestLoadTxt(LoadTxtBase): with open(path, "wb") as f: f.write(butf8) with open(path, "rb") as f: - x = np.loadtxt(f, encoding="UTF-8", dtype=np.unicode) + x = np.loadtxt(f, encoding="UTF-8", dtype=np.unicode_) assert_array_equal(x, sutf8) # test broken latin1 conversion people now rely on with open(path, "rb") as f: @@ -1587,7 +1584,7 @@ M 33 21.99 with open(path, 'wb') as f: f.write(b'skip,skip,2001-01-01' + utf8 + b',1.0,skip') test = np.genfromtxt(path, delimiter=",", names=None, dtype=float, - usecols=(2, 3), converters={2: np.unicode}, + usecols=(2, 3), converters={2: np.compat.unicode}, encoding='UTF-8') control = np.array([('2001-01-01' + utf8.decode('UTF-8'), 1.)], dtype=[('', '|U11'), ('', float)]) @@ -1688,6 +1685,10 @@ M 33 21.99 test = np.genfromtxt(data) assert_equal(test, np.array([])) + # when skip_header > 0 + test = np.genfromtxt(data, skip_header=1) + assert_equal(test, np.array([])) + def test_fancy_dtype_alt(self): # Check that a nested dtype isn't MIA data = TextIO('1,2,3.0\n4,5,6.0\n') @@ -1867,7 +1868,7 @@ M 33 21.99 data = ["1, 1, 1, 1, -1.1"] * 50 mdata = TextIO("\n".join(data)) - converters = {4: lambda x: "(%s)" % x} + converters = {4: lambda x: "(%s)" % x.decode()} kwargs = dict(delimiter=",", converters=converters, dtype=[(_, int) for _ in 'abcde'],) assert_raises(ValueError, np.genfromtxt, mdata, **kwargs) @@ -2122,7 +2123,7 @@ M 33 21.99 ctl = np.array([ ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"], ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"]], - dtype=np.unicode) + dtype=np.unicode_) assert_array_equal(test, ctl) # test a mixed dtype @@ -2165,7 +2166,7 @@ M 33 21.99 ["norm1", "norm2", "norm3"], ["norm1", latin1, "norm3"], ["test1", "testNonethe" + utf8, "test3"]], - dtype=np.unicode) + dtype=np.unicode_) assert_array_equal(test, ctl) def test_recfromtxt(self): diff --git a/numpy/lib/tests/test_nanfunctions.py b/numpy/lib/tests/test_nanfunctions.py index b7261c63f..da2d0cc52 100644 --- a/numpy/lib/tests/test_nanfunctions.py +++ b/numpy/lib/tests/test_nanfunctions.py @@ -4,7 +4,7 @@ import warnings import pytest import numpy as np -from numpy.lib.nanfunctions import _nan_mask +from numpy.lib.nanfunctions import _nan_mask, _replace_nan from numpy.testing import ( assert_, assert_equal, assert_almost_equal, assert_no_warnings, assert_raises, assert_array_equal, suppress_warnings @@ -953,3 +953,30 @@ def test__nan_mask(arr, expected): # for types that can't possibly contain NaN if type(expected) is not np.ndarray: assert actual is True + + +def test__replace_nan(): + """ Test that _replace_nan returns the original array if there are no + NaNs, not a copy. + """ + for dtype in [np.bool, np.int32, np.int64]: + arr = np.array([0, 1], dtype=dtype) + result, mask = _replace_nan(arr, 0) + assert mask is None + # do not make a copy if there are no nans + assert result is arr + + for dtype in [np.float32, np.float64]: + arr = np.array([0, 1], dtype=dtype) + result, mask = _replace_nan(arr, 2) + assert (mask == False).all() + # mask is not None, so we make a copy + assert result is not arr + assert_equal(result, arr) + + arr_nan = np.array([0, 1, np.nan], dtype=dtype) + result_nan, mask_nan = _replace_nan(arr_nan, 2) + assert_equal(mask_nan, np.array([False, False, True])) + assert result_nan is not arr_nan + assert_equal(result_nan, np.array([0, 1, 2])) + assert np.isnan(arr_nan[-1]) diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py index 0c839d486..fa5f4dec2 100644 --- a/numpy/lib/tests/test_recfunctions.py +++ b/numpy/lib/tests/test_recfunctions.py @@ -91,8 +91,10 @@ class TestRecFunctions(object): control = np.array([(1,), (4,)], dtype=[('a', int)]) assert_equal(test, control) + # dropping all fields results in an array with no fields test = drop_fields(a, ['a', 'b']) - assert_(test is None) + control = np.array([(), ()], dtype=[]) + assert_equal(test, control) def test_rename_fields(self): # Test rename fields @@ -378,8 +380,8 @@ class TestMergeArrays(object): z = np.array( [('A', 1.), ('B', 2.)], dtype=[('A', '|S3'), ('B', float)]) w = np.array( - [(1, (2, 3.0)), (4, (5, 6.0))], - dtype=[('a', int), ('b', [('ba', float), ('bb', int)])]) + [(1, (2, 3.0, ())), (4, (5, 6.0, ()))], + dtype=[('a', int), ('b', [('ba', float), ('bb', int), ('bc', [])])]) self.data = (w, x, y, z) def test_solo(self): @@ -450,8 +452,8 @@ class TestMergeArrays(object): test = merge_arrays((x, w), flatten=False) controldtype = [('f0', int), ('f1', [('a', int), - ('b', [('ba', float), ('bb', int)])])] - control = np.array([(1., (1, (2, 3.0))), (2, (4, (5, 6.0)))], + ('b', [('ba', float), ('bb', int), ('bc', [])])])] + control = np.array([(1., (1, (2, 3.0, ()))), (2, (4, (5, 6.0, ())))], dtype=controldtype) assert_equal(test, control) diff --git a/numpy/lib/tests/test_regression.py b/numpy/lib/tests/test_regression.py index 4c46bc46b..4cd812f5d 100644 --- a/numpy/lib/tests/test_regression.py +++ b/numpy/lib/tests/test_regression.py @@ -21,8 +21,8 @@ class TestRegression(object): # Ticket #91 x = np.random.random((3, 3)) y = x.copy() - np.cov(x, rowvar=1) - np.cov(y, rowvar=0) + np.cov(x, rowvar=True) + np.cov(y, rowvar=False) assert_array_equal(x, y) def test_mem_digitize(self): @@ -56,7 +56,7 @@ class TestRegression(object): def test_poly1d_nan_roots(self): # Ticket #396 - p = np.poly1d([np.nan, np.nan, 1], r=0) + p = np.poly1d([np.nan, np.nan, 1], r=False) assert_raises(np.linalg.LinAlgError, getattr, p, "r") def test_mem_polymul(self): diff --git a/numpy/lib/tests/test_shape_base.py b/numpy/lib/tests/test_shape_base.py index 01ea028bb..be1604a75 100644 --- a/numpy/lib/tests/test_shape_base.py +++ b/numpy/lib/tests/test_shape_base.py @@ -289,14 +289,26 @@ class TestExpandDims(object): assert_(b.shape[axis] == 1) assert_(np.squeeze(b).shape == s) - def test_deprecations(self): - # 2017-05-17, 1.13.0 + def test_axis_tuple(self): + a = np.empty((3, 3, 3)) + assert np.expand_dims(a, axis=(0, 1, 2)).shape == (1, 1, 1, 3, 3, 3) + assert np.expand_dims(a, axis=(0, -1, -2)).shape == (1, 3, 3, 3, 1, 1) + assert np.expand_dims(a, axis=(0, 3, 5)).shape == (1, 3, 3, 1, 3, 1) + assert np.expand_dims(a, axis=(0, -3, -5)).shape == (1, 1, 3, 1, 3, 3) + + def test_axis_out_of_range(self): s = (2, 3, 4, 5) a = np.empty(s) - with warnings.catch_warnings(): - warnings.simplefilter("always") - assert_warns(DeprecationWarning, expand_dims, a, -6) - assert_warns(DeprecationWarning, expand_dims, a, 5) + assert_raises(np.AxisError, expand_dims, a, -6) + assert_raises(np.AxisError, expand_dims, a, 5) + + a = np.empty((3, 3, 3)) + assert_raises(np.AxisError, expand_dims, a, (0, -6)) + assert_raises(np.AxisError, expand_dims, a, (0, 5)) + + def test_repeated_axis(self): + a = np.empty((3, 3, 3)) + assert_raises(ValueError, expand_dims, a, axis=(1, 1)) def test_subclasses(self): a = np.arange(10).reshape((2, 5)) diff --git a/numpy/lib/tests/test_ufunclike.py b/numpy/lib/tests/test_ufunclike.py index 0f06876a1..64280616f 100644 --- a/numpy/lib/tests/test_ufunclike.py +++ b/numpy/lib/tests/test_ufunclike.py @@ -21,7 +21,7 @@ class TestUfunclike(object): assert_equal(res, tgt) assert_equal(out, tgt) - a = a.astype(np.complex) + a = a.astype(np.complex_) with assert_raises(TypeError): ufl.isposinf(a) @@ -36,7 +36,7 @@ class TestUfunclike(object): assert_equal(res, tgt) assert_equal(out, tgt) - a = a.astype(np.complex) + a = a.astype(np.complex_) with assert_raises(TypeError): ufl.isneginf(a) diff --git a/numpy/lib/twodim_base.py b/numpy/lib/twodim_base.py index f3dc6c8e1..f45392188 100644 --- a/numpy/lib/twodim_base.py +++ b/numpy/lib/twodim_base.py @@ -368,7 +368,7 @@ def tri(N, M=None, k=0, dtype=float): ------- tri : ndarray of shape (N, M) Array with its lower triangle filled with ones and zero elsewhere; - in other words ``T[i,j] == 1`` for ``i <= j + k``, 0 otherwise. + in other words ``T[i,j] == 1`` for ``j <= i + k``, 0 otherwise. Examples -------- diff --git a/numpy/lib/type_check.py b/numpy/lib/type_check.py index ac4b03a6c..977117235 100644 --- a/numpy/lib/type_check.py +++ b/numpy/lib/type_check.py @@ -68,16 +68,14 @@ def mintypecode(typechars, typeset='GDFgdf', default='d'): 'G' """ - typecodes = [(isinstance(t, str) and t) or asarray(t).dtype.char - for t in typechars] - intersection = [t for t in typecodes if t in typeset] + typecodes = ((isinstance(t, str) and t) or asarray(t).dtype.char + for t in typechars) + intersection = set(t for t in typecodes if t in typeset) if not intersection: return default if 'F' in intersection and 'd' in intersection: return 'D' - l = [(_typecodes_by_elsize.index(t), t) for t in intersection] - l.sort() - return l[0][1] + return min(intersection, key=_typecodes_by_elsize.index) def _asfarray_dispatcher(a, dtype=None): @@ -395,19 +393,27 @@ def nan_to_num(x, copy=True, nan=0.0, posinf=None, neginf=None): in-place (False). The in-place operation only occurs if casting to an array does not require a copy. Default is True. + + .. versionadded:: 1.13 nan : int, float, optional Value to be used to fill NaN values. If no value is passed then NaN values will be replaced with 0.0. + + .. versionadded:: 1.17 posinf : int, float, optional Value to be used to fill positive infinity values. If no value is passed then positive infinity values will be replaced with a very large number. + + .. versionadded:: 1.17 neginf : int, float, optional Value to be used to fill negative infinity values. If no value is passed then negative infinity values will be replaced with a very small (or negative) number. + + .. versionadded:: 1.17 - .. versionadded:: 1.13 + Returns ------- diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index c7dbcc5f9..3c71d2a7c 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -788,13 +788,8 @@ def lookfor(what, module=None, import_modules=True, regenerate=False, if kind in ('module', 'object'): # don't show modules or objects continue - ok = True doc = docstring.lower() - for w in whats: - if w not in doc: - ok = False - break - if ok: + if all(w in doc for w in whats): found.append(name) # Relevance sort @@ -1003,93 +998,6 @@ def _getmembers(item): if hasattr(item, x)] return members -#----------------------------------------------------------------------------- - -# The following SafeEval class and company are adapted from Michael Spencer's -# ASPN Python Cookbook recipe: https://code.activestate.com/recipes/364469/ -# -# Accordingly it is mostly Copyright 2006 by Michael Spencer. -# The recipe, like most of the other ASPN Python Cookbook recipes was made -# available under the Python license. -# https://en.wikipedia.org/wiki/Python_License - -# It has been modified to: -# * handle unary -/+ -# * support True/False/None -# * raise SyntaxError instead of a custom exception. - -class SafeEval(object): - """ - Object to evaluate constant string expressions. - - This includes strings with lists, dicts and tuples using the abstract - syntax tree created by ``compiler.parse``. - - .. deprecated:: 1.10.0 - - See Also - -------- - safe_eval - - """ - def __init__(self): - # 2014-10-15, 1.10 - warnings.warn("SafeEval is deprecated in 1.10 and will be removed.", - DeprecationWarning, stacklevel=2) - - def visit(self, node): - cls = node.__class__ - meth = getattr(self, 'visit' + cls.__name__, self.default) - return meth(node) - - def default(self, node): - raise SyntaxError("Unsupported source construct: %s" - % node.__class__) - - def visitExpression(self, node): - return self.visit(node.body) - - def visitNum(self, node): - return node.n - - def visitStr(self, node): - return node.s - - def visitBytes(self, node): - return node.s - - def visitDict(self, node,**kw): - return dict([(self.visit(k), self.visit(v)) - for k, v in zip(node.keys, node.values)]) - - def visitTuple(self, node): - return tuple([self.visit(i) for i in node.elts]) - - def visitList(self, node): - return [self.visit(i) for i in node.elts] - - def visitUnaryOp(self, node): - import ast - if isinstance(node.op, ast.UAdd): - return +self.visit(node.operand) - elif isinstance(node.op, ast.USub): - return -self.visit(node.operand) - else: - raise SyntaxError("Unknown unary op: %r" % node.op) - - def visitName(self, node): - if node.id == 'False': - return False - elif node.id == 'True': - return True - elif node.id == 'None': - return None - else: - raise SyntaxError("Unknown name: %s" % node.id) - - def visitNameConstant(self, node): - return node.value - def safe_eval(source): """ diff --git a/numpy/linalg/__init__.py b/numpy/linalg/__init__.py index 4b696c883..55560815d 100644 --- a/numpy/linalg/__init__.py +++ b/numpy/linalg/__init__.py @@ -1,53 +1,77 @@ """ -Core Linear Algebra Tools -========================= - -=============== ========================================================== -Linear algebra basics -========================================================================== -norm Vector or matrix norm -inv Inverse of a square matrix -solve Solve a linear system of equations -det Determinant of a square matrix -slogdet Logarithm of the determinant of a square matrix -lstsq Solve linear least-squares problem -pinv Pseudo-inverse (Moore-Penrose) calculated using a singular - value decomposition -matrix_power Integer power of a square matrix -matrix_rank Calculate matrix rank using an SVD-based method -=============== ========================================================== - -=============== ========================================================== -Eigenvalues and decompositions -========================================================================== -eig Eigenvalues and vectors of a square matrix -eigh Eigenvalues and eigenvectors of a Hermitian matrix -eigvals Eigenvalues of a square matrix -eigvalsh Eigenvalues of a Hermitian matrix -qr QR decomposition of a matrix -svd Singular value decomposition of a matrix -cholesky Cholesky decomposition of a matrix -=============== ========================================================== - -=============== ========================================================== -Tensor operations -========================================================================== -tensorsolve Solve a linear tensor equation -tensorinv Calculate an inverse of a tensor -=============== ========================================================== - -=============== ========================================================== +``numpy.linalg`` +================ + +The NumPy linear algebra functions rely on BLAS and LAPACK to provide efficient +low level implementations of standard linear algebra algorithms. Those +libraries may be provided by NumPy itself using C versions of a subset of their +reference implementations but, when possible, highly optimized libraries that +take advantage of specialized processor functionality are preferred. Examples +of such libraries are OpenBLAS, MKL (TM), and ATLAS. Because those libraries +are multithreaded and processor dependent, environmental variables and external +packages such as threadpoolctl may be needed to control the number of threads +or specify the processor architecture. + +- OpenBLAS: https://www.openblas.net/ +- threadpoolctl: https://github.com/joblib/threadpoolctl + +Please note that the most-used linear algebra functions in NumPy are present in +the main ``numpy`` namespace rather than in ``numpy.linalg``. There are: +``dot``, ``vdot``, ``inner``, ``outer``, ``matmul``, ``tensordot``, ``einsum``, +``einsum_path`` and ``kron``. + +Functions present in numpy.linalg are listed below. + + +Matrix and vector products +-------------------------- + + multi_dot + matrix_power + +Decompositions +-------------- + + cholesky + qr + svd + +Matrix eigenvalues +------------------ + + eig + eigh + eigvals + eigvalsh + +Norms and other numbers +----------------------- + + norm + cond + det + matrix_rank + slogdet + +Solving equations and inverting matrices +---------------------------------------- + + solve + tensorsolve + lstsq + inv + pinv + tensorinv + Exceptions -========================================================================== -LinAlgError Indicates a failed linear algebra operation -=============== ========================================================== +---------- + + LinAlgError """ from __future__ import division, absolute_import, print_function # To get sub-modules -from .info import __doc__ - from .linalg import * from numpy._pytesttester import PytestTester diff --git a/numpy/linalg/info.py b/numpy/linalg/info.py deleted file mode 100644 index 646ecda04..000000000 --- a/numpy/linalg/info.py +++ /dev/null @@ -1,37 +0,0 @@ -"""\ -Core Linear Algebra Tools -------------------------- -Linear algebra basics: - -- norm Vector or matrix norm -- inv Inverse of a square matrix -- solve Solve a linear system of equations -- det Determinant of a square matrix -- lstsq Solve linear least-squares problem -- pinv Pseudo-inverse (Moore-Penrose) calculated using a singular - value decomposition -- matrix_power Integer power of a square matrix - -Eigenvalues and decompositions: - -- eig Eigenvalues and vectors of a square matrix -- eigh Eigenvalues and eigenvectors of a Hermitian matrix -- eigvals Eigenvalues of a square matrix -- eigvalsh Eigenvalues of a Hermitian matrix -- qr QR decomposition of a matrix -- svd Singular value decomposition of a matrix -- cholesky Cholesky decomposition of a matrix - -Tensor operations: - -- tensorsolve Solve a linear tensor equation -- tensorinv Calculate an inverse of a tensor - -Exceptions: - -- LinAlgError Indicates a failed linear algebra operation - -""" -from __future__ import division, absolute_import, print_function - -depends = ['core'] diff --git a/numpy/linalg/lapack_lite/fortran.py b/numpy/linalg/lapack_lite/fortran.py index 87c27aab9..dc0a5ebd9 100644 --- a/numpy/linalg/lapack_lite/fortran.py +++ b/numpy/linalg/lapack_lite/fortran.py @@ -54,7 +54,7 @@ class PushbackIterator(object): Return an iterator for which items can be pushed back into. Call the .pushback(item) method to have item returned as the next - value of .next(). + value of next(). """ def __init__(self, iterable): object.__init__(self) diff --git a/numpy/linalg/lapack_lite/python_xerbla.c b/numpy/linalg/lapack_lite/python_xerbla.c index dfc195556..4dbb92e1f 100644 --- a/numpy/linalg/lapack_lite/python_xerbla.c +++ b/numpy/linalg/lapack_lite/python_xerbla.c @@ -1,4 +1,6 @@ #include "Python.h" +#include "numpy/npy_common.h" +#include "npy_cblas.h" #undef c_abs #include "f2c.h" @@ -20,7 +22,7 @@ info: Number of the invalid parameter. */ -int xerbla_(char *srname, integer *info) +CBLAS_INT BLAS_FUNC(xerbla)(char *srname, CBLAS_INT *info) { static const char format[] = "On entry to %.*s" \ " parameter number %d had an illegal value"; @@ -38,7 +40,7 @@ int xerbla_(char *srname, integer *info) #ifdef WITH_THREAD save = PyGILState_Ensure(); #endif - PyOS_snprintf(buf, sizeof(buf), format, len, srname, *info); + PyOS_snprintf(buf, sizeof(buf), format, len, srname, (int)*info); PyErr_SetString(PyExc_ValueError, buf); #ifdef WITH_THREAD PyGILState_Release(save); diff --git a/numpy/linalg/lapack_litemodule.c b/numpy/linalg/lapack_litemodule.c index 696a6d874..4c80317f5 100644 --- a/numpy/linalg/lapack_litemodule.c +++ b/numpy/linalg/lapack_litemodule.c @@ -6,46 +6,66 @@ More modifications by Jeff Whitaker #include "Python.h" #include "numpy/arrayobject.h" +#include "npy_cblas.h" + + +#define FNAME(name) BLAS_FUNC(name) + +typedef CBLAS_INT fortran_int; + +#ifdef HAVE_BLAS_ILP64 + +#if NPY_BITSOF_SHORT == 64 +#define FINT_PYFMT "h" +#elif NPY_BITSOF_INT == 64 +#define FINT_PYFMT "i" +#elif NPY_BITSOF_LONG == 64 +#define FINT_PYFMT "l" +#elif NPY_BITSOF_LONGLONG == 64 +#define FINT_PYFMT "L" +#else +#error No compatible 64-bit integer size. \ + Please contact NumPy maintainers and give detailed information about your \ + compiler and platform, or set NPY_USE_BLAS64_=0 +#endif -#ifdef NO_APPEND_FORTRAN -# define FNAME(x) x #else -# define FNAME(x) x##_ +#define FINT_PYFMT "i" #endif typedef struct { float r, i; } f2c_complex; typedef struct { double r, i; } f2c_doublecomplex; /* typedef long int (*L_fp)(); */ -extern int FNAME(dgelsd)(int *m, int *n, int *nrhs, - double a[], int *lda, double b[], int *ldb, - double s[], double *rcond, int *rank, - double work[], int *lwork, int iwork[], int *info); +extern fortran_int FNAME(dgelsd)(fortran_int *m, fortran_int *n, fortran_int *nrhs, + double a[], fortran_int *lda, double b[], fortran_int *ldb, + double s[], double *rcond, fortran_int *rank, + double work[], fortran_int *lwork, fortran_int iwork[], fortran_int *info); -extern int FNAME(zgelsd)(int *m, int *n, int *nrhs, - f2c_doublecomplex a[], int *lda, - f2c_doublecomplex b[], int *ldb, - double s[], double *rcond, int *rank, - f2c_doublecomplex work[], int *lwork, - double rwork[], int iwork[], int *info); +extern fortran_int FNAME(zgelsd)(fortran_int *m, fortran_int *n, fortran_int *nrhs, + f2c_doublecomplex a[], fortran_int *lda, + f2c_doublecomplex b[], fortran_int *ldb, + double s[], double *rcond, fortran_int *rank, + f2c_doublecomplex work[], fortran_int *lwork, + double rwork[], fortran_int iwork[], fortran_int *info); -extern int FNAME(dgeqrf)(int *m, int *n, double a[], int *lda, +extern fortran_int FNAME(dgeqrf)(fortran_int *m, fortran_int *n, double a[], fortran_int *lda, double tau[], double work[], - int *lwork, int *info); + fortran_int *lwork, fortran_int *info); -extern int FNAME(zgeqrf)(int *m, int *n, f2c_doublecomplex a[], int *lda, +extern fortran_int FNAME(zgeqrf)(fortran_int *m, fortran_int *n, f2c_doublecomplex a[], fortran_int *lda, f2c_doublecomplex tau[], f2c_doublecomplex work[], - int *lwork, int *info); + fortran_int *lwork, fortran_int *info); -extern int FNAME(dorgqr)(int *m, int *n, int *k, double a[], int *lda, +extern fortran_int FNAME(dorgqr)(fortran_int *m, fortran_int *n, fortran_int *k, double a[], fortran_int *lda, double tau[], double work[], - int *lwork, int *info); + fortran_int *lwork, fortran_int *info); -extern int FNAME(zungqr)(int *m, int *n, int *k, f2c_doublecomplex a[], - int *lda, f2c_doublecomplex tau[], - f2c_doublecomplex work[], int *lwork, int *info); +extern fortran_int FNAME(zungqr)(fortran_int *m, fortran_int *n, fortran_int *k, f2c_doublecomplex a[], + fortran_int *lda, f2c_doublecomplex tau[], + f2c_doublecomplex work[], fortran_int *lwork, fortran_int *info); -extern int FNAME(xerbla)(char *srname, int *info); +extern fortran_int FNAME(xerbla)(char *srname, fortran_int *info); static PyObject *LapackError; @@ -90,27 +110,31 @@ check_object(PyObject *ob, int t, char *obname, #define FDATA(p) ((float *) PyArray_DATA((PyArrayObject *)p)) #define CDATA(p) ((f2c_complex *) PyArray_DATA((PyArrayObject *)p)) #define ZDATA(p) ((f2c_doublecomplex *) PyArray_DATA((PyArrayObject *)p)) -#define IDATA(p) ((int *) PyArray_DATA((PyArrayObject *)p)) +#define IDATA(p) ((fortran_int *) PyArray_DATA((PyArrayObject *)p)) static PyObject * lapack_lite_dgelsd(PyObject *NPY_UNUSED(self), PyObject *args) { - int lapack_lite_status; - int m; - int n; - int nrhs; + fortran_int lapack_lite_status; + fortran_int m; + fortran_int n; + fortran_int nrhs; PyObject *a; - int lda; + fortran_int lda; PyObject *b; - int ldb; + fortran_int ldb; PyObject *s; double rcond; - int rank; + fortran_int rank; PyObject *work; PyObject *iwork; - int lwork; - int info; - TRY(PyArg_ParseTuple(args,"iiiOiOiOdiOiOi:dgelsd", + fortran_int lwork; + fortran_int info; + + TRY(PyArg_ParseTuple(args, + (FINT_PYFMT FINT_PYFMT FINT_PYFMT "O" FINT_PYFMT "O" + FINT_PYFMT "O" "d" FINT_PYFMT "O" FINT_PYFMT "O" + FINT_PYFMT ":dgelsd"), &m,&n,&nrhs,&a,&lda,&b,&ldb,&s,&rcond, &rank,&work,&lwork,&iwork,&info)); @@ -118,7 +142,11 @@ lapack_lite_dgelsd(PyObject *NPY_UNUSED(self), PyObject *args) TRY(check_object(b,NPY_DOUBLE,"b","NPY_DOUBLE","dgelsd")); TRY(check_object(s,NPY_DOUBLE,"s","NPY_DOUBLE","dgelsd")); TRY(check_object(work,NPY_DOUBLE,"work","NPY_DOUBLE","dgelsd")); +#ifndef NPY_UMATH_USE_BLAS64_ TRY(check_object(iwork,NPY_INT,"iwork","NPY_INT","dgelsd")); +#else + TRY(check_object(iwork,NPY_INT64,"iwork","NPY_INT64","dgelsd")); +#endif lapack_lite_status = FNAME(dgelsd)(&m,&n,&nrhs,DDATA(a),&lda,DDATA(b),&ldb, @@ -128,8 +156,11 @@ lapack_lite_dgelsd(PyObject *NPY_UNUSED(self), PyObject *args) return NULL; } - return Py_BuildValue("{s:i,s:i,s:i,s:i,s:i,s:i,s:d,s:i,s:i,s:i}","dgelsd_", - lapack_lite_status,"m",m,"n",n,"nrhs",nrhs, + return Py_BuildValue(("{s:" FINT_PYFMT ",s:" FINT_PYFMT ",s:" FINT_PYFMT + ",s:" FINT_PYFMT ",s:" FINT_PYFMT ",s:" FINT_PYFMT + ",s:d,s:" FINT_PYFMT ",s:" FINT_PYFMT + ",s:" FINT_PYFMT "}"), + "dgelsd_",lapack_lite_status,"m",m,"n",n,"nrhs",nrhs, "lda",lda,"ldb",ldb,"rcond",rcond,"rank",rank, "lwork",lwork,"info",info); } @@ -137,13 +168,16 @@ lapack_lite_dgelsd(PyObject *NPY_UNUSED(self), PyObject *args) static PyObject * lapack_lite_dgeqrf(PyObject *NPY_UNUSED(self), PyObject *args) { - int lapack_lite_status; - int m, n, lwork; + fortran_int lapack_lite_status; + fortran_int m, n, lwork; PyObject *a, *tau, *work; - int lda; - int info; + fortran_int lda; + fortran_int info; - TRY(PyArg_ParseTuple(args,"iiOiOOii:dgeqrf",&m,&n,&a,&lda,&tau,&work,&lwork,&info)); + TRY(PyArg_ParseTuple(args, + (FINT_PYFMT FINT_PYFMT "O" FINT_PYFMT "OO" + FINT_PYFMT FINT_PYFMT ":dgeqrf"), + &m,&n,&a,&lda,&tau,&work,&lwork,&info)); /* check objects and convert to right storage order */ TRY(check_object(a,NPY_DOUBLE,"a","NPY_DOUBLE","dgeqrf")); @@ -157,7 +191,9 @@ lapack_lite_dgeqrf(PyObject *NPY_UNUSED(self), PyObject *args) return NULL; } - return Py_BuildValue("{s:i,s:i,s:i,s:i,s:i,s:i}","dgeqrf_", + return Py_BuildValue(("{s:" FINT_PYFMT ",s:" FINT_PYFMT ",s:" FINT_PYFMT + ",s:" FINT_PYFMT ",s:" FINT_PYFMT ",s:" FINT_PYFMT "}"), + "dgeqrf_", lapack_lite_status,"m",m,"n",n,"lda",lda, "lwork",lwork,"info",info); } @@ -166,13 +202,17 @@ lapack_lite_dgeqrf(PyObject *NPY_UNUSED(self), PyObject *args) static PyObject * lapack_lite_dorgqr(PyObject *NPY_UNUSED(self), PyObject *args) { - int lapack_lite_status; - int m, n, k, lwork; + fortran_int lapack_lite_status; + fortran_int m, n, k, lwork; PyObject *a, *tau, *work; - int lda; - int info; - - TRY(PyArg_ParseTuple(args,"iiiOiOOii:dorgqr", &m, &n, &k, &a, &lda, &tau, &work, &lwork, &info)); + fortran_int lda; + fortran_int info; + + TRY(PyArg_ParseTuple(args, + (FINT_PYFMT FINT_PYFMT FINT_PYFMT "O" + FINT_PYFMT "OO" FINT_PYFMT FINT_PYFMT + ":dorgqr"), + &m, &n, &k, &a, &lda, &tau, &work, &lwork, &info)); TRY(check_object(a,NPY_DOUBLE,"a","NPY_DOUBLE","dorgqr")); TRY(check_object(tau,NPY_DOUBLE,"tau","NPY_DOUBLE","dorgqr")); TRY(check_object(work,NPY_DOUBLE,"work","NPY_DOUBLE","dorgqr")); @@ -191,23 +231,26 @@ lapack_lite_dorgqr(PyObject *NPY_UNUSED(self), PyObject *args) static PyObject * lapack_lite_zgelsd(PyObject *NPY_UNUSED(self), PyObject *args) { - int lapack_lite_status; - int m; - int n; - int nrhs; + fortran_int lapack_lite_status; + fortran_int m; + fortran_int n; + fortran_int nrhs; PyObject *a; - int lda; + fortran_int lda; PyObject *b; - int ldb; + fortran_int ldb; PyObject *s; double rcond; - int rank; + fortran_int rank; PyObject *work; - int lwork; + fortran_int lwork; PyObject *rwork; PyObject *iwork; - int info; - TRY(PyArg_ParseTuple(args,"iiiOiOiOdiOiOOi:zgelsd", + fortran_int info; + TRY(PyArg_ParseTuple(args, + (FINT_PYFMT FINT_PYFMT FINT_PYFMT "O" FINT_PYFMT + "O" FINT_PYFMT "Od" FINT_PYFMT "O" FINT_PYFMT + "OO" FINT_PYFMT ":zgelsd"), &m,&n,&nrhs,&a,&lda,&b,&ldb,&s,&rcond, &rank,&work,&lwork,&rwork,&iwork,&info)); @@ -216,7 +259,11 @@ lapack_lite_zgelsd(PyObject *NPY_UNUSED(self), PyObject *args) TRY(check_object(s,NPY_DOUBLE,"s","NPY_DOUBLE","zgelsd")); TRY(check_object(work,NPY_CDOUBLE,"work","NPY_CDOUBLE","zgelsd")); TRY(check_object(rwork,NPY_DOUBLE,"rwork","NPY_DOUBLE","zgelsd")); +#ifndef NPY_UMATH_USE_BLAS64_ TRY(check_object(iwork,NPY_INT,"iwork","NPY_INT","zgelsd")); +#else + TRY(check_object(iwork,NPY_INT64,"iwork","NPY_INT64","zgelsd")); +#endif lapack_lite_status = FNAME(zgelsd)(&m,&n,&nrhs,ZDATA(a),&lda,ZDATA(b),&ldb,DDATA(s),&rcond, @@ -225,7 +272,11 @@ lapack_lite_zgelsd(PyObject *NPY_UNUSED(self), PyObject *args) return NULL; } - return Py_BuildValue("{s:i,s:i,s:i,s:i,s:i,s:i,s:i,s:i,s:i}","zgelsd_", + return Py_BuildValue(("{s:" FINT_PYFMT ",s:" FINT_PYFMT ",s:" FINT_PYFMT + ",s:" FINT_PYFMT ",s:" FINT_PYFMT ",s:" FINT_PYFMT + ",s:" FINT_PYFMT ",s:" FINT_PYFMT ",s:" FINT_PYFMT + "}"), + "zgelsd_", lapack_lite_status,"m",m,"n",n,"nrhs",nrhs,"lda",lda, "ldb",ldb,"rank",rank,"lwork",lwork,"info",info); } @@ -233,13 +284,16 @@ lapack_lite_zgelsd(PyObject *NPY_UNUSED(self), PyObject *args) static PyObject * lapack_lite_zgeqrf(PyObject *NPY_UNUSED(self), PyObject *args) { - int lapack_lite_status; - int m, n, lwork; + fortran_int lapack_lite_status; + fortran_int m, n, lwork; PyObject *a, *tau, *work; - int lda; - int info; + fortran_int lda; + fortran_int info; - TRY(PyArg_ParseTuple(args,"iiOiOOii:zgeqrf",&m,&n,&a,&lda,&tau,&work,&lwork,&info)); + TRY(PyArg_ParseTuple(args, + (FINT_PYFMT FINT_PYFMT "O" FINT_PYFMT "OO" + FINT_PYFMT "" FINT_PYFMT ":zgeqrf"), + &m,&n,&a,&lda,&tau,&work,&lwork,&info)); /* check objects and convert to right storage order */ TRY(check_object(a,NPY_CDOUBLE,"a","NPY_CDOUBLE","zgeqrf")); @@ -253,20 +307,27 @@ lapack_lite_zgeqrf(PyObject *NPY_UNUSED(self), PyObject *args) return NULL; } - return Py_BuildValue("{s:i,s:i,s:i,s:i,s:i,s:i}","zgeqrf_",lapack_lite_status,"m",m,"n",n,"lda",lda,"lwork",lwork,"info",info); + return Py_BuildValue(("{s:" FINT_PYFMT ",s:" FINT_PYFMT + ",s:" FINT_PYFMT ",s:" FINT_PYFMT + ",s:" FINT_PYFMT ",s:" FINT_PYFMT "}"), + "zgeqrf_",lapack_lite_status,"m",m,"n",n,"lda",lda,"lwork",lwork,"info",info); } static PyObject * lapack_lite_zungqr(PyObject *NPY_UNUSED(self), PyObject *args) { - int lapack_lite_status; - int m, n, k, lwork; + fortran_int lapack_lite_status; + fortran_int m, n, k, lwork; PyObject *a, *tau, *work; - int lda; - int info; - - TRY(PyArg_ParseTuple(args,"iiiOiOOii:zungqr", &m, &n, &k, &a, &lda, &tau, &work, &lwork, &info)); + fortran_int lda; + fortran_int info; + + TRY(PyArg_ParseTuple(args, + (FINT_PYFMT FINT_PYFMT FINT_PYFMT "O" + FINT_PYFMT "OO" FINT_PYFMT "" FINT_PYFMT + ":zungqr"), + &m, &n, &k, &a, &lda, &tau, &work, &lwork, &info)); TRY(check_object(a,NPY_CDOUBLE,"a","NPY_CDOUBLE","zungqr")); TRY(check_object(tau,NPY_CDOUBLE,"tau","NPY_CDOUBLE","zungqr")); TRY(check_object(work,NPY_CDOUBLE,"work","NPY_CDOUBLE","zungqr")); @@ -279,7 +340,8 @@ lapack_lite_zungqr(PyObject *NPY_UNUSED(self), PyObject *args) return NULL; } - return Py_BuildValue("{s:i,s:i}","zungqr_",lapack_lite_status, + return Py_BuildValue(("{s:" FINT_PYFMT ",s:" FINT_PYFMT "}"), + "zungqr_",lapack_lite_status, "info",info); } @@ -287,7 +349,7 @@ lapack_lite_zungqr(PyObject *NPY_UNUSED(self), PyObject *args) static PyObject * lapack_lite_xerbla(PyObject *NPY_UNUSED(self), PyObject *args) { - int info = -1; + fortran_int info = -1; NPY_BEGIN_THREADS_DEF; NPY_BEGIN_THREADS; diff --git a/numpy/linalg/linalg.py b/numpy/linalg/linalg.py index 267c42afb..f1b2c2228 100644 --- a/numpy/linalg/linalg.py +++ b/numpy/linalg/linalg.py @@ -194,37 +194,33 @@ def _fastCopyAndTranspose(type, *arrays): else: return cast_arrays -def _assertRank2(*arrays): +def _assert_2d(*arrays): for a in arrays: if a.ndim != 2: raise LinAlgError('%d-dimensional array given. Array must be ' 'two-dimensional' % a.ndim) -def _assertRankAtLeast2(*arrays): +def _assert_stacked_2d(*arrays): for a in arrays: if a.ndim < 2: raise LinAlgError('%d-dimensional array given. Array must be ' 'at least two-dimensional' % a.ndim) -def _assertNdSquareness(*arrays): +def _assert_stacked_square(*arrays): for a in arrays: m, n = a.shape[-2:] if m != n: raise LinAlgError('Last 2 dimensions of the array must be square') -def _assertFinite(*arrays): +def _assert_finite(*arrays): for a in arrays: - if not (isfinite(a).all()): + if not isfinite(a).all(): raise LinAlgError("Array must not contain infs or NaNs") -def _isEmpty2d(arr): +def _is_empty_2d(arr): # check size first for efficiency return arr.size == 0 and product(arr.shape[-2:]) == 0 -def _assertNoEmpty2d(*arrays): - for a in arrays: - if _isEmpty2d(a): - raise LinAlgError("Arrays cannot be empty") def transpose(a): """ @@ -386,8 +382,8 @@ def solve(a, b): """ a, _ = _makearray(a) - _assertRankAtLeast2(a) - _assertNdSquareness(a) + _assert_stacked_2d(a) + _assert_stacked_square(a) b, wrap = _makearray(b) t, result_t = _commonType(a, b) @@ -542,8 +538,8 @@ def inv(a): """ a, wrap = _makearray(a) - _assertRankAtLeast2(a) - _assertNdSquareness(a) + _assert_stacked_2d(a) + _assert_stacked_square(a) t, result_t = _commonType(a) signature = 'D->D' if isComplexType(t) else 'd->d' @@ -622,8 +618,8 @@ def matrix_power(a, n): """ a = asanyarray(a) - _assertRankAtLeast2(a) - _assertNdSquareness(a) + _assert_stacked_2d(a) + _assert_stacked_square(a) try: n = operator.index(n) @@ -752,8 +748,8 @@ def cholesky(a): extobj = get_linalg_error_extobj(_raise_linalgerror_nonposdef) gufunc = _umath_linalg.cholesky_lo a, wrap = _makearray(a) - _assertRankAtLeast2(a) - _assertNdSquareness(a) + _assert_stacked_2d(a) + _assert_stacked_square(a) t, result_t = _commonType(a) signature = 'D->D' if isComplexType(t) else 'd->d' r = gufunc(a, signature=signature, extobj=extobj) @@ -778,15 +774,13 @@ def qr(a, mode='reduced'): ---------- a : array_like, shape (M, N) Matrix to be factored. - mode : {'reduced', 'complete', 'r', 'raw', 'full', 'economic'}, optional + mode : {'reduced', 'complete', 'r', 'raw'}, optional If K = min(M, N), then * 'reduced' : returns q, r with dimensions (M, K), (K, N) (default) * 'complete' : returns q, r with dimensions (M, M), (M, N) * 'r' : returns r only with dimensions (K, N) * 'raw' : returns h, tau with dimensions (N, M), (K,) - * 'full' : alias of 'reduced', deprecated - * 'economic' : returns h from 'raw', deprecated. The options 'reduced', 'complete, and 'raw' are new in numpy 1.8, see the notes for more information. The default is 'reduced', and to @@ -848,12 +842,8 @@ def qr(a, mode='reduced'): >>> np.allclose(a, np.dot(q, r)) # a does equal qr True >>> r2 = np.linalg.qr(a, mode='r') - >>> r3 = np.linalg.qr(a, mode='economic') >>> np.allclose(r, r2) # mode='r' returns the same r as mode='full' True - >>> # But only triu parts are guaranteed equal when mode='economic' - >>> np.allclose(r, np.triu(r3[:6,:6], k=0)) - True Example illustrating a common use of `qr`: solving of least squares problems @@ -901,7 +891,7 @@ def qr(a, mode='reduced'): raise ValueError("Unrecognized mode '%s'" % mode) a, wrap = _makearray(a) - _assertRank2(a) + _assert_2d(a) m, n = a.shape t, result_t = _commonType(a) a = _fastCopyAndTranspose(t, a) @@ -1053,9 +1043,9 @@ def eigvals(a): """ a, wrap = _makearray(a) - _assertRankAtLeast2(a) - _assertNdSquareness(a) - _assertFinite(a) + _assert_stacked_2d(a) + _assert_stacked_square(a) + _assert_finite(a) t, result_t = _commonType(a) extobj = get_linalg_error_extobj( @@ -1163,8 +1153,8 @@ def eigvalsh(a, UPLO='L'): gufunc = _umath_linalg.eigvalsh_up a, wrap = _makearray(a) - _assertRankAtLeast2(a) - _assertNdSquareness(a) + _assert_stacked_2d(a) + _assert_stacked_square(a) t, result_t = _commonType(a) signature = 'D->d' if isComplexType(t) else 'd->d' w = gufunc(a, signature=signature, extobj=extobj) @@ -1300,9 +1290,9 @@ def eig(a): """ a, wrap = _makearray(a) - _assertRankAtLeast2(a) - _assertNdSquareness(a) - _assertFinite(a) + _assert_stacked_2d(a) + _assert_stacked_square(a) + _assert_finite(a) t, result_t = _commonType(a) extobj = get_linalg_error_extobj( @@ -1441,8 +1431,8 @@ def eigh(a, UPLO='L'): raise ValueError("UPLO argument must be 'L' or 'U'") a, wrap = _makearray(a) - _assertRankAtLeast2(a) - _assertNdSquareness(a) + _assert_stacked_2d(a) + _assert_stacked_square(a) t, result_t = _commonType(a) extobj = get_linalg_error_extobj( @@ -1614,7 +1604,7 @@ def svd(a, full_matrices=True, compute_uv=True, hermitian=False): s = abs(s) return s - _assertRankAtLeast2(a) + _assert_stacked_2d(a) t, result_t = _commonType(a) extobj = get_linalg_error_extobj(_raise_linalgerror_svd_nonconvergence) @@ -1730,12 +1720,13 @@ def cond(x, p=None): 1.4142135623730951 >>> LA.cond(a, -2) 0.70710678118654746 # may vary - >>> min(LA.svd(a, compute_uv=0))*min(LA.svd(LA.inv(a), compute_uv=0)) + >>> min(LA.svd(a, compute_uv=False))*min(LA.svd(LA.inv(a), compute_uv=False)) 0.70710678118654746 # may vary """ x = asarray(x) # in case we have a matrix - _assertNoEmpty2d(x) + if _is_empty_2d(x): + raise LinAlgError("cond is not defined on empty arrays") if p is None or p == 2 or p == -2: s = svd(x, compute_uv=False) with errstate(all='ignore'): @@ -1746,8 +1737,8 @@ def cond(x, p=None): else: # Call inv(x) ignoring errors. The result array will # contain nans in the entries where inversion failed. - _assertRankAtLeast2(x) - _assertNdSquareness(x) + _assert_stacked_2d(x) + _assert_stacked_square(x) t, result_t = _commonType(x) signature = 'D->D' if isComplexType(t) else 'd->d' with errstate(all='ignore'): @@ -1962,7 +1953,7 @@ def pinv(a, rcond=1e-15, hermitian=False): """ a, wrap = _makearray(a) rcond = asarray(rcond) - if _isEmpty2d(a): + if _is_empty_2d(a): m, n = a.shape[-2:] res = empty(a.shape[:-2] + (n, m), dtype=a.dtype) return wrap(res) @@ -2058,8 +2049,8 @@ def slogdet(a): """ a = asarray(a) - _assertRankAtLeast2(a) - _assertNdSquareness(a) + _assert_stacked_2d(a) + _assert_stacked_square(a) t, result_t = _commonType(a) real_t = _realType(result_t) signature = 'D->Dd' if isComplexType(t) else 'd->dd' @@ -2118,8 +2109,8 @@ def det(a): """ a = asarray(a) - _assertRankAtLeast2(a) - _assertNdSquareness(a) + _assert_stacked_2d(a) + _assert_stacked_square(a) t, result_t = _commonType(a) signature = 'D->D' if isComplexType(t) else 'd->d' r = _umath_linalg.det(a, signature=signature) @@ -2230,7 +2221,7 @@ def lstsq(a, b, rcond="warn"): is_1d = b.ndim == 1 if is_1d: b = b[:, newaxis] - _assertRank2(a, b) + _assert_2d(a, b) m, n = a.shape[-2:] m2, n_rhs = b.shape[-2:] if m != m2: @@ -2314,7 +2305,7 @@ def _multi_svd_norm(x, row_axis, col_axis, op): """ y = moveaxis(x, (row_axis, col_axis), (-2, -1)) - result = op(svd(y, compute_uv=0), axis=-1) + result = op(svd(y, compute_uv=False), axis=-1) return result @@ -2334,16 +2325,19 @@ def norm(x, ord=None, axis=None, keepdims=False): Parameters ---------- x : array_like - Input array. If `axis` is None, `x` must be 1-D or 2-D. + Input array. If `axis` is None, `x` must be 1-D or 2-D, unless `ord` + is None. If both `axis` and `ord` are None, the 2-norm of + ``x.ravel`` will be returned. ord : {non-zero int, inf, -inf, 'fro', 'nuc'}, optional Order of the norm (see table under ``Notes``). inf means numpy's - `inf` object. - axis : {int, 2-tuple of ints, None}, optional + `inf` object. The default is None. + axis : {None, int, 2-tuple of ints}, optional. If `axis` is an integer, it specifies the axis of `x` along which to compute the vector norms. If `axis` is a 2-tuple, it specifies the axes that hold 2-D matrices, and the matrix norms of these matrices are computed. If `axis` is None then either a vector norm (when `x` - is 1-D) or a matrix norm (when `x` is 2-D) is returned. + is 1-D) or a matrix norm (when `x` is 2-D) is returned. The default + is None. .. versionadded:: 1.8.0 @@ -2663,7 +2657,7 @@ def multi_dot(arrays): arrays[0] = atleast_2d(arrays[0]) if arrays[-1].ndim == 1: arrays[-1] = atleast_2d(arrays[-1]).T - _assertRank2(*arrays) + _assert_2d(*arrays) # _multi_dot_three is much faster than _multi_dot_matrix_chain_order if n == 3: diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py index 66c07c9e1..6315a34b4 100644 --- a/numpy/linalg/setup.py +++ b/numpy/linalg/setup.py @@ -26,7 +26,10 @@ def configuration(parent_package='', top_path=None): ] all_sources = config.paths(lapack_lite_src) - lapack_info = get_info('lapack_opt', 0) # and {} + if os.environ.get('NPY_USE_BLAS_ILP64', "0") != "0": + lapack_info = get_info('lapack_ilp64_opt', 2) + else: + lapack_info = get_info('lapack_opt', 0) # and {} def get_lapack_lite_sources(ext, build_dir): if not lapack_info: diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index 831c059d0..bd3df1ca4 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -20,8 +20,9 @@ from numpy.linalg.linalg import _multi_dot_matrix_chain_order from numpy.testing import ( assert_, assert_equal, assert_raises, assert_array_equal, assert_almost_equal, assert_allclose, suppress_warnings, - assert_raises_regex, + assert_raises_regex, HAS_LAPACK64, ) +from numpy.testing._private.utils import requires_memory def consistent_subclass(out, in_): @@ -650,7 +651,7 @@ class SVDBaseTests(object): class SVDCases(LinalgSquareTestCase, LinalgGeneralizedSquareTestCase): def do(self, a, b, tags): - u, s, vt = linalg.svd(a, 0) + u, s, vt = linalg.svd(a, False) assert_allclose(a, dot_generalized(np.asarray(u) * np.asarray(s)[..., None, :], np.asarray(vt)), rtol=get_rtol(u.dtype)) @@ -677,7 +678,7 @@ class TestSVD(SVDCases, SVDBaseTests): class SVDHermitianCases(HermitianTestCase, HermitianGeneralizedTestCase): def do(self, a, b, tags): - u, s, vt = linalg.svd(a, 0, hermitian=True) + u, s, vt = linalg.svd(a, False, hermitian=True) assert_allclose(a, dot_generalized(np.asarray(u) * np.asarray(s)[..., None, :], np.asarray(vt)), rtol=get_rtol(u.dtype)) @@ -897,7 +898,7 @@ class LstsqCases(LinalgSquareTestCase, LinalgNonsquareTestCase): def do(self, a, b, tags): arr = np.asarray(a) m, n = arr.shape - u, s, vt = linalg.svd(a, 0) + u, s, vt = linalg.svd(a, False) x, residuals, rank, sv = linalg.lstsq(a, b, rcond=-1) if m == 0: assert_((x == 0).all()) @@ -2002,3 +2003,16 @@ def test_unsupported_commontype(): arr = np.array([[1, -2], [2, 5]], dtype='float16') with assert_raises_regex(TypeError, "unsupported in linalg"): linalg.cholesky(arr) + + +@pytest.mark.slow +@pytest.mark.xfail(not HAS_LAPACK64, run=False, + reason="Numpy not compiled with 64-bit BLAS/LAPACK") +@requires_memory(free_bytes=16e9) +def test_blas64_dot(): + n = 2**32 + a = np.zeros([1, n], dtype=np.float32) + b = np.ones([1, 1], dtype=np.float32) + a[0,-1] = 1 + c = np.dot(b, a) + assert_equal(c[0,-1], 1) diff --git a/numpy/linalg/umath_linalg.c.src b/numpy/linalg/umath_linalg.c.src index 9fc68a7aa..e864c541b 100644 --- a/numpy/linalg/umath_linalg.c.src +++ b/numpy/linalg/umath_linalg.c.src @@ -15,6 +15,8 @@ #include "npy_config.h" +#include "npy_cblas.h" + #include <stddef.h> #include <stdio.h> #include <assert.h> @@ -62,301 +64,304 @@ dbg_stack_trace() ***************************************************************************** */ -#ifdef NO_APPEND_FORTRAN -# define FNAME(x) x -#else -# define FNAME(x) x##_ -#endif +#define FNAME(x) BLAS_FUNC(x) + +typedef CBLAS_INT fortran_int; typedef struct { float r, i; } f2c_complex; typedef struct { double r, i; } f2c_doublecomplex; /* typedef long int (*L_fp)(); */ -extern int -FNAME(sgeev)(char *jobvl, char *jobvr, int *n, - float a[], int *lda, float wr[], float wi[], - float vl[], int *ldvl, float vr[], int *ldvr, - float work[], int lwork[], - int *info); -extern int -FNAME(dgeev)(char *jobvl, char *jobvr, int *n, - double a[], int *lda, double wr[], double wi[], - double vl[], int *ldvl, double vr[], int *ldvr, - double work[], int lwork[], - int *info); -extern int -FNAME(cgeev)(char *jobvl, char *jobvr, int *n, - f2c_doublecomplex a[], int *lda, +typedef float fortran_real; +typedef double fortran_doublereal; +typedef f2c_complex fortran_complex; +typedef f2c_doublecomplex fortran_doublecomplex; + +extern fortran_int +FNAME(sgeev)(char *jobvl, char *jobvr, fortran_int *n, + float a[], fortran_int *lda, float wr[], float wi[], + float vl[], fortran_int *ldvl, float vr[], fortran_int *ldvr, + float work[], fortran_int lwork[], + fortran_int *info); +extern fortran_int +FNAME(dgeev)(char *jobvl, char *jobvr, fortran_int *n, + double a[], fortran_int *lda, double wr[], double wi[], + double vl[], fortran_int *ldvl, double vr[], fortran_int *ldvr, + double work[], fortran_int lwork[], + fortran_int *info); +extern fortran_int +FNAME(cgeev)(char *jobvl, char *jobvr, fortran_int *n, + f2c_doublecomplex a[], fortran_int *lda, f2c_doublecomplex w[], - f2c_doublecomplex vl[], int *ldvl, - f2c_doublecomplex vr[], int *ldvr, - f2c_doublecomplex work[], int *lwork, + f2c_doublecomplex vl[], fortran_int *ldvl, + f2c_doublecomplex vr[], fortran_int *ldvr, + f2c_doublecomplex work[], fortran_int *lwork, double rwork[], - int *info); -extern int -FNAME(zgeev)(char *jobvl, char *jobvr, int *n, - f2c_doublecomplex a[], int *lda, + fortran_int *info); +extern fortran_int +FNAME(zgeev)(char *jobvl, char *jobvr, fortran_int *n, + f2c_doublecomplex a[], fortran_int *lda, f2c_doublecomplex w[], - f2c_doublecomplex vl[], int *ldvl, - f2c_doublecomplex vr[], int *ldvr, - f2c_doublecomplex work[], int *lwork, + f2c_doublecomplex vl[], fortran_int *ldvl, + f2c_doublecomplex vr[], fortran_int *ldvr, + f2c_doublecomplex work[], fortran_int *lwork, double rwork[], - int *info); - -extern int -FNAME(ssyevd)(char *jobz, char *uplo, int *n, - float a[], int *lda, float w[], float work[], - int *lwork, int iwork[], int *liwork, - int *info); -extern int -FNAME(dsyevd)(char *jobz, char *uplo, int *n, - double a[], int *lda, double w[], double work[], - int *lwork, int iwork[], int *liwork, - int *info); -extern int -FNAME(cheevd)(char *jobz, char *uplo, int *n, - f2c_complex a[], int *lda, + fortran_int *info); + +extern fortran_int +FNAME(ssyevd)(char *jobz, char *uplo, fortran_int *n, + float a[], fortran_int *lda, float w[], float work[], + fortran_int *lwork, fortran_int iwork[], fortran_int *liwork, + fortran_int *info); +extern fortran_int +FNAME(dsyevd)(char *jobz, char *uplo, fortran_int *n, + double a[], fortran_int *lda, double w[], double work[], + fortran_int *lwork, fortran_int iwork[], fortran_int *liwork, + fortran_int *info); +extern fortran_int +FNAME(cheevd)(char *jobz, char *uplo, fortran_int *n, + f2c_complex a[], fortran_int *lda, float w[], f2c_complex work[], - int *lwork, float rwork[], int *lrwork, int iwork[], - int *liwork, - int *info); -extern int -FNAME(zheevd)(char *jobz, char *uplo, int *n, - f2c_doublecomplex a[], int *lda, + fortran_int *lwork, float rwork[], fortran_int *lrwork, fortran_int iwork[], + fortran_int *liwork, + fortran_int *info); +extern fortran_int +FNAME(zheevd)(char *jobz, char *uplo, fortran_int *n, + f2c_doublecomplex a[], fortran_int *lda, double w[], f2c_doublecomplex work[], - int *lwork, double rwork[], int *lrwork, int iwork[], - int *liwork, - int *info); - -extern int -FNAME(sgelsd)(int *m, int *n, int *nrhs, - float a[], int *lda, float b[], int *ldb, - float s[], float *rcond, int *rank, - float work[], int *lwork, int iwork[], - int *info); -extern int -FNAME(dgelsd)(int *m, int *n, int *nrhs, - double a[], int *lda, double b[], int *ldb, - double s[], double *rcond, int *rank, - double work[], int *lwork, int iwork[], - int *info); -extern int -FNAME(cgelsd)(int *m, int *n, int *nrhs, - f2c_complex a[], int *lda, - f2c_complex b[], int *ldb, - float s[], float *rcond, int *rank, - f2c_complex work[], int *lwork, - float rwork[], int iwork[], - int *info); -extern int -FNAME(zgelsd)(int *m, int *n, int *nrhs, - f2c_doublecomplex a[], int *lda, - f2c_doublecomplex b[], int *ldb, - double s[], double *rcond, int *rank, - f2c_doublecomplex work[], int *lwork, - double rwork[], int iwork[], - int *info); - -extern int -FNAME(sgesv)(int *n, int *nrhs, - float a[], int *lda, - int ipiv[], - float b[], int *ldb, - int *info); -extern int -FNAME(dgesv)(int *n, int *nrhs, - double a[], int *lda, - int ipiv[], - double b[], int *ldb, - int *info); -extern int -FNAME(cgesv)(int *n, int *nrhs, - f2c_complex a[], int *lda, - int ipiv[], - f2c_complex b[], int *ldb, - int *info); -extern int -FNAME(zgesv)(int *n, int *nrhs, - f2c_doublecomplex a[], int *lda, - int ipiv[], - f2c_doublecomplex b[], int *ldb, - int *info); - -extern int -FNAME(sgetrf)(int *m, int *n, - float a[], int *lda, - int ipiv[], - int *info); -extern int -FNAME(dgetrf)(int *m, int *n, - double a[], int *lda, - int ipiv[], - int *info); -extern int -FNAME(cgetrf)(int *m, int *n, - f2c_complex a[], int *lda, - int ipiv[], - int *info); -extern int -FNAME(zgetrf)(int *m, int *n, - f2c_doublecomplex a[], int *lda, - int ipiv[], - int *info); - -extern int -FNAME(spotrf)(char *uplo, int *n, - float a[], int *lda, - int *info); -extern int -FNAME(dpotrf)(char *uplo, int *n, - double a[], int *lda, - int *info); -extern int -FNAME(cpotrf)(char *uplo, int *n, - f2c_complex a[], int *lda, - int *info); -extern int -FNAME(zpotrf)(char *uplo, int *n, - f2c_doublecomplex a[], int *lda, - int *info); - -extern int -FNAME(sgesdd)(char *jobz, int *m, int *n, - float a[], int *lda, float s[], float u[], - int *ldu, float vt[], int *ldvt, float work[], - int *lwork, int iwork[], int *info); -extern int -FNAME(dgesdd)(char *jobz, int *m, int *n, - double a[], int *lda, double s[], double u[], - int *ldu, double vt[], int *ldvt, double work[], - int *lwork, int iwork[], int *info); -extern int -FNAME(cgesdd)(char *jobz, int *m, int *n, - f2c_complex a[], int *lda, - float s[], f2c_complex u[], int *ldu, - f2c_complex vt[], int *ldvt, - f2c_complex work[], int *lwork, - float rwork[], int iwork[], int *info); -extern int -FNAME(zgesdd)(char *jobz, int *m, int *n, - f2c_doublecomplex a[], int *lda, - double s[], f2c_doublecomplex u[], int *ldu, - f2c_doublecomplex vt[], int *ldvt, - f2c_doublecomplex work[], int *lwork, - double rwork[], int iwork[], int *info); - -extern int -FNAME(spotrs)(char *uplo, int *n, int *nrhs, - float a[], int *lda, - float b[], int *ldb, - int *info); -extern int -FNAME(dpotrs)(char *uplo, int *n, int *nrhs, - double a[], int *lda, - double b[], int *ldb, - int *info); -extern int -FNAME(cpotrs)(char *uplo, int *n, int *nrhs, - f2c_complex a[], int *lda, - f2c_complex b[], int *ldb, - int *info); -extern int -FNAME(zpotrs)(char *uplo, int *n, int *nrhs, - f2c_doublecomplex a[], int *lda, - f2c_doublecomplex b[], int *ldb, - int *info); - -extern int -FNAME(spotri)(char *uplo, int *n, - float a[], int *lda, - int *info); -extern int -FNAME(dpotri)(char *uplo, int *n, - double a[], int *lda, - int *info); -extern int -FNAME(cpotri)(char *uplo, int *n, - f2c_complex a[], int *lda, - int *info); -extern int -FNAME(zpotri)(char *uplo, int *n, - f2c_doublecomplex a[], int *lda, - int *info); - -extern int -FNAME(scopy)(int *n, - float *sx, int *incx, - float *sy, int *incy); -extern int -FNAME(dcopy)(int *n, - double *sx, int *incx, - double *sy, int *incy); -extern int -FNAME(ccopy)(int *n, - f2c_complex *sx, int *incx, - f2c_complex *sy, int *incy); -extern int -FNAME(zcopy)(int *n, - f2c_doublecomplex *sx, int *incx, - f2c_doublecomplex *sy, int *incy); + fortran_int *lwork, double rwork[], fortran_int *lrwork, fortran_int iwork[], + fortran_int *liwork, + fortran_int *info); + +extern fortran_int +FNAME(sgelsd)(fortran_int *m, fortran_int *n, fortran_int *nrhs, + float a[], fortran_int *lda, float b[], fortran_int *ldb, + float s[], float *rcond, fortran_int *rank, + float work[], fortran_int *lwork, fortran_int iwork[], + fortran_int *info); +extern fortran_int +FNAME(dgelsd)(fortran_int *m, fortran_int *n, fortran_int *nrhs, + double a[], fortran_int *lda, double b[], fortran_int *ldb, + double s[], double *rcond, fortran_int *rank, + double work[], fortran_int *lwork, fortran_int iwork[], + fortran_int *info); +extern fortran_int +FNAME(cgelsd)(fortran_int *m, fortran_int *n, fortran_int *nrhs, + f2c_complex a[], fortran_int *lda, + f2c_complex b[], fortran_int *ldb, + float s[], float *rcond, fortran_int *rank, + f2c_complex work[], fortran_int *lwork, + float rwork[], fortran_int iwork[], + fortran_int *info); +extern fortran_int +FNAME(zgelsd)(fortran_int *m, fortran_int *n, fortran_int *nrhs, + f2c_doublecomplex a[], fortran_int *lda, + f2c_doublecomplex b[], fortran_int *ldb, + double s[], double *rcond, fortran_int *rank, + f2c_doublecomplex work[], fortran_int *lwork, + double rwork[], fortran_int iwork[], + fortran_int *info); + +extern fortran_int +FNAME(sgesv)(fortran_int *n, fortran_int *nrhs, + float a[], fortran_int *lda, + fortran_int ipiv[], + float b[], fortran_int *ldb, + fortran_int *info); +extern fortran_int +FNAME(dgesv)(fortran_int *n, fortran_int *nrhs, + double a[], fortran_int *lda, + fortran_int ipiv[], + double b[], fortran_int *ldb, + fortran_int *info); +extern fortran_int +FNAME(cgesv)(fortran_int *n, fortran_int *nrhs, + f2c_complex a[], fortran_int *lda, + fortran_int ipiv[], + f2c_complex b[], fortran_int *ldb, + fortran_int *info); +extern fortran_int +FNAME(zgesv)(fortran_int *n, fortran_int *nrhs, + f2c_doublecomplex a[], fortran_int *lda, + fortran_int ipiv[], + f2c_doublecomplex b[], fortran_int *ldb, + fortran_int *info); + +extern fortran_int +FNAME(sgetrf)(fortran_int *m, fortran_int *n, + float a[], fortran_int *lda, + fortran_int ipiv[], + fortran_int *info); +extern fortran_int +FNAME(dgetrf)(fortran_int *m, fortran_int *n, + double a[], fortran_int *lda, + fortran_int ipiv[], + fortran_int *info); +extern fortran_int +FNAME(cgetrf)(fortran_int *m, fortran_int *n, + f2c_complex a[], fortran_int *lda, + fortran_int ipiv[], + fortran_int *info); +extern fortran_int +FNAME(zgetrf)(fortran_int *m, fortran_int *n, + f2c_doublecomplex a[], fortran_int *lda, + fortran_int ipiv[], + fortran_int *info); + +extern fortran_int +FNAME(spotrf)(char *uplo, fortran_int *n, + float a[], fortran_int *lda, + fortran_int *info); +extern fortran_int +FNAME(dpotrf)(char *uplo, fortran_int *n, + double a[], fortran_int *lda, + fortran_int *info); +extern fortran_int +FNAME(cpotrf)(char *uplo, fortran_int *n, + f2c_complex a[], fortran_int *lda, + fortran_int *info); +extern fortran_int +FNAME(zpotrf)(char *uplo, fortran_int *n, + f2c_doublecomplex a[], fortran_int *lda, + fortran_int *info); + +extern fortran_int +FNAME(sgesdd)(char *jobz, fortran_int *m, fortran_int *n, + float a[], fortran_int *lda, float s[], float u[], + fortran_int *ldu, float vt[], fortran_int *ldvt, float work[], + fortran_int *lwork, fortran_int iwork[], fortran_int *info); +extern fortran_int +FNAME(dgesdd)(char *jobz, fortran_int *m, fortran_int *n, + double a[], fortran_int *lda, double s[], double u[], + fortran_int *ldu, double vt[], fortran_int *ldvt, double work[], + fortran_int *lwork, fortran_int iwork[], fortran_int *info); +extern fortran_int +FNAME(cgesdd)(char *jobz, fortran_int *m, fortran_int *n, + f2c_complex a[], fortran_int *lda, + float s[], f2c_complex u[], fortran_int *ldu, + f2c_complex vt[], fortran_int *ldvt, + f2c_complex work[], fortran_int *lwork, + float rwork[], fortran_int iwork[], fortran_int *info); +extern fortran_int +FNAME(zgesdd)(char *jobz, fortran_int *m, fortran_int *n, + f2c_doublecomplex a[], fortran_int *lda, + double s[], f2c_doublecomplex u[], fortran_int *ldu, + f2c_doublecomplex vt[], fortran_int *ldvt, + f2c_doublecomplex work[], fortran_int *lwork, + double rwork[], fortran_int iwork[], fortran_int *info); + +extern fortran_int +FNAME(spotrs)(char *uplo, fortran_int *n, fortran_int *nrhs, + float a[], fortran_int *lda, + float b[], fortran_int *ldb, + fortran_int *info); +extern fortran_int +FNAME(dpotrs)(char *uplo, fortran_int *n, fortran_int *nrhs, + double a[], fortran_int *lda, + double b[], fortran_int *ldb, + fortran_int *info); +extern fortran_int +FNAME(cpotrs)(char *uplo, fortran_int *n, fortran_int *nrhs, + f2c_complex a[], fortran_int *lda, + f2c_complex b[], fortran_int *ldb, + fortran_int *info); +extern fortran_int +FNAME(zpotrs)(char *uplo, fortran_int *n, fortran_int *nrhs, + f2c_doublecomplex a[], fortran_int *lda, + f2c_doublecomplex b[], fortran_int *ldb, + fortran_int *info); + +extern fortran_int +FNAME(spotri)(char *uplo, fortran_int *n, + float a[], fortran_int *lda, + fortran_int *info); +extern fortran_int +FNAME(dpotri)(char *uplo, fortran_int *n, + double a[], fortran_int *lda, + fortran_int *info); +extern fortran_int +FNAME(cpotri)(char *uplo, fortran_int *n, + f2c_complex a[], fortran_int *lda, + fortran_int *info); +extern fortran_int +FNAME(zpotri)(char *uplo, fortran_int *n, + f2c_doublecomplex a[], fortran_int *lda, + fortran_int *info); + +extern fortran_int +FNAME(scopy)(fortran_int *n, + float *sx, fortran_int *incx, + float *sy, fortran_int *incy); +extern fortran_int +FNAME(dcopy)(fortran_int *n, + double *sx, fortran_int *incx, + double *sy, fortran_int *incy); +extern fortran_int +FNAME(ccopy)(fortran_int *n, + f2c_complex *sx, fortran_int *incx, + f2c_complex *sy, fortran_int *incy); +extern fortran_int +FNAME(zcopy)(fortran_int *n, + f2c_doublecomplex *sx, fortran_int *incx, + f2c_doublecomplex *sy, fortran_int *incy); extern float -FNAME(sdot)(int *n, - float *sx, int *incx, - float *sy, int *incy); +FNAME(sdot)(fortran_int *n, + float *sx, fortran_int *incx, + float *sy, fortran_int *incy); extern double -FNAME(ddot)(int *n, - double *sx, int *incx, - double *sy, int *incy); +FNAME(ddot)(fortran_int *n, + double *sx, fortran_int *incx, + double *sy, fortran_int *incy); extern void -FNAME(cdotu)(f2c_complex *ret, int *n, - f2c_complex *sx, int *incx, - f2c_complex *sy, int *incy); +FNAME(cdotu)(f2c_complex *ret, fortran_int *n, + f2c_complex *sx, fortran_int *incx, + f2c_complex *sy, fortran_int *incy); extern void -FNAME(zdotu)(f2c_doublecomplex *ret, int *n, - f2c_doublecomplex *sx, int *incx, - f2c_doublecomplex *sy, int *incy); +FNAME(zdotu)(f2c_doublecomplex *ret, fortran_int *n, + f2c_doublecomplex *sx, fortran_int *incx, + f2c_doublecomplex *sy, fortran_int *incy); extern void -FNAME(cdotc)(f2c_complex *ret, int *n, - f2c_complex *sx, int *incx, - f2c_complex *sy, int *incy); +FNAME(cdotc)(f2c_complex *ret, fortran_int *n, + f2c_complex *sx, fortran_int *incx, + f2c_complex *sy, fortran_int *incy); extern void -FNAME(zdotc)(f2c_doublecomplex *ret, int *n, - f2c_doublecomplex *sx, int *incx, - f2c_doublecomplex *sy, int *incy); +FNAME(zdotc)(f2c_doublecomplex *ret, fortran_int *n, + f2c_doublecomplex *sx, fortran_int *incx, + f2c_doublecomplex *sy, fortran_int *incy); -extern int +extern fortran_int FNAME(sgemm)(char *transa, char *transb, - int *m, int *n, int *k, + fortran_int *m, fortran_int *n, fortran_int *k, float *alpha, - float *a, int *lda, - float *b, int *ldb, + float *a, fortran_int *lda, + float *b, fortran_int *ldb, float *beta, - float *c, int *ldc); -extern int + float *c, fortran_int *ldc); +extern fortran_int FNAME(dgemm)(char *transa, char *transb, - int *m, int *n, int *k, + fortran_int *m, fortran_int *n, fortran_int *k, double *alpha, - double *a, int *lda, - double *b, int *ldb, + double *a, fortran_int *lda, + double *b, fortran_int *ldb, double *beta, - double *c, int *ldc); -extern int + double *c, fortran_int *ldc); +extern fortran_int FNAME(cgemm)(char *transa, char *transb, - int *m, int *n, int *k, + fortran_int *m, fortran_int *n, fortran_int *k, f2c_complex *alpha, - f2c_complex *a, int *lda, - f2c_complex *b, int *ldb, + f2c_complex *a, fortran_int *lda, + f2c_complex *b, fortran_int *ldb, f2c_complex *beta, - f2c_complex *c, int *ldc); -extern int + f2c_complex *c, fortran_int *ldc); +extern fortran_int FNAME(zgemm)(char *transa, char *transb, - int *m, int *n, int *k, + fortran_int *m, fortran_int *n, fortran_int *k, f2c_doublecomplex *alpha, - f2c_doublecomplex *a, int *lda, - f2c_doublecomplex *b, int *ldb, + f2c_doublecomplex *a, fortran_int *lda, + f2c_doublecomplex *b, fortran_int *ldb, f2c_doublecomplex *beta, - f2c_doublecomplex *c, int *ldc); + f2c_doublecomplex *c, fortran_int *ldc); #define LAPACK_T(FUNC) \ @@ -369,12 +374,6 @@ FNAME(zgemm)(char *transa, char *transb, #define LAPACK(FUNC) \ FNAME(FUNC) -typedef int fortran_int; -typedef float fortran_real; -typedef double fortran_doublereal; -typedef f2c_complex fortran_complex; -typedef f2c_doublecomplex fortran_doublecomplex; - /* ***************************************************************************** @@ -2522,8 +2521,6 @@ init_@lapack_func@(GESDD_PARAMS_t *params, params->VT = vt; params->RWORK = NULL; params->IWORK = iwork; - params->M = m; - params->N = n; params->LDA = ld; params->LDU = ld; params->LDVT = vt_column_count; diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 33ac936d0..2baf547a4 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -59,14 +59,14 @@ __all__ = [ 'choose', 'clip', 'common_fill_value', 'compress', 'compressed', 'concatenate', 'conjugate', 'convolve', 'copy', 'correlate', 'cos', 'cosh', 'count', 'cumprod', 'cumsum', 'default_fill_value', 'diag', 'diagonal', - 'diff', 'divide', 'dump', 'dumps', 'empty', 'empty_like', 'equal', 'exp', + 'diff', 'divide', 'empty', 'empty_like', 'equal', 'exp', 'expand_dims', 'fabs', 'filled', 'fix_invalid', 'flatten_mask', 'flatten_structured_array', 'floor', 'floor_divide', 'fmod', 'frombuffer', 'fromflex', 'fromfunction', 'getdata', 'getmask', 'getmaskarray', 'greater', 'greater_equal', 'harden_mask', 'hypot', 'identity', 'ids', 'indices', 'inner', 'innerproduct', 'isMA', 'isMaskedArray', 'is_mask', 'is_masked', 'isarray', 'left_shift', - 'less', 'less_equal', 'load', 'loads', 'log', 'log10', 'log2', + 'less', 'less_equal', 'log', 'log10', 'log2', 'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'make_mask', 'make_mask_descr', 'make_mask_none', 'mask_or', 'masked', 'masked_array', 'masked_equal', 'masked_greater', @@ -77,7 +77,7 @@ __all__ = [ 'maximum_fill_value', 'mean', 'min', 'minimum', 'minimum_fill_value', 'mod', 'multiply', 'mvoid', 'ndim', 'negative', 'nomask', 'nonzero', 'not_equal', 'ones', 'outer', 'outerproduct', 'power', 'prod', - 'product', 'ptp', 'put', 'putmask', 'rank', 'ravel', 'remainder', + 'product', 'ptp', 'put', 'putmask', 'ravel', 'remainder', 'repeat', 'reshape', 'resize', 'right_shift', 'round', 'round_', 'set_fill_value', 'shape', 'sin', 'sinh', 'size', 'soften_mask', 'sometrue', 'sort', 'sqrt', 'squeeze', 'std', 'subtract', 'sum', @@ -602,8 +602,10 @@ def filled(a, fill_value=None): ---------- a : MaskedArray or array_like An input object. - fill_value : scalar, optional - Filling value. Default is None. + fill_value : array_like, optional. + Can be scalar or non-scalar. If non-scalar, the + resulting filled array should be broadcastable + over input array. Default is None. Returns ------- @@ -623,10 +625,19 @@ def filled(a, fill_value=None): array([[999999, 1, 2], [999999, 4, 5], [ 6, 7, 8]]) + >>> x.filled(fill_value=333) + array([[333, 1, 2], + [333, 4, 5], + [ 6, 7, 8]]) + >>> x.filled(fill_value=np.arange(3)) + array([[0, 1, 2], + [0, 4, 5], + [6, 7, 8]]) """ if hasattr(a, 'filled'): return a.filled(fill_value) + elif isinstance(a, ndarray): # Should we check for contiguity ? and a.flags['CONTIGUOUS']: return a @@ -800,7 +811,7 @@ class _DomainCheckInterval(object): def __init__(self, a, b): "domain_check_interval(a,b)(x) = true where x < a or y > b" - if (a > b): + if a > b: (a, b) = (b, a) self.a = a self.b = b @@ -1060,7 +1071,7 @@ class _MaskedBinaryOperation(_MaskedUFunc): if t.shape == (): t = t.reshape(1) if m is not nomask: - m = make_mask(m, copy=1) + m = make_mask(m, copy=True) m.shape = (1,) if m is nomask: @@ -1165,7 +1176,7 @@ class _DomainedBinaryOperation(_MaskedUFunc): if domain is not None: m |= domain(da, db) # Take care of the scalar case first - if (not m.ndim): + if not m.ndim: if m: return masked else: @@ -1554,7 +1565,7 @@ def make_mask(m, copy=False, shrink=True, dtype=MaskType): Return `m` as a boolean mask, creating a copy if necessary or requested. The function can accept any sequence that is convertible to integers, or ``nomask``. Does not require that contents must be 0s and 1s, values - of 0 are interepreted as False, everything else as True. + of 0 are interpreted as False, everything else as True. Parameters ---------- @@ -1743,7 +1754,7 @@ def mask_or(m1, m2, copy=False, shrink=True): if m1 is m2 and is_mask(m1): return m1 (dtype1, dtype2) = (getattr(m1, 'dtype', None), getattr(m2, 'dtype', None)) - if (dtype1 != dtype2): + if dtype1 != dtype2: raise ValueError("Incompatible dtypes '%s'<>'%s'" % (dtype1, dtype2)) if dtype1.names is not None: # Allocate an output mask array with the properly broadcast shape. @@ -2681,15 +2692,13 @@ class MaskedIterator(object): -------- >>> x = np.ma.array([3, 2], mask=[0, 1]) >>> fl = x.flat - >>> fl.next() + >>> next(fl) 3 - >>> fl.next() + >>> next(fl) masked - >>> fl.next() + >>> next(fl) Traceback (most recent call last): - File "<stdin>", line 1, in <module> - File "/home/ralf/python/numpy/numpy/ma/core.py", line 2243, in next - d = self.dataiter.next() + ... StopIteration """ @@ -3077,7 +3086,7 @@ class MaskedArray(ndarray): def view(self, dtype=None, type=None, fill_value=None): """ - Return a view of the MaskedArray data + Return a view of the MaskedArray data. Parameters ---------- @@ -3091,6 +3100,14 @@ class MaskedArray(ndarray): type : Python type, optional Type of the returned view, either ndarray or a subclass. The default None results in type preservation. + fill_value : scalar, optional + The value to use for invalid entries (None by default). + If None, then this argument is inferred from the passed `dtype`, or + in its absence the original array, as discussed in the notes below. + + See Also + -------- + numpy.ndarray.view : Equivalent method on ndarray object. Notes ----- @@ -3143,7 +3160,7 @@ class MaskedArray(ndarray): # also make the mask be a view (so attr changes to the view's # mask do no affect original object's mask) # (especially important to avoid affecting np.masked singleton) - if (getmask(output) is not nomask): + if getmask(output) is not nomask: output._mask = output._mask.view() # Make sure to reset the _fill_value if needed @@ -3156,7 +3173,6 @@ class MaskedArray(ndarray): else: output.fill_value = fill_value return output - view.__doc__ = ndarray.view.__doc__ def __getitem__(self, indx): """ @@ -3382,7 +3398,7 @@ class MaskedArray(ndarray): if mask is masked: mask = True - if (current_mask is nomask): + if current_mask is nomask: # Make sure the mask is set # Just don't do anything if there's nothing to do. if mask is nomask: @@ -3648,6 +3664,14 @@ class MaskedArray(ndarray): @fill_value.setter def fill_value(self, value=None): target = _check_fill_value(value, self.dtype) + if not target.ndim == 0: + # 2019-11-12, 1.18.0 + warnings.warn( + "Non-scalar arrays for the fill value are deprecated. Use " + "arrays with scalar values instead. The filled function " + "still supports any array as `fill_value`.", + DeprecationWarning, stacklevel=2) + _fill_value = self._fill_value if _fill_value is None: # Create the attribute if it was undefined @@ -3668,9 +3692,11 @@ class MaskedArray(ndarray): Parameters ---------- - fill_value : scalar, optional - The value to use for invalid entries (None by default). - If None, the `fill_value` attribute of the array is used instead. + fill_value : array_like, optional + The value to use for invalid entries. Can be scalar or non-scalar. + If non-scalar, the resulting ndarray must be broadcastable over + input array. Default is None, in which case, the `fill_value` + attribute of the array is used instead. Returns ------- @@ -3689,6 +3715,8 @@ class MaskedArray(ndarray): >>> x = np.ma.array([1,2,3,4,5], mask=[0,0,1,0,1], fill_value=-999) >>> x.filled() array([ 1, 2, -999, 4, -999]) + >>> x.filled(fill_value=1000) + array([ 1, 2, 1000, 4, 1000]) >>> type(x.filled()) <class 'numpy.ndarray'> @@ -4389,7 +4417,7 @@ class MaskedArray(ndarray): ---------- axis : None or int or tuple of ints, optional Axis or axes along which the count is performed. - The default (`axis` = `None`) performs the count over all + The default, None, performs the count over all the dimensions of the input array. `axis` may be negative, in which case it counts from the last to the first axis. @@ -4769,7 +4797,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.all : corresponding function for ndarrays + numpy.ndarray.all : corresponding function for ndarrays numpy.all : equivalent function Examples @@ -4807,7 +4835,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.any : corresponding function for ndarrays + numpy.ndarray.any : corresponding function for ndarrays numpy.any : equivalent function """ @@ -4861,7 +4889,7 @@ class MaskedArray(ndarray): flatnonzero : Return indices that are non-zero in the flattened version of the input array. - ndarray.nonzero : + numpy.ndarray.nonzero : Equivalent ndarray method. count_nonzero : Counts the number of non-zero elements in the input array. @@ -4989,7 +5017,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.sum : corresponding function for ndarrays + numpy.ndarray.sum : corresponding function for ndarrays numpy.sum : equivalent function Examples @@ -5036,7 +5064,7 @@ class MaskedArray(ndarray): result = self.filled(0).sum(axis, dtype=dtype, out=out, **kwargs) if isinstance(out, MaskedArray): outmask = getmask(out) - if (outmask is nomask): + if outmask is nomask: outmask = out._mask = make_mask_none(out.shape) outmask.flat = newmask return out @@ -5060,7 +5088,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.cumsum : corresponding function for ndarrays + numpy.ndarray.cumsum : corresponding function for ndarrays numpy.cumsum : equivalent function Examples @@ -5097,7 +5125,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.prod : corresponding function for ndarrays + numpy.ndarray.prod : corresponding function for ndarrays numpy.prod : equivalent function """ kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims} @@ -5118,7 +5146,7 @@ class MaskedArray(ndarray): result = self.filled(1).prod(axis, dtype=dtype, out=out, **kwargs) if isinstance(out, MaskedArray): outmask = getmask(out) - if (outmask is nomask): + if outmask is nomask: outmask = out._mask = make_mask_none(out.shape) outmask.flat = newmask return out @@ -5143,7 +5171,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.cumprod : corresponding function for ndarrays + numpy.ndarray.cumprod : corresponding function for ndarrays numpy.cumprod : equivalent function """ result = self.filled(1).cumprod(axis=axis, dtype=dtype, out=out) @@ -5166,7 +5194,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.mean : corresponding function for ndarrays + numpy.ndarray.mean : corresponding function for ndarrays numpy.mean : Equivalent function numpy.ma.average: Weighted average. @@ -5197,7 +5225,7 @@ class MaskedArray(ndarray): out.flat = result if isinstance(out, MaskedArray): outmask = getmask(out) - if (outmask is nomask): + if outmask is nomask: outmask = out._mask = make_mask_none(out.shape) outmask.flat = getmask(result) return out @@ -5239,9 +5267,9 @@ class MaskedArray(ndarray): return m if not axis: - return (self - m) + return self - m else: - return (self - expand_dims(m, axis)) + return self - expand_dims(m, axis) def var(self, axis=None, dtype=None, out=None, ddof=0, keepdims=np._NoValue): @@ -5255,7 +5283,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.var : corresponding function for ndarrays + numpy.ndarray.var : corresponding function for ndarrays numpy.var : Equivalent function """ kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims} @@ -5318,7 +5346,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.std : corresponding function for ndarrays + numpy.ndarray.std : corresponding function for ndarrays numpy.std : Equivalent function """ kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims} @@ -5339,7 +5367,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.around : corresponding function for ndarrays + numpy.ndarray.around : corresponding function for ndarrays numpy.around : equivalent function """ result = self._data.round(decimals=decimals, out=out).view(type(self)) @@ -5401,7 +5429,7 @@ class MaskedArray(ndarray): -------- MaskedArray.sort : Describes sorting algorithms used. lexsort : Indirect stable sort with multiple keys. - ndarray.sort : Inplace sort. + numpy.ndarray.sort : Inplace sort. Notes ----- @@ -5553,7 +5581,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.sort : Method to sort an array in-place. + numpy.ndarray.sort : Method to sort an array in-place. argsort : Indirect sort. lexsort : Indirect stable sort on multiple keys. searchsorted : Find elements in a sorted array. @@ -5616,6 +5644,10 @@ class MaskedArray(ndarray): fill_value : {var}, optional Value used to fill in the masked values. If None, use the output of `minimum_fill_value`. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left + in the result as dimensions with size one. With this option, + the result will broadcast correctly against the array. Returns ------- @@ -5652,7 +5684,7 @@ class MaskedArray(ndarray): result = self.filled(fill_value).min(axis=axis, out=out, **kwargs) if isinstance(out, MaskedArray): outmask = getmask(out) - if (outmask is nomask): + if outmask is nomask: outmask = out._mask = make_mask_none(out.shape) outmask.flat = newmask else: @@ -5746,6 +5778,10 @@ class MaskedArray(ndarray): fill_value : {var}, optional Value used to fill in the masked values. If None, use the output of maximum_fill_value(). + keepdims : bool, optional + If this is set to True, the axes which are reduced are left + in the result as dimensions with size one. With this option, + the result will broadcast correctly against the array. Returns ------- @@ -5782,7 +5818,7 @@ class MaskedArray(ndarray): result = self.filled(fill_value).max(axis=axis, out=out, **kwargs) if isinstance(out, MaskedArray): outmask = getmask(out) - if (outmask is nomask): + if outmask is nomask: outmask = out._mask = make_mask_none(out.shape) outmask.flat = newmask else: @@ -5810,6 +5846,10 @@ class MaskedArray(ndarray): but the type will be cast if necessary. fill_value : {var}, optional Value used to fill in the masked values. + keepdims : bool, optional + If this is set to True, the axes which are reduced are left + in the result as dimensions with size one. With this option, + the result will broadcast correctly against the array. Returns ------- @@ -5961,7 +6001,7 @@ class MaskedArray(ndarray): See Also -------- - ndarray.tobytes + numpy.ndarray.tobytes tolist, tofile Notes @@ -6203,9 +6243,11 @@ class mvoid(MaskedArray): Parameters ---------- - fill_value : scalar, optional - The value to use for invalid entries (None by default). - If None, the `fill_value` attribute is used instead. + fill_value : array_like, optional + The value to use for invalid entries. Can be scalar or + non-scalar. If latter is the case, the filled array should + be broadcastable over input array. Default is None, in + which case the `fill_value` attribute is used instead. Returns ------- @@ -6720,7 +6762,7 @@ def power(a, b, third=None): invalid = np.logical_not(np.isfinite(result.view(ndarray))) # Add the initial mask if m is not nomask: - if not (result.ndim): + if not result.ndim: return masked result._mask = np.logical_or(m, invalid) # Fix the invalid parts @@ -7081,23 +7123,6 @@ def resize(x, new_shape): return result -def rank(obj): - """ - maskedarray version of the numpy function. - - .. note:: - Deprecated since 1.10.0 - - """ - # 2015-04-12, 1.10.0 - warnings.warn( - "`rank` is deprecated; use the `ndim` function instead. ", - np.VisibleDeprecationWarning, stacklevel=2) - return np.ndim(getdata(obj)) - -rank.__doc__ = np.rank.__doc__ - - def ndim(obj): """ maskedarray version of the numpy function. @@ -7280,7 +7305,7 @@ def choose(indices, choices, out=None, mode='raise'): # Construct the mask outputmask = np.choose(c, masks, mode=mode) outputmask = make_mask(mask_or(outputmask, getmask(indices)), - copy=0, shrink=True) + copy=False, shrink=True) # Get the choices. d = np.choose(c, data, mode=mode, out=out).view(MaskedArray) if out is not None: @@ -7532,7 +7557,7 @@ def outer(a, b): return masked_array(d) ma = getmaskarray(a) mb = getmaskarray(b) - m = make_mask(1 - np.outer(1 - ma, 1 - mb), copy=0) + m = make_mask(1 - np.outer(1 - ma, 1 - mb), copy=False) return masked_array(d, mask=m) outer.__doc__ = doc_note(np.outer.__doc__, "Masked values are replaced by 0.") @@ -7886,93 +7911,6 @@ def _pickle_warn(method): stacklevel=3) -def dump(a, F): - """ - Pickle a masked array to a file. - - This is a wrapper around ``cPickle.dump``. - - Parameters - ---------- - a : MaskedArray - The array to be pickled. - F : str or file-like object - The file to pickle `a` to. If a string, the full path to the file. - - """ - _pickle_warn('dump') - if not hasattr(F, 'readline'): - with open(F, 'w') as F: - pickle.dump(a, F) - else: - pickle.dump(a, F) - - -def dumps(a): - """ - Return a string corresponding to the pickling of a masked array. - - This is a wrapper around ``cPickle.dumps``. - - Parameters - ---------- - a : MaskedArray - The array for which the string representation of the pickle is - returned. - - """ - _pickle_warn('dumps') - return pickle.dumps(a) - - -def load(F): - """ - Wrapper around ``cPickle.load`` which accepts either a file-like object - or a filename. - - Parameters - ---------- - F : str or file - The file or file name to load. - - See Also - -------- - dump : Pickle an array - - Notes - ----- - This is different from `numpy.load`, which does not use cPickle but loads - the NumPy binary .npy format. - - """ - _pickle_warn('load') - if not hasattr(F, 'readline'): - with open(F, 'r') as F: - return pickle.load(F) - else: - return pickle.load(F) - - -def loads(strg): - """ - Load a pickle from the current string. - - The result of ``cPickle.loads(strg)`` is returned. - - Parameters - ---------- - strg : str - The string to load. - - See Also - -------- - dumps : Return a string corresponding to the pickling of a masked array. - - """ - _pickle_warn('loads') - return pickle.loads(strg) - - def fromfile(file, dtype=float, count=-1, sep=''): raise NotImplementedError( "fromfile() not yet implemented for a MaskedArray.") diff --git a/numpy/ma/extras.py b/numpy/ma/extras.py index 639b3dd1f..f4a914471 100644 --- a/numpy/ma/extras.py +++ b/numpy/ma/extras.py @@ -542,15 +542,18 @@ def average(a, axis=None, weights=None, returned=False): Data to be averaged. Masked entries are not taken into account in the computation. axis : int, optional - Axis along which to average `a`. If `None`, averaging is done over + Axis along which to average `a`. If None, averaging is done over the flattened array. weights : array_like, optional The importance that each element has in the computation of the average. The weights array can either be 1-D (in which case its length must be the size of `a` along the given axis) or of the same shape as `a`. If ``weights=None``, then all data in `a` are assumed to have a - weight equal to one. If `weights` is complex, the imaginary parts - are ignored. + weight equal to one. The 1-D calculation is:: + + avg = sum(a * weights) / sum(weights) + + The only constraint on `weights` is that `sum(weights)` must not be 0. returned : bool, optional Flag indicating whether a tuple ``(result, sum of weights)`` should be returned as output (True), or just the result (False). @@ -934,7 +937,7 @@ def compress_cols(a): raise NotImplementedError("compress_cols works for 2D arrays only.") return compress_rowcols(a, 1) -def mask_rows(a, axis=None): +def mask_rows(a, axis=np._NoValue): """ Mask rows of a 2D array that contain masked values. @@ -976,9 +979,15 @@ def mask_rows(a, axis=None): fill_value=1) """ + if axis is not np._NoValue: + # remove the axis argument when this deprecation expires + # NumPy 1.18.0, 2019-11-28 + warnings.warn( + "The axis argument has always been ignored, in future passing it " + "will raise TypeError", DeprecationWarning, stacklevel=2) return mask_rowcols(a, 0) -def mask_cols(a, axis=None): +def mask_cols(a, axis=np._NoValue): """ Mask columns of a 2D array that contain masked values. @@ -1019,6 +1028,12 @@ def mask_cols(a, axis=None): fill_value=1) """ + if axis is not np._NoValue: + # remove the axis argument when this deprecation expires + # NumPy 1.18.0, 2019-11-28 + warnings.warn( + "The axis argument has always been ignored, in future passing it " + "will raise TypeError", DeprecationWarning, stacklevel=2) return mask_rowcols(a, 1) diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index cf11b6096..b72ce56aa 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -347,7 +347,7 @@ class TestMaskedArray(object): m = make_mask(n) m2 = make_mask(m) assert_(m is m2) - m3 = make_mask(m, copy=1) + m3 = make_mask(m, copy=True) assert_(m is not m3) x1 = np.arange(5) diff --git a/numpy/ma/tests/test_extras.py b/numpy/ma/tests/test_extras.py index afcfd126e..c75c47801 100644 --- a/numpy/ma/tests/test_extras.py +++ b/numpy/ma/tests/test_extras.py @@ -11,6 +11,7 @@ from __future__ import division, absolute_import, print_function import warnings import itertools +import pytest import numpy as np from numpy.testing import ( @@ -148,7 +149,7 @@ class TestAverage(object): ott = array([0., 1., 2., 3.], mask=[True, False, False, False]) assert_equal(2.0, average(ott, axis=0)) assert_equal(2.0, average(ott, weights=[1., 1., 2., 1.])) - result, wts = average(ott, weights=[1., 1., 2., 1.], returned=1) + result, wts = average(ott, weights=[1., 1., 2., 1.], returned=True) assert_equal(2.0, result) assert_(wts == 4.0) ott[:] = masked @@ -159,7 +160,7 @@ class TestAverage(object): assert_equal(average(ott, axis=0), [2.0, 0.0]) assert_equal(average(ott, axis=1).mask[0], [True]) assert_equal([2., 0.], average(ott, axis=0)) - result, wts = average(ott, axis=0, returned=1) + result, wts = average(ott, axis=0, returned=True) assert_equal(wts, [1., 0.]) def test_testAverage2(self): @@ -200,14 +201,14 @@ class TestAverage(object): # Yet more tests of average! a = arange(6) b = arange(6) * 3 - r1, w1 = average([[a, b], [b, a]], axis=1, returned=1) + r1, w1 = average([[a, b], [b, a]], axis=1, returned=True) assert_equal(shape(r1), shape(w1)) assert_equal(r1.shape, w1.shape) - r2, w2 = average(ones((2, 2, 3)), axis=0, weights=[3, 1], returned=1) + r2, w2 = average(ones((2, 2, 3)), axis=0, weights=[3, 1], returned=True) assert_equal(shape(w2), shape(r2)) - r2, w2 = average(ones((2, 2, 3)), returned=1) + r2, w2 = average(ones((2, 2, 3)), returned=True) assert_equal(shape(w2), shape(r2)) - r2, w2 = average(ones((2, 2, 3)), weights=ones((2, 2, 3)), returned=1) + r2, w2 = average(ones((2, 2, 3)), weights=ones((2, 2, 3)), returned=True) assert_equal(shape(w2), shape(r2)) a2d = array([[1, 2], [0, 4]], float) a2dm = masked_array(a2d, [[False, False], [True, False]]) @@ -552,6 +553,18 @@ class TestCompressFunctions(object): assert_(mask_rowcols(x, 0).mask.all()) assert_(mask_rowcols(x, 1).mask.all()) + @pytest.mark.parametrize("axis", [None, 0, 1]) + @pytest.mark.parametrize(["func", "rowcols_axis"], + [(np.ma.mask_rows, 0), (np.ma.mask_cols, 1)]) + def test_mask_row_cols_axis_deprecation(self, axis, func, rowcols_axis): + # Test deprecation of the axis argument to `mask_rows` and `mask_cols` + x = array(np.arange(9).reshape(3, 3), + mask=[[1, 0, 0], [0, 0, 0], [0, 0, 0]]) + + with assert_warns(DeprecationWarning): + res = func(x, axis=axis) + assert_equal(res, mask_rowcols(x, rowcols_axis)) + def test_dot(self): # Tests dot product n = np.arange(1, 7) diff --git a/numpy/ma/tests/test_old_ma.py b/numpy/ma/tests/test_old_ma.py index 1c523768d..7100eccbb 100644 --- a/numpy/ma/tests/test_old_ma.py +++ b/numpy/ma/tests/test_old_ma.py @@ -263,7 +263,7 @@ class TestMa(object): m = make_mask(n) m2 = make_mask(m) assert_(m is m2) - m3 = make_mask(m, copy=1) + m3 = make_mask(m, copy=True) assert_(m is not m3) x1 = np.arange(5) @@ -570,7 +570,7 @@ class TestMa(object): ott = array([0., 1., 2., 3.], mask=[1, 0, 0, 0]) assert_(eq(2.0, average(ott, axis=0))) assert_(eq(2.0, average(ott, weights=[1., 1., 2., 1.]))) - result, wts = average(ott, weights=[1., 1., 2., 1.], returned=1) + result, wts = average(ott, weights=[1., 1., 2., 1.], returned=True) assert_(eq(2.0, result)) assert_(wts == 4.0) ott[:] = masked @@ -581,7 +581,7 @@ class TestMa(object): assert_(eq(average(ott, axis=0), [2.0, 0.0])) assert_(average(ott, axis=1)[0] is masked) assert_(eq([2., 0.], average(ott, axis=0))) - result, wts = average(ott, axis=0, returned=1) + result, wts = average(ott, axis=0, returned=True) assert_(eq(wts, [1., 0.])) def test_testAverage2(self): @@ -622,14 +622,14 @@ class TestMa(object): a = arange(6) b = arange(6) * 3 - r1, w1 = average([[a, b], [b, a]], axis=1, returned=1) + r1, w1 = average([[a, b], [b, a]], axis=1, returned=True) assert_equal(shape(r1), shape(w1)) assert_equal(r1.shape, w1.shape) - r2, w2 = average(ones((2, 2, 3)), axis=0, weights=[3, 1], returned=1) + r2, w2 = average(ones((2, 2, 3)), axis=0, weights=[3, 1], returned=True) assert_equal(shape(w2), shape(r2)) - r2, w2 = average(ones((2, 2, 3)), returned=1) + r2, w2 = average(ones((2, 2, 3)), returned=True) assert_equal(shape(w2), shape(r2)) - r2, w2 = average(ones((2, 2, 3)), weights=ones((2, 2, 3)), returned=1) + r2, w2 = average(ones((2, 2, 3)), weights=ones((2, 2, 3)), returned=True) assert_(shape(w2) == shape(r2)) a2d = array([[1, 2], [0, 4]], float) a2dm = masked_array(a2d, [[0, 0], [1, 0]]) diff --git a/numpy/ma/version.py b/numpy/ma/version.py deleted file mode 100644 index a2c5c42a8..000000000 --- a/numpy/ma/version.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Version number - -""" -from __future__ import division, absolute_import, print_function - -version = '1.00' -release = False - -if not release: - from . import core - from . import extras - revision = [core.__revision__.split(':')[-1][:-1].strip(), - extras.__revision__.split(':')[-1][:-1].strip(),] - version += '.dev%04i' % max([int(rev) for rev in revision]) diff --git a/numpy/matlib.py b/numpy/matlib.py index 9e115943a..b1b155586 100644 --- a/numpy/matlib.py +++ b/numpy/matlib.py @@ -2,7 +2,7 @@ from __future__ import division, absolute_import, print_function import numpy as np from numpy.matrixlib.defmatrix import matrix, asmatrix -# need * as we're copying the numpy namespace +# need * as we're copying the numpy namespace (FIXME: this makes little sense) from numpy import * __version__ = np.__version__ @@ -239,7 +239,7 @@ def rand(*args): See Also -------- - randn, numpy.random.rand + randn, numpy.random.RandomState.rand Examples -------- @@ -285,7 +285,7 @@ def randn(*args): See Also -------- - rand, random.randn + rand, numpy.random.RandomState.randn Notes ----- diff --git a/numpy/matrixlib/defmatrix.py b/numpy/matrixlib/defmatrix.py index 3c7e8ffc2..cabd41367 100644 --- a/numpy/matrixlib/defmatrix.py +++ b/numpy/matrixlib/defmatrix.py @@ -1046,7 +1046,7 @@ def bmat(obj, ldict=None, gdict=None): referenced by name. ldict : dict, optional A dictionary that replaces local operands in current frame. - Ignored if `obj` is not a string or `gdict` is `None`. + Ignored if `obj` is not a string or `gdict` is None. gdict : dict, optional A dictionary that replaces global operands in current frame. Ignored if `obj` is not a string. diff --git a/numpy/polynomial/chebyshev.py b/numpy/polynomial/chebyshev.py index e4d10bcb8..0cd9c4d23 100644 --- a/numpy/polynomial/chebyshev.py +++ b/numpy/polynomial/chebyshev.py @@ -909,7 +909,7 @@ def chebder(c, m=1, scl=1, axis=0): array([12., 96.]) """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) cnt = pu._deprecate_as_int(m, "the order of derivation") @@ -1026,7 +1026,7 @@ def chebint(c, m=1, k=[], lbnd=0, scl=1, axis=0): array([-1., 1., -1., -1.]) """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) if not np.iterable(k): @@ -1131,7 +1131,7 @@ def chebval(x, c, tensor=True): -------- """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) if isinstance(x, (tuple, list)): @@ -1404,7 +1404,7 @@ def chebvander(x, deg): if ideg < 0: raise ValueError("deg must be non-negative") - x = np.array(x, copy=0, ndmin=1) + 0.0 + x = np.array(x, copy=False, ndmin=1) + 0.0 dims = (ideg + 1,) + x.shape dtyp = x.dtype v = np.empty(dims, dtype=dtyp) @@ -1468,7 +1468,7 @@ def chebvander2d(x, y, deg): .. versionadded:: 1.7.0 """ - return pu._vander2d(chebvander, x, y, deg) + return pu._vander_nd_flat((chebvander, chebvander), (x, y), deg) def chebvander3d(x, y, z, deg): @@ -1522,7 +1522,7 @@ def chebvander3d(x, y, z, deg): .. versionadded:: 1.7.0 """ - return pu._vander3d(chebvander, x, y, z, deg) + return pu._vander_nd_flat((chebvander, chebvander, chebvander), (x, y, z), deg) def chebfit(x, y, deg, rcond=None, full=False, w=None): diff --git a/numpy/polynomial/hermite.py b/numpy/polynomial/hermite.py index 4bfd89e52..9b1aea239 100644 --- a/numpy/polynomial/hermite.py +++ b/numpy/polynomial/hermite.py @@ -627,7 +627,7 @@ def hermder(c, m=1, scl=1, axis=0): array([1., 2., 3.]) """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) cnt = pu._deprecate_as_int(m, "the order of derivation") @@ -738,7 +738,7 @@ def hermint(c, m=1, k=[], lbnd=0, scl=1, axis=0): array([ 1.66666667, -0.5 , 0.125 , 0.08333333, 0.0625 ]) # may vary """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) if not np.iterable(k): @@ -846,7 +846,7 @@ def hermval(x, c, tensor=True): [115., 203.]]) """ - c = np.array(c, ndmin=1, copy=0) + c = np.array(c, ndmin=1, copy=False) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) if isinstance(x, (tuple, list)): @@ -1130,7 +1130,7 @@ def hermvander(x, deg): if ideg < 0: raise ValueError("deg must be non-negative") - x = np.array(x, copy=0, ndmin=1) + 0.0 + x = np.array(x, copy=False, ndmin=1) + 0.0 dims = (ideg + 1,) + x.shape dtyp = x.dtype v = np.empty(dims, dtype=dtyp) @@ -1193,7 +1193,7 @@ def hermvander2d(x, y, deg): .. versionadded:: 1.7.0 """ - return pu._vander2d(hermvander, x, y, deg) + return pu._vander_nd_flat((hermvander, hermvander), (x, y), deg) def hermvander3d(x, y, z, deg): @@ -1247,7 +1247,7 @@ def hermvander3d(x, y, z, deg): .. versionadded:: 1.7.0 """ - return pu._vander3d(hermvander, x, y, z, deg) + return pu._vander_nd_flat((hermvander, hermvander, hermvander), (x, y, z), deg) def hermfit(x, y, deg, rcond=None, full=False, w=None): diff --git a/numpy/polynomial/hermite_e.py b/numpy/polynomial/hermite_e.py index 735d66e6c..c5a0a05a2 100644 --- a/numpy/polynomial/hermite_e.py +++ b/numpy/polynomial/hermite_e.py @@ -622,7 +622,7 @@ def hermeder(c, m=1, scl=1, axis=0): array([1., 2., 3.]) """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) cnt = pu._deprecate_as_int(m, "the order of derivation") @@ -733,7 +733,7 @@ def hermeint(c, m=1, k=[], lbnd=0, scl=1, axis=0): array([ 1.83333333, 0. , 0.5 , 0.33333333, 0.25 ]) # may vary """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) if not np.iterable(k): @@ -841,7 +841,7 @@ def hermeval(x, c, tensor=True): [31., 54.]]) """ - c = np.array(c, ndmin=1, copy=0) + c = np.array(c, ndmin=1, copy=False) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) if isinstance(x, (tuple, list)): @@ -1124,7 +1124,7 @@ def hermevander(x, deg): if ideg < 0: raise ValueError("deg must be non-negative") - x = np.array(x, copy=0, ndmin=1) + 0.0 + x = np.array(x, copy=False, ndmin=1) + 0.0 dims = (ideg + 1,) + x.shape dtyp = x.dtype v = np.empty(dims, dtype=dtyp) @@ -1186,7 +1186,7 @@ def hermevander2d(x, y, deg): .. versionadded:: 1.7.0 """ - return pu._vander2d(hermevander, x, y, deg) + return pu._vander_nd_flat((hermevander, hermevander), (x, y), deg) def hermevander3d(x, y, z, deg): @@ -1240,7 +1240,7 @@ def hermevander3d(x, y, z, deg): .. versionadded:: 1.7.0 """ - return pu._vander3d(hermevander, x, y, z, deg) + return pu._vander_nd_flat((hermevander, hermevander, hermevander), (x, y, z), deg) def hermefit(x, y, deg, rcond=None, full=False, w=None): diff --git a/numpy/polynomial/laguerre.py b/numpy/polynomial/laguerre.py index 025a09929..538a1d449 100644 --- a/numpy/polynomial/laguerre.py +++ b/numpy/polynomial/laguerre.py @@ -624,7 +624,7 @@ def lagder(c, m=1, scl=1, axis=0): array([1., 2., 3.]) """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) @@ -739,7 +739,7 @@ def lagint(c, m=1, k=[], lbnd=0, scl=1, axis=0): array([ 11.16666667, -5. , -3. , 2. ]) # may vary """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) if not np.iterable(k): @@ -848,7 +848,7 @@ def lagval(x, c, tensor=True): [-4.5, -2. ]]) """ - c = np.array(c, ndmin=1, copy=0) + c = np.array(c, ndmin=1, copy=False) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) if isinstance(x, (tuple, list)): @@ -1131,7 +1131,7 @@ def lagvander(x, deg): if ideg < 0: raise ValueError("deg must be non-negative") - x = np.array(x, copy=0, ndmin=1) + 0.0 + x = np.array(x, copy=False, ndmin=1) + 0.0 dims = (ideg + 1,) + x.shape dtyp = x.dtype v = np.empty(dims, dtype=dtyp) @@ -1193,7 +1193,7 @@ def lagvander2d(x, y, deg): .. versionadded:: 1.7.0 """ - return pu._vander2d(lagvander, x, y, deg) + return pu._vander_nd_flat((lagvander, lagvander), (x, y), deg) def lagvander3d(x, y, z, deg): @@ -1247,7 +1247,7 @@ def lagvander3d(x, y, z, deg): .. versionadded:: 1.7.0 """ - return pu._vander3d(lagvander, x, y, z, deg) + return pu._vander_nd_flat((lagvander, lagvander, lagvander), (x, y, z), deg) def lagfit(x, y, deg, rcond=None, full=False, w=None): diff --git a/numpy/polynomial/legendre.py b/numpy/polynomial/legendre.py index f06b294cf..c11824761 100644 --- a/numpy/polynomial/legendre.py +++ b/numpy/polynomial/legendre.py @@ -669,7 +669,7 @@ def legder(c, m=1, scl=1, axis=0): array([ 9., 60.]) """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) cnt = pu._deprecate_as_int(m, "the order of derivation") @@ -786,7 +786,7 @@ def legint(c, m=1, k=[], lbnd=0, scl=1, axis=0): array([ 0.66666667, 0.8 , 1.33333333, 1.2 ]) # may vary """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) if not np.iterable(k): @@ -891,7 +891,7 @@ def legval(x, c, tensor=True): -------- """ - c = np.array(c, ndmin=1, copy=0) + c = np.array(c, ndmin=1, copy=False) if c.dtype.char in '?bBhHiIlLqQpP': c = c.astype(np.double) if isinstance(x, (tuple, list)): @@ -1165,7 +1165,7 @@ def legvander(x, deg): if ideg < 0: raise ValueError("deg must be non-negative") - x = np.array(x, copy=0, ndmin=1) + 0.0 + x = np.array(x, copy=False, ndmin=1) + 0.0 dims = (ideg + 1,) + x.shape dtyp = x.dtype v = np.empty(dims, dtype=dtyp) @@ -1229,7 +1229,7 @@ def legvander2d(x, y, deg): .. versionadded:: 1.7.0 """ - return pu._vander2d(legvander, x, y, deg) + return pu._vander_nd_flat((legvander, legvander), (x, y), deg) def legvander3d(x, y, z, deg): @@ -1283,7 +1283,7 @@ def legvander3d(x, y, z, deg): .. versionadded:: 1.7.0 """ - return pu._vander3d(legvander, x, y, z, deg) + return pu._vander_nd_flat((legvander, legvander, legvander), (x, y, z), deg) def legfit(x, y, deg, rcond=None, full=False, w=None): diff --git a/numpy/polynomial/polynomial.py b/numpy/polynomial/polynomial.py index 99f9fc2fd..315ea1495 100644 --- a/numpy/polynomial/polynomial.py +++ b/numpy/polynomial/polynomial.py @@ -491,7 +491,7 @@ def polyder(c, m=1, scl=1, axis=0): array([ 6., 24.]) """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': # astype fails with NA c = c + 0.0 @@ -599,7 +599,7 @@ def polyint(c, m=1, k=[], lbnd=0, scl=1, axis=0): array([ 0., -2., -2., -2.]) """ - c = np.array(c, ndmin=1, copy=1) + c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': # astype doesn't preserve mask attribute. c = c + 0.0 @@ -721,7 +721,7 @@ def polyval(x, c, tensor=True): array([2., 7.]) """ - c = np.array(c, ndmin=1, copy=0) + c = np.array(c, ndmin=1, copy=False) if c.dtype.char in '?bBhHiIlLqQpP': # astype fails with NA c = c + 0.0 @@ -811,7 +811,7 @@ def polyvalfromroots(x, r, tensor=True): >>> polyvalfromroots(b, r, tensor=False) array([-0., 0.]) """ - r = np.array(r, ndmin=1, copy=0) + r = np.array(r, ndmin=1, copy=False) if r.dtype.char in '?bBhHiIlLqQpP': r = r.astype(np.double) if isinstance(x, (tuple, list)): @@ -1076,7 +1076,7 @@ def polyvander(x, deg): if ideg < 0: raise ValueError("deg must be non-negative") - x = np.array(x, copy=0, ndmin=1) + 0.0 + x = np.array(x, copy=False, ndmin=1) + 0.0 dims = (ideg + 1,) + x.shape dtyp = x.dtype v = np.empty(dims, dtype=dtyp) @@ -1133,7 +1133,7 @@ def polyvander2d(x, y, deg): polyvander, polyvander3d, polyval2d, polyval3d """ - return pu._vander2d(polyvander, x, y, deg) + return pu._vander_nd_flat((polyvander, polyvander), (x, y), deg) def polyvander3d(x, y, z, deg): @@ -1187,7 +1187,7 @@ def polyvander3d(x, y, z, deg): .. versionadded:: 1.7.0 """ - return pu._vander3d(polyvander, x, y, z, deg) + return pu._vander_nd_flat((polyvander, polyvander, polyvander), (x, y, z), deg) def polyfit(x, y, deg, rcond=None, full=False, w=None): diff --git a/numpy/polynomial/polyutils.py b/numpy/polynomial/polyutils.py index 841421d0e..5dcfa7a7a 100644 --- a/numpy/polynomial/polyutils.py +++ b/numpy/polynomial/polyutils.py @@ -46,6 +46,7 @@ Functions from __future__ import division, absolute_import, print_function import operator +import functools import warnings import numpy as np @@ -174,7 +175,7 @@ def as_series(alist, trim=True): [array([2.]), array([1.1, 0. ])] """ - arrays = [np.array(a, ndmin=1, copy=0) for a in alist] + arrays = [np.array(a, ndmin=1, copy=False) for a in alist] if min([a.size for a in arrays]) == 0: raise ValueError("Coefficient array is empty") if any([a.ndim != 1 for a in arrays]): @@ -196,7 +197,7 @@ def as_series(alist, trim=True): dtype = np.common_type(*arrays) except Exception: raise ValueError("Coefficient arrays have no common type") - ret = [np.array(a, copy=1, dtype=dtype) for a in arrays] + ret = [np.array(a, copy=True, dtype=dtype) for a in arrays] return ret @@ -415,51 +416,89 @@ def mapdomain(x, old, new): return off + scl*x -def _vander2d(vander_f, x, y, deg): - """ - Helper function used to implement the ``<type>vander2d`` functions. +def _nth_slice(i, ndim): + sl = [np.newaxis] * ndim + sl[i] = slice(None) + return tuple(sl) + + +def _vander_nd(vander_fs, points, degrees): + r""" + A generalization of the Vandermonde matrix for N dimensions + + The result is built by combining the results of 1d Vandermonde matrices, + + .. math:: + W[i_0, \ldots, i_M, j_0, \ldots, j_N] = \prod_{k=0}^N{V_k(x_k)[i_0, \ldots, i_M, j_k]} + + where + + .. math:: + N &= \texttt{len(points)} = \texttt{len(degrees)} = \texttt{len(vander\_fs)} \\ + M &= \texttt{points[k].ndim} \\ + V_k &= \texttt{vander\_fs[k]} \\ + x_k &= \texttt{points[k]} \\ + 0 \le j_k &\le \texttt{degrees[k]} + + Expanding the one-dimensional :math:`V_k` functions gives: + + .. math:: + W[i_0, \ldots, i_M, j_0, \ldots, j_N] = \prod_{k=0}^N{B_{k, j_k}(x_k[i_0, \ldots, i_M])} + + where :math:`B_{k,m}` is the m'th basis of the polynomial construction used along + dimension :math:`k`. For a regular polynomial, :math:`B_{k, m}(x) = P_m(x) = x^m`. Parameters ---------- - vander_f : function(array_like, int) -> ndarray - The 1d vander function, such as ``polyvander`` - x, y, deg : - See the ``<type>vander2d`` functions for more detail + vander_fs : Sequence[function(array_like, int) -> ndarray] + The 1d vander function to use for each axis, such as ``polyvander`` + points : Sequence[array_like] + Arrays of point coordinates, all of the same shape. The dtypes + will be converted to either float64 or complex128 depending on + whether any of the elements are complex. Scalars are converted to + 1-D arrays. + This must be the same length as `vander_fs`. + degrees : Sequence[int] + The maximum degree (inclusive) to use for each axis. + This must be the same length as `vander_fs`. + + Returns + ------- + vander_nd : ndarray + An array of shape ``points[0].shape + tuple(d + 1 for d in degrees)``. """ - degx, degy = [ - _deprecate_as_int(d, "degrees") - for d in deg - ] - x, y = np.array((x, y), copy=0) + 0.0 + n_dims = len(vander_fs) + if n_dims != len(points): + raise ValueError( + "Expected {} dimensions of sample points, got {}".format(n_dims, len(points))) + if n_dims != len(degrees): + raise ValueError( + "Expected {} dimensions of degrees, got {}".format(n_dims, len(degrees))) + if n_dims == 0: + raise ValueError("Unable to guess a dtype or shape when no points are given") + + # convert to the same shape and type + points = tuple(np.array(tuple(points), copy=False) + 0.0) - vx = vander_f(x, degx) - vy = vander_f(y, degy) - v = vx[..., None]*vy[..., None,:] - return v.reshape(v.shape[:-2] + (-1,)) + # produce the vandermonde matrix for each dimension, placing the last + # axis of each in an independent trailing axis of the output + vander_arrays = ( + vander_fs[i](points[i], degrees[i])[(...,) + _nth_slice(i, n_dims)] + for i in range(n_dims) + ) + # we checked this wasn't empty already, so no `initial` needed + return functools.reduce(operator.mul, vander_arrays) -def _vander3d(vander_f, x, y, z, deg): + +def _vander_nd_flat(vander_fs, points, degrees): """ - Helper function used to implement the ``<type>vander3d`` functions. + Like `_vander_nd`, but flattens the last ``len(degrees)`` axes into a single axis - Parameters - ---------- - vander_f : function(array_like, int) -> ndarray - The 1d vander function, such as ``polyvander`` - x, y, z, deg : - See the ``<type>vander3d`` functions for more detail + Used to implement the public ``<type>vander<n>d`` functions. """ - degx, degy, degz = [ - _deprecate_as_int(d, "degrees") - for d in deg - ] - x, y, z = np.array((x, y, z), copy=0) + 0.0 - - vx = vander_f(x, degx) - vy = vander_f(y, degy) - vz = vander_f(z, degz) - v = vx[..., None, None]*vy[..., None,:, None]*vz[..., None, None,:] - return v.reshape(v.shape[:-3] + (-1,)) + v = _vander_nd(vander_fs, points, degrees) + return v.reshape(v.shape[:-len(degrees)] + (-1,)) def _fromroots(line_f, mul_f, roots): diff --git a/numpy/random/.gitignore b/numpy/random/.gitignore new file mode 100644 index 000000000..fea3f955a --- /dev/null +++ b/numpy/random/.gitignore @@ -0,0 +1,3 @@ +# generated files +_bounded_integers.pyx +_bounded_integers.pxd diff --git a/numpy/random/__init__.pxd b/numpy/random/__init__.pxd new file mode 100644 index 000000000..05e073876 --- /dev/null +++ b/numpy/random/__init__.pxd @@ -0,0 +1,14 @@ +cimport numpy as np +from libc.stdint cimport uint32_t, uint64_t + +cdef extern from "numpy/random/bitgen.h": + struct bitgen: + void *state + uint64_t (*next_uint64)(void *st) nogil + uint32_t (*next_uint32)(void *st) nogil + double (*next_double)(void *st) nogil + uint64_t (*next_raw)(void *st) nogil + + ctypedef bitgen bitgen_t + +from numpy.random._bit_generator cimport BitGenerator, SeedSequence diff --git a/numpy/random/__init__.py b/numpy/random/__init__.py index f7c248451..1ceb5c4dd 100644 --- a/numpy/random/__init__.py +++ b/numpy/random/__init__.py @@ -179,20 +179,19 @@ __all__ = [ # add these for module-freeze analysis (like PyInstaller) from . import _pickle -from . import common -from . import bounded_integers - +from . import _common +from . import _bounded_integers + +from ._generator import Generator, default_rng +from ._bit_generator import SeedSequence, BitGenerator +from ._mt19937 import MT19937 +from ._pcg64 import PCG64 +from ._philox import Philox +from ._sfc64 import SFC64 from .mtrand import * -from .generator import Generator, default_rng -from .bit_generator import SeedSequence -from .mt19937 import MT19937 -from .pcg64 import PCG64 -from .philox import Philox -from .sfc64 import SFC64 -from .mtrand import RandomState __all__ += ['Generator', 'RandomState', 'SeedSequence', 'MT19937', - 'Philox', 'PCG64', 'SFC64', 'default_rng'] + 'Philox', 'PCG64', 'SFC64', 'default_rng', 'BitGenerator'] def __RandomState_ctor(): diff --git a/numpy/random/bit_generator.pxd b/numpy/random/_bit_generator.pxd index 984033f17..bd5e47a20 100644 --- a/numpy/random/bit_generator.pxd +++ b/numpy/random/_bit_generator.pxd @@ -1,6 +1,15 @@ - -from .common cimport bitgen_t, uint32_t cimport numpy as np +from libc.stdint cimport uint32_t, uint64_t + +cdef extern from "numpy/random/bitgen.h": + struct bitgen: + void *state + uint64_t (*next_uint64)(void *st) nogil + uint32_t (*next_uint32)(void *st) nogil + double (*next_double)(void *st) nogil + uint64_t (*next_raw)(void *st) nogil + + ctypedef bitgen bitgen_t cdef class BitGenerator(): cdef readonly object _seed_seq diff --git a/numpy/random/bit_generator.pyx b/numpy/random/_bit_generator.pyx index eb608af6c..21d21e6bb 100644 --- a/numpy/random/bit_generator.pyx +++ b/numpy/random/_bit_generator.pyx @@ -53,9 +53,7 @@ from cpython.pycapsule cimport PyCapsule_New import numpy as np cimport numpy as np -from libc.stdint cimport uint32_t -from .common cimport (random_raw, benchmark, prepare_ctypes, prepare_cffi) -from .distributions cimport bitgen_t +from ._common cimport (random_raw, benchmark, prepare_ctypes, prepare_cffi) __all__ = ['SeedSequence', 'BitGenerator'] @@ -116,7 +114,7 @@ def _coerce_to_uint32_array(x): Examples -------- >>> import numpy as np - >>> from numpy.random.bit_generator import _coerce_to_uint32_array + >>> from numpy.random._bit_generator import _coerce_to_uint32_array >>> _coerce_to_uint32_array(12345) array([12345], dtype=uint32) >>> _coerce_to_uint32_array('12345') @@ -484,13 +482,12 @@ cdef class BitGenerator(): Parameters ---------- - seed : {None, int, array_like[ints], ISeedSequence}, optional + seed : {None, int, array_like[ints], SeedSequence}, optional A seed to initialize the `BitGenerator`. If None, then fresh, unpredictable entropy will be pulled from the OS. If an ``int`` or ``array_like[ints]`` is passed, then it will be passed to - `SeedSequence` to derive the initial `BitGenerator` state. One may also - pass in an implementor of the `ISeedSequence` interface like - `SeedSequence`. + ~`numpy.random.SeedSequence` to derive the initial `BitGenerator` state. + One may also pass in a `SeedSequence` instance. Attributes ---------- diff --git a/numpy/random/bounded_integers.pxd.in b/numpy/random/_bounded_integers.pxd.in index 7a3f224dc..5ae5a8067 100644 --- a/numpy/random/bounded_integers.pxd.in +++ b/numpy/random/_bounded_integers.pxd.in @@ -4,7 +4,7 @@ import numpy as np cimport numpy as np ctypedef np.npy_bool bool_t -from .common cimport bitgen_t +from numpy.random cimport bitgen_t cdef inline uint64_t _gen_mask(uint64_t max_val) nogil: """Mask generator for use in bounded random numbers""" diff --git a/numpy/random/bounded_integers.pyx.in b/numpy/random/_bounded_integers.pyx.in index 411b65a37..7e19471e4 100644 --- a/numpy/random/bounded_integers.pyx.in +++ b/numpy/random/_bounded_integers.pyx.in @@ -4,12 +4,54 @@ import numpy as np cimport numpy as np -from .distributions cimport * - __all__ = [] np.import_array() +cdef extern from "numpy/random/distributions.h": + # Generate random numbers in closed interval [off, off + rng]. + uint64_t random_bounded_uint64(bitgen_t *bitgen_state, + uint64_t off, uint64_t rng, + uint64_t mask, bint use_masked) nogil + uint32_t random_buffered_bounded_uint32(bitgen_t *bitgen_state, + uint32_t off, uint32_t rng, + uint32_t mask, bint use_masked, + int *bcnt, uint32_t *buf) nogil + uint16_t random_buffered_bounded_uint16(bitgen_t *bitgen_state, + uint16_t off, uint16_t rng, + uint16_t mask, bint use_masked, + int *bcnt, uint32_t *buf) nogil + uint8_t random_buffered_bounded_uint8(bitgen_t *bitgen_state, + uint8_t off, uint8_t rng, + uint8_t mask, bint use_masked, + int *bcnt, uint32_t *buf) nogil + np.npy_bool random_buffered_bounded_bool(bitgen_t *bitgen_state, + np.npy_bool off, np.npy_bool rng, + np.npy_bool mask, bint use_masked, + int *bcnt, uint32_t *buf) nogil + void random_bounded_uint64_fill(bitgen_t *bitgen_state, + uint64_t off, uint64_t rng, np.npy_intp cnt, + bint use_masked, + uint64_t *out) nogil + void random_bounded_uint32_fill(bitgen_t *bitgen_state, + uint32_t off, uint32_t rng, np.npy_intp cnt, + bint use_masked, + uint32_t *out) nogil + void random_bounded_uint16_fill(bitgen_t *bitgen_state, + uint16_t off, uint16_t rng, np.npy_intp cnt, + bint use_masked, + uint16_t *out) nogil + void random_bounded_uint8_fill(bitgen_t *bitgen_state, + uint8_t off, uint8_t rng, np.npy_intp cnt, + bint use_masked, + uint8_t *out) nogil + void random_bounded_bool_fill(bitgen_t *bitgen_state, + np.npy_bool off, np.npy_bool rng, np.npy_intp cnt, + bint use_masked, + np.npy_bool *out) nogil + + + _integers_types = {'bool': (0, 2), 'int8': (-2**7, 2**7), 'int16': (-2**15, 2**15), @@ -149,7 +191,7 @@ cdef object _rand_{{nptype}}_broadcast(object low, object high, object size, highm1_arr = <np.ndarray>np.PyArray_FROM_OTF(high_m1, np.{{npctype}}, np.NPY_ALIGNED | np.NPY_FORCECAST) else: # If input is object or a floating type - highm1_arr = <np.ndarray>np.empty_like(high_arr, dtype=np.{{nptype}}) + highm1_arr = <np.ndarray>np.empty_like(high_arr, dtype=np.{{otype}}) highm1_data = <{{nptype}}_t *>np.PyArray_DATA(highm1_arr) cnt = np.PyArray_SIZE(high_arr) flat = high_arr.flat @@ -171,10 +213,10 @@ cdef object _rand_{{nptype}}_broadcast(object low, object high, object size, low_arr = <np.ndarray>np.PyArray_FROM_OTF(low, np.{{npctype}}, np.NPY_ALIGNED | np.NPY_FORCECAST) if size is not None: - out_arr = <np.ndarray>np.empty(size, np.{{nptype}}) + out_arr = <np.ndarray>np.empty(size, np.{{otype}}) else: it = np.PyArray_MultiIterNew2(low_arr, high_arr) - out_arr = <np.ndarray>np.empty(it.shape, np.{{nptype}}) + out_arr = <np.ndarray>np.empty(it.shape, np.{{otype}}) it = np.PyArray_MultiIterNew3(low_arr, high_arr, out_arr) out_data = <uint64_t *>np.PyArray_DATA(out_arr) @@ -216,12 +258,12 @@ cdef object _rand_{{nptype}}(object low, object high, object size, """ _rand_{{nptype}}(low, high, size, use_masked, *state, lock) - Return random np.{{nptype}} integers from `low` (inclusive) to `high` (exclusive). + Return random `np.{{otype}}` integers from `low` (inclusive) to `high` (exclusive). Return random integers from the "discrete uniform" distribution in the interval [`low`, `high`). If `high` is None (the default), then results are from [0, `low`). On entry the arguments are presumed - to have been validated for size and order for the np.{{nptype}} type. + to have been validated for size and order for the `np.{{otype}}` type. Parameters ---------- @@ -247,7 +289,7 @@ cdef object _rand_{{nptype}}(object low, object high, object size, Returns ------- - out : python scalar or ndarray of np.{{nptype}} + out : python scalar or ndarray of np.{{otype}} `size`-shaped array of random integers from the appropriate distribution, or a single such random int if `size` not provided. @@ -266,7 +308,7 @@ cdef object _rand_{{nptype}}(object low, object high, object size, if size is not None: if (np.prod(size) == 0): - return np.empty(size, dtype=np.{{nptype}}) + return np.empty(size, dtype=np.{{otype}}) low_arr = <np.ndarray>np.array(low, copy=False) high_arr = <np.ndarray>np.array(high, copy=False) @@ -295,7 +337,7 @@ cdef object _rand_{{nptype}}(object low, object high, object size, random_bounded_{{utype}}_fill(state, off, rng, 1, use_masked, &out_val) return np.{{otype}}(<{{nptype}}_t>out_val) else: - out_arr = <np.ndarray>np.empty(size, np.{{nptype}}) + out_arr = <np.ndarray>np.empty(size, np.{{otype}}) cnt = np.PyArray_SIZE(out_arr) out_data = <{{utype}}_t *>np.PyArray_DATA(out_arr) with lock, nogil: diff --git a/numpy/random/common.pxd b/numpy/random/_common.pxd index ac0a94bb0..588f613ae 100644 --- a/numpy/random/common.pxd +++ b/numpy/random/_common.pxd @@ -1,23 +1,12 @@ #cython: language_level=3 -from libc.stdint cimport (uint8_t, uint16_t, uint32_t, uint64_t, - int8_t, int16_t, int32_t, int64_t, intptr_t, - uintptr_t) -from libc.math cimport sqrt - -cdef extern from "src/bitgen.h": - struct bitgen: - void *state - uint64_t (*next_uint64)(void *st) nogil - uint32_t (*next_uint32)(void *st) nogil - double (*next_double)(void *st) nogil - uint64_t (*next_raw)(void *st) nogil - - ctypedef bitgen bitgen_t +from libc.stdint cimport uint32_t, uint64_t, int32_t, int64_t import numpy as np cimport numpy as np +from numpy.random cimport bitgen_t + cdef double POISSON_LAM_MAX cdef double LEGACY_POISSON_LAM_MAX cdef uint64_t MAXSIZE @@ -44,7 +33,7 @@ cdef object prepare_ctypes(bitgen_t *bitgen) cdef int check_constraint(double val, object name, constraint_type cons) except -1 cdef int check_array_constraint(np.ndarray val, object name, constraint_type cons) except -1 -cdef extern from "src/aligned_malloc/aligned_malloc.h": +cdef extern from "include/aligned_malloc.h": cdef void *PyArray_realloc_aligned(void *p, size_t n) cdef void *PyArray_malloc_aligned(size_t n) cdef void *PyArray_calloc_aligned(size_t n, size_t s) @@ -56,6 +45,7 @@ ctypedef double (*random_double_1)(void *state, double a) nogil ctypedef double (*random_double_2)(void *state, double a, double b) nogil ctypedef double (*random_double_3)(void *state, double a, double b, double c) nogil +ctypedef double (*random_float_fill)(bitgen_t *state, np.npy_intp count, float* out) nogil ctypedef float (*random_float_0)(bitgen_t *state) nogil ctypedef float (*random_float_1)(bitgen_t *state, float a) nogil diff --git a/numpy/random/common.pyx b/numpy/random/_common.pyx index 74cd5f033..ef1afac7c 100644 --- a/numpy/random/common.pyx +++ b/numpy/random/_common.pyx @@ -6,7 +6,7 @@ import sys import numpy as np cimport numpy as np -from .common cimport * +from libc.stdint cimport uintptr_t __all__ = ['interface'] @@ -262,14 +262,16 @@ cdef object double_fill(void *func, bitgen_t *state, object size, object lock, o return out_array cdef object float_fill(void *func, bitgen_t *state, object size, object lock, object out): - cdef random_float_0 random_func = (<random_float_0>func) + cdef random_float_fill random_func = (<random_float_fill>func) + cdef float out_val cdef float *out_array_data cdef np.ndarray out_array cdef np.npy_intp i, n if size is None and out is None: with lock: - return random_func(state) + random_func(state, 1, &out_val) + return out_val if out is not None: check_output(out, np.float32, size) @@ -280,8 +282,7 @@ cdef object float_fill(void *func, bitgen_t *state, object size, object lock, ob n = np.PyArray_SIZE(out_array) out_array_data = <float *>np.PyArray_DATA(out_array) with lock, nogil: - for i in range(n): - out_array_data[i] = random_func(state) + random_func(state, n, out_array_data) return out_array cdef object float_fill_from_double(void *func, bitgen_t *state, object size, object lock, object out): diff --git a/numpy/random/_examples/cffi/extending.py b/numpy/random/_examples/cffi/extending.py new file mode 100644 index 000000000..8440d400e --- /dev/null +++ b/numpy/random/_examples/cffi/extending.py @@ -0,0 +1,40 @@ +""" +Use cffi to access any of the underlying C functions from distributions.h +""" +import os +import numpy as np +import cffi +from .parse import parse_distributions_h +ffi = cffi.FFI() + +inc_dir = os.path.join(np.get_include(), 'numpy') + +# Basic numpy types +ffi.cdef(''' + typedef intptr_t npy_intp; + typedef unsigned char npy_bool; + +''') + +parse_distributions_h(ffi, inc_dir) + +lib = ffi.dlopen(np.random._generator.__file__) + +# Compare the distributions.h random_standard_normal_fill to +# Generator.standard_random +bit_gen = np.random.PCG64() +rng = np.random.Generator(bit_gen) +state = bit_gen.state + +interface = rng.bit_generator.cffi +n = 100 +vals_cffi = ffi.new('double[%d]' % n) +lib.random_standard_normal_fill(interface.bit_generator, n, vals_cffi) + +# reset the state +bit_gen.state = state + +vals = rng.standard_normal(n) + +for i in range(n): + assert vals[i] == vals_cffi[i] diff --git a/numpy/random/_examples/cffi/parse.py b/numpy/random/_examples/cffi/parse.py new file mode 100644 index 000000000..73d8646c7 --- /dev/null +++ b/numpy/random/_examples/cffi/parse.py @@ -0,0 +1,46 @@ +import os + + +def parse_distributions_h(ffi, inc_dir): + """ + Parse distributions.h located in inc_dir for CFFI, filling in the ffi.cdef + + Read the function declarations without the "#define ..." macros that will + be filled in when loading the library. + """ + + with open(os.path.join(inc_dir, 'random', 'bitgen.h')) as fid: + s = [] + for line in fid: + # massage the include file + if line.strip().startswith('#'): + continue + s.append(line) + ffi.cdef('\n'.join(s)) + + with open(os.path.join(inc_dir, 'random', 'distributions.h')) as fid: + s = [] + in_skip = 0 + for line in fid: + # massage the include file + if line.strip().startswith('#'): + continue + + # skip any inlined function definition + # which starts with 'static NPY_INLINE xxx(...) {' + # and ends with a closing '}' + if line.strip().startswith('static NPY_INLINE'): + in_skip += line.count('{') + continue + elif in_skip > 0: + in_skip += line.count('{') + in_skip -= line.count('}') + continue + + # replace defines with their value or remove them + line = line.replace('DECLDIR', '') + line = line.replace('NPY_INLINE', '') + line = line.replace('RAND_INT_TYPE', 'int64_t') + s.append(line) + ffi.cdef('\n'.join(s)) + diff --git a/numpy/random/examples/cython/extending.pyx b/numpy/random/_examples/cython/extending.pyx index a6a4ba4bf..7a0dfe078 100644 --- a/numpy/random/examples/cython/extending.pyx +++ b/numpy/random/_examples/cython/extending.pyx @@ -8,7 +8,7 @@ import numpy as np cimport numpy as np cimport cython -from numpy.random.common cimport bitgen_t +from numpy.random cimport bitgen_t from numpy.random import PCG64 np.import_array() @@ -39,7 +39,7 @@ def uniform_mean(Py_ssize_t n): return randoms.mean() -# This function is declated nogil so it can be used without the GIL below +# This function is declared nogil so it can be used without the GIL below cdef uint32_t bounded_uint(uint32_t lb, uint32_t ub, bitgen_t *rng) nogil: cdef uint32_t mask, delta, val mask = delta = ub - lb diff --git a/numpy/random/examples/cython/extending_distributions.pyx b/numpy/random/_examples/cython/extending_distributions.pyx index 3cefec97e..1bef506ef 100644 --- a/numpy/random/examples/cython/extending_distributions.pyx +++ b/numpy/random/_examples/cython/extending_distributions.pyx @@ -1,21 +1,25 @@ #!/usr/bin/env python #cython: language_level=3 """ -This file shows how the distributions that are accessed through -distributions.pxd can be used Cython code. +This file shows how the to use a BitGenerator to create a distribution. """ import numpy as np cimport numpy as np cimport cython from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer -from numpy.random.common cimport * -from numpy.random.distributions cimport random_gauss_zig +from libc.stdint cimport uint16_t, uint64_t +from numpy.random cimport bitgen_t from numpy.random import PCG64 @cython.boundscheck(False) @cython.wraparound(False) -def normals_zig(Py_ssize_t n): +def uniforms(Py_ssize_t n): + """ + Create an array of `n` uniformly distributed doubles. + A 'real' distribution would want to process the values into + some non-uniform distribution + """ cdef Py_ssize_t i cdef bitgen_t *rng cdef const char *capsule_name = "BitGenerator" @@ -23,37 +27,48 @@ def normals_zig(Py_ssize_t n): x = PCG64() capsule = x.capsule + # Optional check that the capsule if from a BitGenerator if not PyCapsule_IsValid(capsule, capsule_name): raise ValueError("Invalid pointer to anon_func_state") + # Cast the pointer rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name) - random_values = np.empty(n) - # Best practice is to release GIL and acquire the lock + random_values = np.empty(n, dtype='float64') with x.lock, nogil: for i in range(n): - random_values[i] = random_gauss_zig(rng) + # Call the function + random_values[i] = rng.next_double(rng.state) randoms = np.asarray(random_values) - return randoms - + return randoms + +# cython example 2 @cython.boundscheck(False) @cython.wraparound(False) -def uniforms(Py_ssize_t n): +def uint10_uniforms(Py_ssize_t n): + """Uniform 10 bit integers stored as 16-bit unsigned integers""" cdef Py_ssize_t i cdef bitgen_t *rng cdef const char *capsule_name = "BitGenerator" - cdef double[::1] random_values + cdef uint16_t[::1] random_values + cdef int bits_remaining + cdef int width = 10 + cdef uint64_t buff, mask = 0x3FF x = PCG64() capsule = x.capsule - # Optional check that the capsule if from a BitGenerator if not PyCapsule_IsValid(capsule, capsule_name): raise ValueError("Invalid pointer to anon_func_state") - # Cast the pointer rng = <bitgen_t *> PyCapsule_GetPointer(capsule, capsule_name) - random_values = np.empty(n) + random_values = np.empty(n, dtype='uint16') + # Best practice is to release GIL and acquire the lock + bits_remaining = 0 with x.lock, nogil: for i in range(n): - # Call the function - random_values[i] = rng.next_double(rng.state) + if bits_remaining < width: + buff = rng.next_uint64(rng.state) + random_values[i] = buff & mask + buff >>= width + randoms = np.asarray(random_values) return randoms + diff --git a/numpy/random/_examples/cython/setup.py b/numpy/random/_examples/cython/setup.py new file mode 100644 index 000000000..19f045fc0 --- /dev/null +++ b/numpy/random/_examples/cython/setup.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +""" +Build the Cython demonstrations of low-level access to NumPy random + +Usage: python setup.py build_ext -i +""" + +import numpy as np +from distutils.core import setup +from Cython.Build import cythonize +from setuptools.extension import Extension +from os.path import join, abspath, dirname + +path = abspath(dirname(__file__)) + +extending = Extension("extending", + sources=[join(path, 'extending.pyx')], + include_dirs=[ + np.get_include(), + join(path, '..', '..') + ], + ) +distributions = Extension("extending_distributions", + sources=[join(path, 'extending_distributions.pyx')], + include_dirs=[np.get_include()]) + +extensions = [extending, distributions] + +setup( + ext_modules=cythonize(extensions) +) diff --git a/numpy/random/_examples/numba/extending.py b/numpy/random/_examples/numba/extending.py new file mode 100644 index 000000000..0d240596b --- /dev/null +++ b/numpy/random/_examples/numba/extending.py @@ -0,0 +1,84 @@ +import numpy as np +import numba as nb + +from numpy.random import PCG64 +from timeit import timeit + +bit_gen = PCG64() +next_d = bit_gen.cffi.next_double +state_addr = bit_gen.cffi.state_address + +def normals(n, state): + out = np.empty(n) + for i in range((n + 1) // 2): + x1 = 2.0 * next_d(state) - 1.0 + x2 = 2.0 * next_d(state) - 1.0 + r2 = x1 * x1 + x2 * x2 + while r2 >= 1.0 or r2 == 0.0: + x1 = 2.0 * next_d(state) - 1.0 + x2 = 2.0 * next_d(state) - 1.0 + r2 = x1 * x1 + x2 * x2 + f = np.sqrt(-2.0 * np.log(r2) / r2) + out[2 * i] = f * x1 + if 2 * i + 1 < n: + out[2 * i + 1] = f * x2 + return out + +# Compile using Numba +normalsj = nb.jit(normals, nopython=True) +# Must use state address not state with numba +n = 10000 + +def numbacall(): + return normalsj(n, state_addr) + +rg = np.random.Generator(PCG64()) + +def numpycall(): + return rg.normal(size=n) + +# Check that the functions work +r1 = numbacall() +r2 = numpycall() +assert r1.shape == (n,) +assert r1.shape == r2.shape + +t1 = timeit(numbacall, number=1000) +print('{:.2f} secs for {} PCG64 (Numba/PCG64) gaussian randoms'.format(t1, n)) +t2 = timeit(numpycall, number=1000) +print('{:.2f} secs for {} PCG64 (NumPy/PCG64) gaussian randoms'.format(t2, n)) + +# example 2 + +next_u32 = bit_gen.ctypes.next_uint32 +ctypes_state = bit_gen.ctypes.state + +@nb.jit(nopython=True) +def bounded_uint(lb, ub, state): + mask = delta = ub - lb + mask |= mask >> 1 + mask |= mask >> 2 + mask |= mask >> 4 + mask |= mask >> 8 + mask |= mask >> 16 + + val = next_u32(state) & mask + while val > delta: + val = next_u32(state) & mask + + return lb + val + + +print(bounded_uint(323, 2394691, ctypes_state.value)) + + +@nb.jit(nopython=True) +def bounded_uints(lb, ub, n, state): + out = np.empty(n, dtype=np.uint32) + for i in range(n): + out[i] = bounded_uint(lb, ub, state) + + +bounded_uints(323, 2394691, 10000000, ctypes_state.value) + + diff --git a/numpy/random/examples/numba/extending_distributions.py b/numpy/random/_examples/numba/extending_distributions.py index 9233ccced..7cf8bf0b0 100644 --- a/numpy/random/examples/numba/extending_distributions.py +++ b/numpy/random/_examples/numba/extending_distributions.py @@ -1,22 +1,28 @@ r""" -On *nix, execute in randomgen/src/distributions +Building the required library in this example requires a source distribution +of NumPy or clone of the NumPy git repository since distributions.c is not +included in binary distributions. +On *nix, execute in numpy/random/src/distributions + +export ${PYTHON_VERSION}=3.8 # Python version export PYTHON_INCLUDE=#path to Python's include folder, usually \ ${PYTHON_HOME}/include/python${PYTHON_VERSION}m export NUMPY_INCLUDE=#path to numpy's include folder, usually \ ${PYTHON_HOME}/lib/python${PYTHON_VERSION}/site-packages/numpy/core/include gcc -shared -o libdistributions.so -fPIC distributions.c \ -I${NUMPY_INCLUDE} -I${PYTHON_INCLUDE} -mv libdistributions.so ../../examples/numba/ +mv libdistributions.so ../../_examples/numba/ On Windows -rem PYTHON_HOME is setup dependent, this is an example +rem PYTHON_HOME and PYTHON_VERSION are setup dependent, this is an example set PYTHON_HOME=c:\Anaconda +set PYTHON_VERSION=38 cl.exe /LD .\distributions.c -DDLL_EXPORT \ -I%PYTHON_HOME%\lib\site-packages\numpy\core\include \ - -I%PYTHON_HOME%\include %PYTHON_HOME%\libs\python36.lib -move distributions.dll ../../examples/numba/ + -I%PYTHON_HOME%\include %PYTHON_HOME%\libs\python%PYTHON_VERSION%.lib +move distributions.dll ../../_examples/numba/ """ import os @@ -35,19 +41,19 @@ else: raise RuntimeError('Required DLL/so file was not found.') ffi.cdef(""" -double random_gauss_zig(void *bitgen_state); +double random_standard_normal(void *bitgen_state); """) x = PCG64() xffi = x.cffi bit_generator = xffi.bit_generator -random_gauss_zig = lib.random_gauss_zig +random_standard_normal = lib.random_standard_normal def normals(n, bit_generator): out = np.empty(n) for i in range(n): - out[i] = random_gauss_zig(bit_generator) + out[i] = random_standard_normal(bit_generator) return out diff --git a/numpy/random/generator.pyx b/numpy/random/_generator.pyx index 26fd95129..d76cde44c 100644 --- a/numpy/random/generator.pyx +++ b/numpy/random/_generator.pyx @@ -3,35 +3,157 @@ import operator import warnings -import numpy as np - -from .bounded_integers import _integers_types -from .pcg64 import PCG64 - from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer from cpython cimport (Py_INCREF, PyFloat_AsDouble) -from libc cimport string cimport cython +import numpy as np cimport numpy as np +from numpy.core.multiarray import normalize_axis_index -from .bounded_integers cimport * -from .common cimport * -from .distributions cimport * +from libc cimport string +from libc.stdint cimport (uint8_t, uint16_t, uint32_t, uint64_t, + int32_t, int64_t, INT64_MAX, SIZE_MAX) +from ._bounded_integers cimport (_rand_bool, _rand_int32, _rand_int64, + _rand_int16, _rand_int8, _rand_uint64, _rand_uint32, _rand_uint16, + _rand_uint8, _gen_mask) +from ._bounded_integers import _integers_types +from ._pcg64 import PCG64 +from numpy.random cimport bitgen_t +from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE, + CONS_NON_NEGATIVE, CONS_BOUNDED_0_1, CONS_BOUNDED_GT_0_1, + CONS_GT_1, CONS_POSITIVE_NOT_NAN, CONS_POISSON, + double_fill, cont, kahan_sum, cont_broadcast_3, float_fill, cont_f, + check_array_constraint, check_constraint, disc, discrete_broadcast_iii, + ) + + +cdef extern from "numpy/random/distributions.h": + + struct s_binomial_t: + int has_binomial + double psave + int64_t nsave + double r + double q + double fm + int64_t m + double p1 + double xm + double xl + double xr + double c + double laml + double lamr + double p2 + double p3 + double p4 + + ctypedef s_binomial_t binomial_t + + double random_standard_uniform(bitgen_t *bitgen_state) nogil + void random_standard_uniform_fill(bitgen_t* bitgen_state, np.npy_intp cnt, double *out) nogil + double random_standard_exponential(bitgen_t *bitgen_state) nogil + double random_standard_exponential_f(bitgen_t *bitgen_state) nogil + void random_standard_exponential_fill(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil + void random_standard_exponential_fill_f(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil + void random_standard_exponential_inv_fill(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil + void random_standard_exponential_inv_fill_f(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil + double random_standard_normal(bitgen_t* bitgen_state) nogil + void random_standard_normal_fill(bitgen_t *bitgen_state, np.npy_intp count, double *out) nogil + void random_standard_normal_fill_f(bitgen_t *bitgen_state, np.npy_intp count, float *out) nogil + double random_standard_gamma(bitgen_t *bitgen_state, double shape) nogil + + float random_standard_uniform_f(bitgen_t *bitgen_state) nogil + void random_standard_uniform_fill_f(bitgen_t* bitgen_state, np.npy_intp cnt, float *out) nogil + float random_standard_normal_f(bitgen_t* bitgen_state) nogil + float random_standard_gamma_f(bitgen_t *bitgen_state, float shape) nogil + + int64_t random_positive_int64(bitgen_t *bitgen_state) nogil + int32_t random_positive_int32(bitgen_t *bitgen_state) nogil + int64_t random_positive_int(bitgen_t *bitgen_state) nogil + uint64_t random_uint(bitgen_t *bitgen_state) nogil + + double random_normal(bitgen_t *bitgen_state, double loc, double scale) nogil + + double random_gamma(bitgen_t *bitgen_state, double shape, double scale) nogil + float random_gamma_f(bitgen_t *bitgen_state, float shape, float scale) nogil + + double random_exponential(bitgen_t *bitgen_state, double scale) nogil + double random_uniform(bitgen_t *bitgen_state, double lower, double range) nogil + double random_beta(bitgen_t *bitgen_state, double a, double b) nogil + double random_chisquare(bitgen_t *bitgen_state, double df) nogil + double random_f(bitgen_t *bitgen_state, double dfnum, double dfden) nogil + double random_standard_cauchy(bitgen_t *bitgen_state) nogil + double random_pareto(bitgen_t *bitgen_state, double a) nogil + double random_weibull(bitgen_t *bitgen_state, double a) nogil + double random_power(bitgen_t *bitgen_state, double a) nogil + double random_laplace(bitgen_t *bitgen_state, double loc, double scale) nogil + double random_gumbel(bitgen_t *bitgen_state, double loc, double scale) nogil + double random_logistic(bitgen_t *bitgen_state, double loc, double scale) nogil + double random_lognormal(bitgen_t *bitgen_state, double mean, double sigma) nogil + double random_rayleigh(bitgen_t *bitgen_state, double mode) nogil + double random_standard_t(bitgen_t *bitgen_state, double df) nogil + double random_noncentral_chisquare(bitgen_t *bitgen_state, double df, + double nonc) nogil + double random_noncentral_f(bitgen_t *bitgen_state, double dfnum, + double dfden, double nonc) nogil + double random_wald(bitgen_t *bitgen_state, double mean, double scale) nogil + double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) nogil + double random_triangular(bitgen_t *bitgen_state, double left, double mode, + double right) nogil + + int64_t random_poisson(bitgen_t *bitgen_state, double lam) nogil + int64_t random_negative_binomial(bitgen_t *bitgen_state, double n, double p) nogil + int64_t random_binomial(bitgen_t *bitgen_state, double p, int64_t n, binomial_t *binomial) nogil + int64_t random_logseries(bitgen_t *bitgen_state, double p) nogil + int64_t random_geometric_search(bitgen_t *bitgen_state, double p) nogil + int64_t random_geometric_inversion(bitgen_t *bitgen_state, double p) nogil + int64_t random_geometric(bitgen_t *bitgen_state, double p) nogil + int64_t random_zipf(bitgen_t *bitgen_state, double a) nogil + int64_t random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad, + int64_t sample) nogil + + uint64_t random_interval(bitgen_t *bitgen_state, uint64_t max) nogil + + # Generate random uint64 numbers in closed interval [off, off + rng]. + uint64_t random_bounded_uint64(bitgen_t *bitgen_state, + uint64_t off, uint64_t rng, + uint64_t mask, bint use_masked) nogil + + void random_multinomial(bitgen_t *bitgen_state, int64_t n, int64_t *mnix, + double *pix, np.npy_intp d, binomial_t *binomial) nogil + + int random_multivariate_hypergeometric_count(bitgen_t *bitgen_state, + int64_t total, + size_t num_colors, int64_t *colors, + int64_t nsample, + size_t num_variates, int64_t *variates) nogil + void random_multivariate_hypergeometric_marginals(bitgen_t *bitgen_state, + int64_t total, + size_t num_colors, int64_t *colors, + int64_t nsample, + size_t num_variates, int64_t *variates) nogil +np.import_array() -__all__ = ['Generator', 'beta', 'binomial', 'bytes', 'chisquare', 'choice', - 'dirichlet', 'exponential', 'f', 'gamma', - 'geometric', 'gumbel', 'hypergeometric', 'integers', 'laplace', - 'logistic', 'lognormal', 'logseries', 'multinomial', - 'multivariate_normal', 'negative_binomial', 'noncentral_chisquare', - 'noncentral_f', 'normal', 'pareto', 'permutation', - 'poisson', 'power', 'random', 'rayleigh', 'shuffle', - 'standard_cauchy', 'standard_exponential', 'standard_gamma', - 'standard_normal', 'standard_t', 'triangular', - 'uniform', 'vonmises', 'wald', 'weibull', 'zipf'] -np.import_array() +cdef int64_t _safe_sum_nonneg_int64(size_t num_colors, int64_t *colors): + """ + Sum the values in the array `colors`. + + Return -1 if an overflow occurs. + The values in *colors are assumed to be nonnegative. + """ + cdef size_t i + cdef int64_t sum + + sum = 0 + for i in range(num_colors): + if colors[i] > INT64_MAX - sum: + return -1 + sum += colors[i] + return sum cdef bint _check_bit_generator(object bitgen): @@ -192,9 +314,9 @@ cdef class Generator: cdef double temp key = np.dtype(dtype).name if key == 'float64': - return double_fill(&random_double_fill, &self._bitgen, size, self.lock, out) + return double_fill(&random_standard_uniform_fill, &self._bitgen, size, self.lock, out) elif key == 'float32': - return float_fill(&random_float, &self._bitgen, size, self.lock, out) + return float_fill(&random_standard_uniform_fill_f, &self._bitgen, size, self.lock, out) else: raise TypeError('Unsupported dtype "%s" for random' % key) @@ -335,14 +457,14 @@ cdef class Generator: key = np.dtype(dtype).name if key == 'float64': if method == u'zig': - return double_fill(&random_standard_exponential_zig_fill, &self._bitgen, size, self.lock, out) - else: return double_fill(&random_standard_exponential_fill, &self._bitgen, size, self.lock, out) + else: + return double_fill(&random_standard_exponential_inv_fill, &self._bitgen, size, self.lock, out) elif key == 'float32': if method == u'zig': - return float_fill(&random_standard_exponential_zig_f, &self._bitgen, size, self.lock, out) + return float_fill(&random_standard_exponential_fill_f, &self._bitgen, size, self.lock, out) else: - return float_fill(&random_standard_exponential_f, &self._bitgen, size, self.lock, out) + return float_fill(&random_standard_exponential_inv_fill_f, &self._bitgen, size, self.lock, out) else: raise TypeError('Unsupported dtype "%s" for standard_exponential' % key) @@ -378,7 +500,7 @@ cdef class Generator: Desired dtype of the result. All dtypes are determined by their name, i.e., 'int64', 'int', etc, so byteorder is not available and a specific precision may have different C types depending - on the platform. The default value is 'np.int'. + on the platform. The default value is `np.int_`. endpoint : bool, optional If true, sample from the interval [low, high] instead of the default [low, high) @@ -471,7 +593,7 @@ cdef class Generator: elif key == 'bool': ret = _rand_bool(low, high, size, _masked, endpoint, &self._bitgen, self.lock) - if size is None and dtype in (np.bool, np.int, np.long): + if size is None and dtype in (bool, int, np.compat.long): if np.array(ret).shape == (): return dtype(ret) return ret @@ -780,7 +902,6 @@ cdef class Generator: -------- integers : Discrete uniform distribution, yielding integers. random : Floats uniformly distributed over ``[0, 1)``. - random : Alias for `random`. Notes ----- @@ -881,6 +1002,12 @@ cdef class Generator: A floating-point array of shape ``size`` of drawn samples, or a single sample if ``size`` was not specified. + See Also + -------- + normal : + Equivalent function with additional ``loc`` and ``scale`` arguments + for setting the mean and standard deviation. + Notes ----- For random samples from :math:`N(\\mu, \\sigma^2)`, use one of:: @@ -888,12 +1015,6 @@ cdef class Generator: mu + sigma * gen.standard_normal(size=...) gen.normal(mu, sigma, size=...) - See Also - -------- - normal : - Equivalent function with additional ``loc`` and ``scale`` arguments - for setting the mean and standard deviation. - Examples -------- >>> rng = np.random.default_rng() @@ -919,9 +1040,9 @@ cdef class Generator: """ key = np.dtype(dtype).name if key == 'float64': - return double_fill(&random_gauss_zig_fill, &self._bitgen, size, self.lock, out) + return double_fill(&random_standard_normal_fill, &self._bitgen, size, self.lock, out) elif key == 'float32': - return float_fill(&random_gauss_zig_f, &self._bitgen, size, self.lock, out) + return float_fill(&random_standard_normal_fill_f, &self._bitgen, size, self.lock, out) else: raise TypeError('Unsupported dtype "%s" for standard_normal' % key) @@ -1022,7 +1143,7 @@ cdef class Generator: [ 0.39924804, 4.68456316, 4.99394529, 4.84057254]]) # random """ - return cont(&random_normal_zig, &self._bitgen, size, self.lock, 2, + return cont(&random_normal, &self._bitgen, size, self.lock, 2, loc, '', CONS_NONE, scale, 'scale', CONS_NON_NEGATIVE, 0.0, '', CONS_NONE, @@ -1108,13 +1229,13 @@ cdef class Generator: cdef void *func key = np.dtype(dtype).name if key == 'float64': - return cont(&random_standard_gamma_zig, &self._bitgen, size, self.lock, 1, + return cont(&random_standard_gamma, &self._bitgen, size, self.lock, 1, shape, 'shape', CONS_NON_NEGATIVE, 0.0, '', CONS_NONE, 0.0, '', CONS_NONE, out) if key == 'float32': - return cont_f(&random_standard_gamma_zig_f, &self._bitgen, size, self.lock, + return cont_f(&random_standard_gamma_f, &self._bitgen, size, self.lock, shape, 'shape', CONS_NON_NEGATIVE, out) else: @@ -3146,6 +3267,8 @@ cdef class Generator: See Also -------- + multivariate_hypergeometric : Draw samples from the multivariate + hypergeometric distribution. scipy.stats.hypergeom : probability density function, distribution or cumulative density function, etc. @@ -3331,7 +3454,7 @@ cdef class Generator: # Multivariate distributions: def multivariate_normal(self, mean, cov, size=None, check_valid='warn', - tol=1e-8): + tol=1e-8, *, method='svd'): """ multivariate_normal(mean, cov, size=None, check_valid='warn', tol=1e-8) @@ -3361,6 +3484,15 @@ cdef class Generator: tol : float, optional Tolerance when checking the singular values in covariance matrix. cov is cast to double before the check. + method : { 'svd', 'eigh', 'cholesky'}, optional + The cov input is used to compute a factor matrix A such that + ``A @ A.T = cov``. This argument is used to select the method + used to compute the factor matrix A. The default method 'svd' is + the slowest, while 'cholesky' is the fastest but less robust than + the slowest method. The method `eigh` uses eigen decomposition to + compute A and is faster than svd but slower than cholesky. + + .. versionadded:: 1.18.0 Returns ------- @@ -3421,10 +3553,16 @@ cdef class Generator: -------- >>> mean = (1, 2) >>> cov = [[1, 0], [0, 1]] - >>> x = np.random.default_rng().multivariate_normal(mean, cov, (3, 3)) + >>> rng = np.random.default_rng() + >>> x = rng.multivariate_normal(mean, cov, (3, 3)) >>> x.shape (3, 3, 2) + We can use a different method other than the default to factorize cov: + >>> y = rng.multivariate_normal(mean, cov, (3, 3), method='cholesky') + >>> y.shape + (3, 3, 2) + The following is probably true, given that 0.6 is roughly twice the standard deviation: @@ -3432,7 +3570,9 @@ cdef class Generator: [True, True] # random """ - from numpy.dual import svd + if method not in {'eigh', 'svd', 'cholesky'}: + raise ValueError( + "method must be one of {'eigh', 'svd', 'cholesky'}") # Check preconditions on arguments mean = np.array(mean) @@ -3475,13 +3615,27 @@ cdef class Generator: # GH10839, ensure double to make tol meaningful cov = cov.astype(np.double) - (u, s, v) = svd(cov) + if method == 'svd': + from numpy.dual import svd + (u, s, vh) = svd(cov) + elif method == 'eigh': + from numpy.dual import eigh + # could call linalg.svd(hermitian=True), but that calculates a vh we don't need + (s, u) = eigh(cov) + else: + from numpy.dual import cholesky + l = cholesky(cov) - if check_valid != 'ignore': + # make sure check_valid is ignored whe method == 'cholesky' + # since the decomposition will have failed if cov is not valid. + if check_valid != 'ignore' and method != 'cholesky': if check_valid != 'warn' and check_valid != 'raise': - raise ValueError("check_valid must equal 'warn', 'raise', or 'ignore'") - - psd = np.allclose(np.dot(v.T * s, v), cov, rtol=tol, atol=tol) + raise ValueError( + "check_valid must equal 'warn', 'raise', or 'ignore'") + if method == 'svd': + psd = np.allclose(np.dot(vh.T * s, vh), cov, rtol=tol, atol=tol) + else: + psd = not np.any(s < -tol) if not psd: if check_valid == 'warn': warnings.warn("covariance is not positive-semidefinite.", @@ -3489,7 +3643,17 @@ cdef class Generator: else: raise ValueError("covariance is not positive-semidefinite.") - x = np.dot(x, np.sqrt(s)[:, None] * v) + if method == 'cholesky': + _factor = l + elif method == 'eigh': + # if check_valid == 'ignore' we need to ensure that np.sqrt does not + # return a NaN if s is a very small negative number that is + # approximately zero or when the covariance is not positive-semidefinite + _factor = u * np.sqrt(abs(s)) + else: + _factor = np.sqrt(s)[:, None] * vh + + x = np.dot(x, _factor) x += mean x.shape = tuple(final_shape) return x @@ -3644,6 +3808,222 @@ cdef class Generator: return multin + def multivariate_hypergeometric(self, object colors, object nsample, + size=None, method='marginals'): + """ + multivariate_hypergeometric(colors, nsample, size=None, + method='marginals') + + Generate variates from a multivariate hypergeometric distribution. + + The multivariate hypergeometric distribution is a generalization + of the hypergeometric distribution. + + Choose ``nsample`` items at random without replacement from a + collection with ``N`` distinct types. ``N`` is the length of + ``colors``, and the values in ``colors`` are the number of occurrences + of that type in the collection. The total number of items in the + collection is ``sum(colors)``. Each random variate generated by this + function is a vector of length ``N`` holding the counts of the + different types that occurred in the ``nsample`` items. + + The name ``colors`` comes from a common description of the + distribution: it is the probability distribution of the number of + marbles of each color selected without replacement from an urn + containing marbles of different colors; ``colors[i]`` is the number + of marbles in the urn with color ``i``. + + Parameters + ---------- + colors : sequence of integers + The number of each type of item in the collection from which + a sample is drawn. The values in ``colors`` must be nonnegative. + To avoid loss of precision in the algorithm, ``sum(colors)`` + must be less than ``10**9`` when `method` is "marginals". + nsample : int + The number of items selected. ``nsample`` must not be greater + than ``sum(colors)``. + size : int or tuple of ints, optional + The number of variates to generate, either an integer or a tuple + holding the shape of the array of variates. If the given size is, + e.g., ``(k, m)``, then ``k * m`` variates are drawn, where one + variate is a vector of length ``len(colors)``, and the return value + has shape ``(k, m, len(colors))``. If `size` is an integer, the + output has shape ``(size, len(colors))``. Default is None, in + which case a single variate is returned as an array with shape + ``(len(colors),)``. + method : string, optional + Specify the algorithm that is used to generate the variates. + Must be 'count' or 'marginals' (the default). See the Notes + for a description of the methods. + + Returns + ------- + variates : ndarray + Array of variates drawn from the multivariate hypergeometric + distribution. + + See Also + -------- + hypergeometric : Draw samples from the (univariate) hypergeometric + distribution. + + Notes + ----- + The two methods do not return the same sequence of variates. + + The "count" algorithm is roughly equivalent to the following numpy + code:: + + choices = np.repeat(np.arange(len(colors)), colors) + selection = np.random.choice(choices, nsample, replace=False) + variate = np.bincount(selection, minlength=len(colors)) + + The "count" algorithm uses a temporary array of integers with length + ``sum(colors)``. + + The "marginals" algorithm generates a variate by using repeated + calls to the univariate hypergeometric sampler. It is roughly + equivalent to:: + + variate = np.zeros(len(colors), dtype=np.int64) + # `remaining` is the cumulative sum of `colors` from the last + # element to the first; e.g. if `colors` is [3, 1, 5], then + # `remaining` is [9, 6, 5]. + remaining = np.cumsum(colors[::-1])[::-1] + for i in range(len(colors)-1): + if nsample < 1: + break + variate[i] = hypergeometric(colors[i], remaining[i+1], + nsample) + nsample -= variate[i] + variate[-1] = nsample + + The default method is "marginals". For some cases (e.g. when + `colors` contains relatively small integers), the "count" method + can be significantly faster than the "marginals" method. If + performance of the algorithm is important, test the two methods + with typical inputs to decide which works best. + + .. versionadded:: 1.18.0 + + Examples + -------- + >>> colors = [16, 8, 4] + >>> seed = 4861946401452 + >>> gen = np.random.Generator(np.random.PCG64(seed)) + >>> gen.multivariate_hypergeometric(colors, 6) + array([5, 0, 1]) + >>> gen.multivariate_hypergeometric(colors, 6, size=3) + array([[5, 0, 1], + [2, 2, 2], + [3, 3, 0]]) + >>> gen.multivariate_hypergeometric(colors, 6, size=(2, 2)) + array([[[3, 2, 1], + [3, 2, 1]], + [[4, 1, 1], + [3, 2, 1]]]) + """ + cdef int64_t nsamp + cdef size_t num_colors + cdef int64_t total + cdef int64_t *colors_ptr + cdef int64_t max_index + cdef size_t num_variates + cdef int64_t *variates_ptr + cdef int result + + if method not in ['count', 'marginals']: + raise ValueError('method must be "count" or "marginals".') + + try: + operator.index(nsample) + except TypeError: + raise ValueError('nsample must be an integer') + + if nsample < 0: + raise ValueError("nsample must be nonnegative.") + if nsample > INT64_MAX: + raise ValueError("nsample must not exceed %d" % INT64_MAX) + nsamp = nsample + + # Validation of colors, a 1-d sequence of nonnegative integers. + invalid_colors = False + try: + colors = np.asarray(colors) + if colors.ndim != 1: + invalid_colors = True + elif colors.size > 0 and not np.issubdtype(colors.dtype, + np.integer): + invalid_colors = True + elif np.any((colors < 0) | (colors > INT64_MAX)): + invalid_colors = True + except ValueError: + invalid_colors = True + if invalid_colors: + raise ValueError('colors must be a one-dimensional sequence ' + 'of nonnegative integers not exceeding %d.' % + INT64_MAX) + + colors = np.ascontiguousarray(colors, dtype=np.int64) + num_colors = colors.size + + colors_ptr = <int64_t *> np.PyArray_DATA(colors) + + total = _safe_sum_nonneg_int64(num_colors, colors_ptr) + if total == -1: + raise ValueError("sum(colors) must not exceed the maximum value " + "of a 64 bit signed integer (%d)" % INT64_MAX) + + if method == 'marginals' and total >= 1000000000: + raise ValueError('When method is "marginals", sum(colors) must ' + 'be less than 1000000000.') + + # The C code that implements the 'count' method will malloc an + # array of size total*sizeof(size_t). Here we ensure that that + # product does not overflow. + if SIZE_MAX > <uint64_t>INT64_MAX: + max_index = INT64_MAX // sizeof(size_t) + else: + max_index = SIZE_MAX // sizeof(size_t) + if method == 'count' and total > max_index: + raise ValueError("When method is 'count', sum(colors) must not " + "exceed %d" % max_index) + if nsamp > total: + raise ValueError("nsample > sum(colors)") + + # Figure out the shape of the return array. + if size is None: + shape = (num_colors,) + elif np.isscalar(size): + shape = (size, num_colors) + else: + shape = tuple(size) + (num_colors,) + variates = np.zeros(shape, dtype=np.int64) + + if num_colors == 0: + return variates + + # One variate is a vector of length num_colors. + num_variates = variates.size // num_colors + variates_ptr = <int64_t *> np.PyArray_DATA(variates) + + if method == 'count': + with self.lock, nogil: + result = random_multivariate_hypergeometric_count(&self._bitgen, + total, num_colors, colors_ptr, nsamp, + num_variates, variates_ptr) + if result == -1: + raise MemoryError("Insufficent memory for multivariate_" + "hypergeometric with method='count' and " + "sum(colors)=%d" % total) + else: + with self.lock, nogil: + random_multivariate_hypergeometric_marginals(&self._bitgen, + total, num_colors, colors_ptr, nsamp, + num_variates, variates_ptr) + return variates + def dirichlet(self, object alpha, size=None): """ dirichlet(alpha, size=None) @@ -3772,7 +4152,7 @@ cdef class Generator: while i < totsize: acc = 0.0 for j in range(k): - val_data[i+j] = random_standard_gamma_zig(&self._bitgen, + val_data[i+j] = random_standard_gamma(&self._bitgen, alpha_data[j]) acc = acc + val_data[i + j] invacc = 1/acc @@ -3783,20 +4163,21 @@ cdef class Generator: return diric # Shuffling and permutations: - def shuffle(self, object x): + def shuffle(self, object x, axis=0): """ - shuffle(x) + shuffle(x, axis=0) Modify a sequence in-place by shuffling its contents. - This function only shuffles the array along the first axis of a - multi-dimensional array. The order of sub-arrays is changed but - their contents remains the same. + The order of sub-arrays is changed but their contents remains the same. Parameters ---------- x : array_like The array or list to be shuffled. + axis : int, optional + The axis which `x` is shuffled along. Default is 0. + It is only supported on `ndarray` objects. Returns ------- @@ -3810,8 +4191,6 @@ cdef class Generator: >>> arr [1 7 5 2 9 4 3 6 0 8] # random - Multi-dimensional arrays are only shuffled along the first axis: - >>> arr = np.arange(9).reshape((3, 3)) >>> rng.shuffle(arr) >>> arr @@ -3819,17 +4198,25 @@ cdef class Generator: [6, 7, 8], [0, 1, 2]]) + >>> arr = np.arange(9).reshape((3, 3)) + >>> rng.shuffle(arr, axis=1) + >>> arr + array([[2, 0, 1], # random + [5, 3, 4], + [8, 6, 7]]) """ cdef: np.npy_intp i, j, n = len(x), stride, itemsize char* x_ptr char* buf_ptr + axis = normalize_axis_index(axis, np.ndim(x)) + if type(x) is np.ndarray and x.ndim == 1 and x.size: # Fast, statically typed path: shuffle the underlying buffer. # Only for non-empty, 1d objects of class ndarray (subclasses such # as MaskedArrays may not support this approach). - x_ptr = <char*><size_t>x.ctypes.data + x_ptr = <char*><size_t>np.PyArray_DATA(x) stride = x.strides[0] itemsize = x.dtype.itemsize # As the array x could contain python objects we use a buffer @@ -3837,7 +4224,7 @@ cdef class Generator: # within the buffer and erroneously decrementing it's refcount # when the function exits. buf = np.empty(itemsize, dtype=np.int8) # GC'd at function exit - buf_ptr = <char*><size_t>buf.ctypes.data + buf_ptr = <char*><size_t>np.PyArray_DATA(buf) with self.lock: # We trick gcc into providing a specialized implementation for # the most common case, yielding a ~33% performance improvement. @@ -3847,9 +4234,10 @@ cdef class Generator: else: self._shuffle_raw(n, 1, itemsize, stride, x_ptr, buf_ptr) elif isinstance(x, np.ndarray) and x.ndim and x.size: + x = np.swapaxes(x, 0, axis) buf = np.empty_like(x[0, ...]) with self.lock: - for i in reversed(range(1, n)): + for i in reversed(range(1, len(x))): j = random_interval(&self._bitgen, i) if i == j: # i == j is not needed and memcpy is undefined. @@ -3859,6 +4247,9 @@ cdef class Generator: x[i] = buf else: # Untyped path. + if axis != 0: + raise NotImplementedError("Axis argument is only supported " + "on ndarray objects") with self.lock: for i in reversed(range(1, n)): j = random_interval(&self._bitgen, i) @@ -3914,13 +4305,11 @@ cdef class Generator: data[j] = data[i] data[i] = temp - def permutation(self, object x): + def permutation(self, object x, axis=0): """ - permutation(x) + permutation(x, axis=0) Randomly permute a sequence, or return a permuted range. - If `x` is a multi-dimensional array, it is only shuffled along its - first index. Parameters ---------- @@ -3928,6 +4317,8 @@ cdef class Generator: If `x` is an integer, randomly permute ``np.arange(x)``. If `x` is an array, make a copy and shuffle the elements randomly. + axis : int, optional + The axis which `x` is shuffled along. Default is 0. Returns ------- @@ -3953,16 +4344,22 @@ cdef class Generator: Traceback (most recent call last): ... numpy.AxisError: x must be an integer or at least 1-dimensional - """ + >>> arr = np.arange(9).reshape((3, 3)) + >>> rng.permutation(arr, axis=1) + array([[0, 2, 1], # random + [3, 5, 4], + [6, 8, 7]]) + + """ if isinstance(x, (int, np.integer)): arr = np.arange(x) self.shuffle(arr) return arr arr = np.asarray(x) - if arr.ndim < 1: - raise np.AxisError("x must be an integer or at least 1-dimensional") + + axis = normalize_axis_index(axis, arr.ndim) # shuffle has fast-path for 1-d if arr.ndim == 1: @@ -3973,9 +4370,11 @@ cdef class Generator: return arr # Shuffle index array, dtype to ensure fast path - idx = np.arange(arr.shape[0], dtype=np.intp) + idx = np.arange(arr.shape[axis], dtype=np.intp) self.shuffle(idx) - return arr[idx] + slices = [slice(None)]*arr.ndim + slices[axis] = idx + return arr[tuple(slices)] def default_rng(seed=None): @@ -3983,21 +4382,24 @@ def default_rng(seed=None): Parameters ---------- - seed : {None, int, array_like[ints], ISeedSequence, BitGenerator, Generator}, optional + seed : {None, int, array_like[ints], SeedSequence, BitGenerator, Generator}, optional A seed to initialize the `BitGenerator`. If None, then fresh, unpredictable entropy will be pulled from the OS. If an ``int`` or ``array_like[ints]`` is passed, then it will be passed to `SeedSequence` to derive the initial `BitGenerator` state. One may also - pass in an implementor of the `ISeedSequence` interface like - `SeedSequence`. + pass in a`SeedSequence` instance Additionally, when passed a `BitGenerator`, it will be wrapped by `Generator`. If passed a `Generator`, it will be returned unaltered. + Returns + ------- + Generator + The initialized generator object. + Notes ----- - When `seed` is omitted or ``None``, a new `BitGenerator` and `Generator` will - be instantiated each time. This function does not manage a default global - instance. + If ``seed`` is not a `BitGenerator` or a `Generator`, a new `BitGenerator` + is instantiated. This function does not manage a default global instance. """ if _check_bit_generator(seed): # We were passed a BitGenerator, so just wrap it up. diff --git a/numpy/random/mt19937.pyx b/numpy/random/_mt19937.pyx index 7d0f6cd22..919a96a4c 100644 --- a/numpy/random/mt19937.pyx +++ b/numpy/random/_mt19937.pyx @@ -3,8 +3,8 @@ import operator import numpy as np cimport numpy as np -from .common cimport * -from .bit_generator cimport BitGenerator, SeedSequence +from libc.stdint cimport uint32_t, uint64_t +from numpy.random cimport BitGenerator, SeedSequence __all__ = ['MT19937'] @@ -48,13 +48,12 @@ cdef class MT19937(BitGenerator): Parameters ---------- - seed : {None, int, array_like[ints], ISeedSequence}, optional + seed : {None, int, array_like[ints], SeedSequence}, optional A seed to initialize the `BitGenerator`. If None, then fresh, unpredictable entropy will be pulled from the OS. If an ``int`` or ``array_like[ints]`` is passed, then it will be passed to `SeedSequence` to derive the initial `BitGenerator` state. One may also - pass in an implementor of the `ISeedSequence` interface like - `SeedSequence`. + pass in a `SeedSequence` instance. Attributes ---------- diff --git a/numpy/random/pcg64.pyx b/numpy/random/_pcg64.pyx index 585520139..05d27db5c 100644 --- a/numpy/random/pcg64.pyx +++ b/numpy/random/_pcg64.pyx @@ -1,8 +1,9 @@ import numpy as np cimport numpy as np -from .common cimport * -from .bit_generator cimport BitGenerator +from libc.stdint cimport uint32_t, uint64_t +from ._common cimport uint64_to_double, wrap_int +from numpy.random cimport BitGenerator __all__ = ['PCG64'] @@ -43,13 +44,12 @@ cdef class PCG64(BitGenerator): Parameters ---------- - seed : {None, int, array_like[ints], ISeedSequence}, optional + seed : {None, int, array_like[ints], SeedSequence}, optional A seed to initialize the `BitGenerator`. If None, then fresh, unpredictable entropy will be pulled from the OS. If an ``int`` or ``array_like[ints]`` is passed, then it will be passed to `SeedSequence` to derive the initial `BitGenerator` state. One may also - pass in an implementor of the `ISeedSequence` interface like - `SeedSequence`. + pass in a `SeedSequence` instance. Notes ----- diff --git a/numpy/random/philox.pyx b/numpy/random/_philox.pyx index 8b7683017..7e8880490 100644 --- a/numpy/random/philox.pyx +++ b/numpy/random/_philox.pyx @@ -6,9 +6,11 @@ except ImportError: from dummy_threading import Lock import numpy as np +cimport numpy as np -from .common cimport * -from .bit_generator cimport BitGenerator +from libc.stdint cimport uint32_t, uint64_t +from ._common cimport uint64_to_double, int_to_array, wrap_int +from numpy.random cimport BitGenerator __all__ = ['Philox'] @@ -62,21 +64,20 @@ cdef class Philox(BitGenerator): Parameters ---------- - seed : {None, int, array_like[ints], ISeedSequence}, optional + seed : {None, int, array_like[ints], SeedSequence}, optional A seed to initialize the `BitGenerator`. If None, then fresh, unpredictable entropy will be pulled from the OS. If an ``int`` or ``array_like[ints]`` is passed, then it will be passed to `SeedSequence` to derive the initial `BitGenerator` state. One may also - pass in an implementor of the `ISeedSequence` interface like - `SeedSequence`. + pass in a `SeedSequence` instance. counter : {None, int, array_like}, optional Counter to use in the Philox state. Can be either a Python int (long in 2.x) in [0, 2**256) or a 4-element uint64 array. If not provided, the RNG is initialized at 0. key : {None, int, array_like}, optional - Key to use in the Philox state. Unlike seed, the value in key is + Key to use in the Philox state. Unlike ``seed``, the value in key is directly set. Can be either a Python int in [0, 2**128) or a 2-element - uint64 array. `key` and `seed` cannot both be used. + uint64 array. `key` and ``seed`` cannot both be used. Attributes ---------- @@ -108,10 +109,10 @@ cdef class Philox(BitGenerator): randoms produced. The second is a key which determined the sequence produced. Using different keys produces independent sequences. - The input seed is processed by `SeedSequence` to generate the key. The + The input ``seed`` is processed by `SeedSequence` to generate the key. The counter is set to 0. - Alternately, one can omit the seed parameter and set the ``key`` and + Alternately, one can omit the ``seed`` parameter and set the ``key`` and ``counter`` directly. **Parallel Features** @@ -146,7 +147,7 @@ cdef class Philox(BitGenerator): **Compatibility Guarantee** - ``Philox`` makes a guarantee that a fixed seed will always produce + ``Philox`` makes a guarantee that a fixed ``seed`` will always produce the same random integer stream. Examples diff --git a/numpy/random/_pickle.py b/numpy/random/_pickle.py index 3b58f21e8..29ff69644 100644 --- a/numpy/random/_pickle.py +++ b/numpy/random/_pickle.py @@ -1,10 +1,10 @@ from .mtrand import RandomState -from .philox import Philox -from .pcg64 import PCG64 -from .sfc64 import SFC64 +from ._philox import Philox +from ._pcg64 import PCG64 +from ._sfc64 import SFC64 -from .generator import Generator -from .mt19937 import MT19937 +from ._generator import Generator +from ._mt19937 import MT19937 BitGenerators = {'MT19937': MT19937, 'PCG64': PCG64, diff --git a/numpy/random/sfc64.pyx b/numpy/random/_sfc64.pyx index a881096e9..1daee34f8 100644 --- a/numpy/random/sfc64.pyx +++ b/numpy/random/_sfc64.pyx @@ -1,8 +1,9 @@ import numpy as np cimport numpy as np -from .common cimport * -from .bit_generator cimport BitGenerator +from libc.stdint cimport uint32_t, uint64_t +from ._common cimport uint64_to_double +from numpy.random cimport BitGenerator __all__ = ['SFC64'] @@ -38,13 +39,12 @@ cdef class SFC64(BitGenerator): Parameters ---------- - seed : {None, int, array_like[ints], ISeedSequence}, optional + seed : {None, int, array_like[ints], SeedSequence}, optional A seed to initialize the `BitGenerator`. If None, then fresh, unpredictable entropy will be pulled from the OS. If an ``int`` or ``array_like[ints]`` is passed, then it will be passed to `SeedSequence` to derive the initial `BitGenerator` state. One may also - pass in an implementor of the `ISeedSequence` interface like - `SeedSequence`. + pass in a `SeedSequence` instance. Notes ----- diff --git a/numpy/random/distributions.pxd b/numpy/random/distributions.pxd deleted file mode 100644 index 75edaee9d..000000000 --- a/numpy/random/distributions.pxd +++ /dev/null @@ -1,140 +0,0 @@ -#cython: language_level=3 - -from .common cimport (uint8_t, uint16_t, uint32_t, uint64_t, - int32_t, int64_t, bitgen_t) -import numpy as np -cimport numpy as np - -cdef extern from "src/distributions/distributions.h": - - struct s_binomial_t: - int has_binomial - double psave - int64_t nsave - double r - double q - double fm - int64_t m - double p1 - double xm - double xl - double xr - double c - double laml - double lamr - double p2 - double p3 - double p4 - - ctypedef s_binomial_t binomial_t - - double random_double(bitgen_t *bitgen_state) nogil - void random_double_fill(bitgen_t* bitgen_state, np.npy_intp cnt, double *out) nogil - double random_standard_exponential(bitgen_t *bitgen_state) nogil - void random_standard_exponential_fill(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil - double random_standard_exponential_zig(bitgen_t *bitgen_state) nogil - void random_standard_exponential_zig_fill(bitgen_t *bitgen_state, np.npy_intp cnt, double *out) nogil - double random_gauss_zig(bitgen_t* bitgen_state) nogil - void random_gauss_zig_fill(bitgen_t *bitgen_state, np.npy_intp count, double *out) nogil - double random_standard_gamma_zig(bitgen_t *bitgen_state, double shape) nogil - - float random_float(bitgen_t *bitgen_state) nogil - float random_standard_exponential_f(bitgen_t *bitgen_state) nogil - float random_standard_exponential_zig_f(bitgen_t *bitgen_state) nogil - float random_gauss_zig_f(bitgen_t* bitgen_state) nogil - float random_standard_gamma_f(bitgen_t *bitgen_state, float shape) nogil - float random_standard_gamma_zig_f(bitgen_t *bitgen_state, float shape) nogil - - int64_t random_positive_int64(bitgen_t *bitgen_state) nogil - int32_t random_positive_int32(bitgen_t *bitgen_state) nogil - int64_t random_positive_int(bitgen_t *bitgen_state) nogil - uint64_t random_uint(bitgen_t *bitgen_state) nogil - - double random_normal_zig(bitgen_t *bitgen_state, double loc, double scale) nogil - - double random_gamma(bitgen_t *bitgen_state, double shape, double scale) nogil - float random_gamma_float(bitgen_t *bitgen_state, float shape, float scale) nogil - - double random_exponential(bitgen_t *bitgen_state, double scale) nogil - double random_uniform(bitgen_t *bitgen_state, double lower, double range) nogil - double random_beta(bitgen_t *bitgen_state, double a, double b) nogil - double random_chisquare(bitgen_t *bitgen_state, double df) nogil - double random_f(bitgen_t *bitgen_state, double dfnum, double dfden) nogil - double random_standard_cauchy(bitgen_t *bitgen_state) nogil - double random_pareto(bitgen_t *bitgen_state, double a) nogil - double random_weibull(bitgen_t *bitgen_state, double a) nogil - double random_power(bitgen_t *bitgen_state, double a) nogil - double random_laplace(bitgen_t *bitgen_state, double loc, double scale) nogil - double random_gumbel(bitgen_t *bitgen_state, double loc, double scale) nogil - double random_logistic(bitgen_t *bitgen_state, double loc, double scale) nogil - double random_lognormal(bitgen_t *bitgen_state, double mean, double sigma) nogil - double random_rayleigh(bitgen_t *bitgen_state, double mode) nogil - double random_standard_t(bitgen_t *bitgen_state, double df) nogil - double random_noncentral_chisquare(bitgen_t *bitgen_state, double df, - double nonc) nogil - double random_noncentral_f(bitgen_t *bitgen_state, double dfnum, - double dfden, double nonc) nogil - double random_wald(bitgen_t *bitgen_state, double mean, double scale) nogil - double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) nogil - double random_triangular(bitgen_t *bitgen_state, double left, double mode, - double right) nogil - - int64_t random_poisson(bitgen_t *bitgen_state, double lam) nogil - int64_t random_negative_binomial(bitgen_t *bitgen_state, double n, double p) nogil - int64_t random_binomial(bitgen_t *bitgen_state, double p, int64_t n, binomial_t *binomial) nogil - int64_t random_logseries(bitgen_t *bitgen_state, double p) nogil - int64_t random_geometric_search(bitgen_t *bitgen_state, double p) nogil - int64_t random_geometric_inversion(bitgen_t *bitgen_state, double p) nogil - int64_t random_geometric(bitgen_t *bitgen_state, double p) nogil - int64_t random_zipf(bitgen_t *bitgen_state, double a) nogil - int64_t random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad, - int64_t sample) nogil - - uint64_t random_interval(bitgen_t *bitgen_state, uint64_t max) nogil - - # Generate random uint64 numbers in closed interval [off, off + rng]. - uint64_t random_bounded_uint64(bitgen_t *bitgen_state, - uint64_t off, uint64_t rng, - uint64_t mask, bint use_masked) nogil - - # Generate random uint32 numbers in closed interval [off, off + rng]. - uint32_t random_buffered_bounded_uint32(bitgen_t *bitgen_state, - uint32_t off, uint32_t rng, - uint32_t mask, bint use_masked, - int *bcnt, uint32_t *buf) nogil - uint16_t random_buffered_bounded_uint16(bitgen_t *bitgen_state, - uint16_t off, uint16_t rng, - uint16_t mask, bint use_masked, - int *bcnt, uint32_t *buf) nogil - uint8_t random_buffered_bounded_uint8(bitgen_t *bitgen_state, - uint8_t off, uint8_t rng, - uint8_t mask, bint use_masked, - int *bcnt, uint32_t *buf) nogil - np.npy_bool random_buffered_bounded_bool(bitgen_t *bitgen_state, - np.npy_bool off, np.npy_bool rng, - np.npy_bool mask, bint use_masked, - int *bcnt, uint32_t *buf) nogil - - void random_bounded_uint64_fill(bitgen_t *bitgen_state, - uint64_t off, uint64_t rng, np.npy_intp cnt, - bint use_masked, - uint64_t *out) nogil - void random_bounded_uint32_fill(bitgen_t *bitgen_state, - uint32_t off, uint32_t rng, np.npy_intp cnt, - bint use_masked, - uint32_t *out) nogil - void random_bounded_uint16_fill(bitgen_t *bitgen_state, - uint16_t off, uint16_t rng, np.npy_intp cnt, - bint use_masked, - uint16_t *out) nogil - void random_bounded_uint8_fill(bitgen_t *bitgen_state, - uint8_t off, uint8_t rng, np.npy_intp cnt, - bint use_masked, - uint8_t *out) nogil - void random_bounded_bool_fill(bitgen_t *bitgen_state, - np.npy_bool off, np.npy_bool rng, np.npy_intp cnt, - bint use_masked, - np.npy_bool *out) nogil - - void random_multinomial(bitgen_t *bitgen_state, int64_t n, int64_t *mnix, - double *pix, np.npy_intp d, binomial_t *binomial) nogil diff --git a/numpy/random/examples/cython/setup.py b/numpy/random/examples/cython/setup.py deleted file mode 100644 index 69f057ed5..000000000 --- a/numpy/random/examples/cython/setup.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -""" -Build the demos - -Usage: python setup.py build_ext -i -""" - -import numpy as np -from distutils.core import setup -from Cython.Build import cythonize -from setuptools.extension import Extension -from os.path import join - -extending = Extension("extending", - sources=['extending.pyx'], - include_dirs=[np.get_include()]) -distributions = Extension("extending_distributions", - sources=['extending_distributions.pyx', - join('..', '..', 'src', - 'distributions', 'distributions.c')], - include_dirs=[np.get_include()]) - -extensions = [extending, distributions] - -setup( - ext_modules=cythonize(extensions) -) diff --git a/numpy/random/examples/numba/extending.py b/numpy/random/examples/numba/extending.py deleted file mode 100644 index d41c2d76f..000000000 --- a/numpy/random/examples/numba/extending.py +++ /dev/null @@ -1,77 +0,0 @@ -import datetime as dt - -import numpy as np -import numba as nb - -from numpy.random import PCG64 - -x = PCG64() -f = x.ctypes.next_uint32 -s = x.ctypes.state - - -@nb.jit(nopython=True) -def bounded_uint(lb, ub, state): - mask = delta = ub - lb - mask |= mask >> 1 - mask |= mask >> 2 - mask |= mask >> 4 - mask |= mask >> 8 - mask |= mask >> 16 - - val = f(state) & mask - while val > delta: - val = f(state) & mask - - return lb + val - - -print(bounded_uint(323, 2394691, s.value)) - - -@nb.jit(nopython=True) -def bounded_uints(lb, ub, n, state): - out = np.empty(n, dtype=np.uint32) - for i in range(n): - out[i] = bounded_uint(lb, ub, state) - - -bounded_uints(323, 2394691, 10000000, s.value) - -g = x.cffi.next_double -cffi_state = x.cffi.state -state_addr = x.cffi.state_address - - -def normals(n, state): - out = np.empty(n) - for i in range((n + 1) // 2): - x1 = 2.0 * g(state) - 1.0 - x2 = 2.0 * g(state) - 1.0 - r2 = x1 * x1 + x2 * x2 - while r2 >= 1.0 or r2 == 0.0: - x1 = 2.0 * g(state) - 1.0 - x2 = 2.0 * g(state) - 1.0 - r2 = x1 * x1 + x2 * x2 - f = np.sqrt(-2.0 * np.log(r2) / r2) - out[2 * i] = f * x1 - if 2 * i + 1 < n: - out[2 * i + 1] = f * x2 - return out - - -print(normals(10, cffi_state).var()) -# Warm up -normalsj = nb.jit(normals, nopython=True) -normalsj(1, state_addr) - -start = dt.datetime.now() -normalsj(1000000, state_addr) -ms = 1000 * (dt.datetime.now() - start).total_seconds() -print('1,000,000 Polar-transform (numba/PCG64) randoms in ' - '{ms:0.1f}ms'.format(ms=ms)) - -start = dt.datetime.now() -np.random.standard_normal(1000000) -ms = 1000 * (dt.datetime.now() - start).total_seconds() -print('1,000,000 Polar-transform (NumPy) randoms in {ms:0.1f}ms'.format(ms=ms)) diff --git a/numpy/random/src/aligned_malloc/aligned_malloc.h b/numpy/random/include/aligned_malloc.h index ea24f6d23..ea24f6d23 100644 --- a/numpy/random/src/aligned_malloc/aligned_malloc.h +++ b/numpy/random/include/aligned_malloc.h diff --git a/numpy/random/src/legacy/legacy-distributions.h b/numpy/random/include/legacy-distributions.h index 3fd3fd267..b8ba0841c 100644 --- a/numpy/random/src/legacy/legacy-distributions.h +++ b/numpy/random/include/legacy-distributions.h @@ -2,7 +2,7 @@ #define _RANDOMDGEN__DISTRIBUTIONS_LEGACY_H_ -#include "../distributions/distributions.h" +#include "numpy/random/distributions.h" typedef struct aug_bitgen { bitgen_t *bit_generator; @@ -16,20 +16,15 @@ extern double legacy_pareto(aug_bitgen_t *aug_state, double a); extern double legacy_weibull(aug_bitgen_t *aug_state, double a); extern double legacy_power(aug_bitgen_t *aug_state, double a); extern double legacy_gamma(aug_bitgen_t *aug_state, double shape, double scale); -extern double legacy_pareto(aug_bitgen_t *aug_state, double a); -extern double legacy_weibull(aug_bitgen_t *aug_state, double a); extern double legacy_chisquare(aug_bitgen_t *aug_state, double df); extern double legacy_noncentral_chisquare(aug_bitgen_t *aug_state, double df, double nonc); - extern double legacy_noncentral_f(aug_bitgen_t *aug_state, double dfnum, double dfden, double nonc); extern double legacy_wald(aug_bitgen_t *aug_state, double mean, double scale); extern double legacy_lognormal(aug_bitgen_t *aug_state, double mean, double sigma); extern double legacy_standard_t(aug_bitgen_t *aug_state, double df); -extern int64_t legacy_negative_binomial(aug_bitgen_t *aug_state, double n, - double p); extern double legacy_standard_cauchy(aug_bitgen_t *state); extern double legacy_beta(aug_bitgen_t *aug_state, double a, double b); extern double legacy_f(aug_bitgen_t *aug_state, double dfnum, double dfden); diff --git a/numpy/random/info.py b/numpy/random/info.py deleted file mode 100644 index b9fd7f26a..000000000 --- a/numpy/random/info.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import division, absolute_import, print_function - -from .. import __doc__ - -depends = ['core'] diff --git a/numpy/random/legacy_distributions.pxd b/numpy/random/legacy_distributions.pxd deleted file mode 100644 index c681388db..000000000 --- a/numpy/random/legacy_distributions.pxd +++ /dev/null @@ -1,50 +0,0 @@ -#cython: language_level=3 - -from libc.stdint cimport int64_t - -import numpy as np -cimport numpy as np - -from .distributions cimport bitgen_t, binomial_t - -cdef extern from "legacy-distributions.h": - - struct aug_bitgen: - bitgen_t *bit_generator - int has_gauss - double gauss - - ctypedef aug_bitgen aug_bitgen_t - - double legacy_gauss(aug_bitgen_t *aug_state) nogil - double legacy_pareto(aug_bitgen_t *aug_state, double a) nogil - double legacy_weibull(aug_bitgen_t *aug_state, double a) nogil - double legacy_standard_gamma(aug_bitgen_t *aug_state, double shape) nogil - double legacy_normal(aug_bitgen_t *aug_state, double loc, double scale) nogil - double legacy_standard_t(aug_bitgen_t *aug_state, double df) nogil - - double legacy_standard_exponential(aug_bitgen_t *aug_state) nogil - double legacy_power(aug_bitgen_t *aug_state, double a) nogil - double legacy_gamma(aug_bitgen_t *aug_state, double shape, double scale) nogil - double legacy_power(aug_bitgen_t *aug_state, double a) nogil - double legacy_chisquare(aug_bitgen_t *aug_state, double df) nogil - double legacy_noncentral_chisquare(aug_bitgen_t *aug_state, double df, - double nonc) nogil - double legacy_noncentral_f(aug_bitgen_t *aug_state, double dfnum, double dfden, - double nonc) nogil - double legacy_wald(aug_bitgen_t *aug_state, double mean, double scale) nogil - double legacy_lognormal(aug_bitgen_t *aug_state, double mean, double sigma) nogil - int64_t legacy_random_binomial(bitgen_t *bitgen_state, double p, - int64_t n, binomial_t *binomial) nogil - int64_t legacy_negative_binomial(aug_bitgen_t *aug_state, double n, double p) nogil - int64_t legacy_random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad, int64_t sample) nogil - int64_t legacy_random_logseries(bitgen_t *bitgen_state, double p) nogil - int64_t legacy_random_poisson(bitgen_t *bitgen_state, double lam) nogil - int64_t legacy_random_zipf(bitgen_t *bitgen_state, double a) nogil - int64_t legacy_random_geometric(bitgen_t *bitgen_state, double p) nogil - void legacy_random_multinomial(bitgen_t *bitgen_state, long n, long *mnix, double *pix, np.npy_intp d, binomial_t *binomial) nogil - double legacy_standard_cauchy(aug_bitgen_t *state) nogil - double legacy_beta(aug_bitgen_t *aug_state, double a, double b) nogil - double legacy_f(aug_bitgen_t *aug_state, double dfnum, double dfden) nogil - double legacy_exponential(aug_bitgen_t *aug_state, double scale) nogil - double legacy_power(aug_bitgen_t *state, double a) nogil diff --git a/numpy/random/mtrand.pyx b/numpy/random/mtrand.pyx index 811b76614..a4d409f37 100644 --- a/numpy/random/mtrand.pyx +++ b/numpy/random/mtrand.pyx @@ -5,19 +5,100 @@ import warnings import numpy as np -from .bounded_integers import _integers_types -from .mt19937 import MT19937 as _MT19937 from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer from cpython cimport (Py_INCREF, PyFloat_AsDouble) -from libc cimport string - cimport cython cimport numpy as np -from .bounded_integers cimport * -from .common cimport * -from .distributions cimport * -from .legacy_distributions cimport * +from libc cimport string +from libc.stdint cimport int64_t, uint64_t +from ._bounded_integers cimport (_rand_bool, _rand_int32, _rand_int64, + _rand_int16, _rand_int8, _rand_uint64, _rand_uint32, _rand_uint16, + _rand_uint8,) +from ._bounded_integers import _integers_types +from ._mt19937 import MT19937 as _MT19937 +from numpy.random cimport bitgen_t +from ._common cimport (POISSON_LAM_MAX, CONS_POSITIVE, CONS_NONE, + CONS_NON_NEGATIVE, CONS_BOUNDED_0_1, CONS_BOUNDED_GT_0_1, CONS_GTE_1, + CONS_GT_1, LEGACY_CONS_POISSON, + double_fill, cont, kahan_sum, cont_broadcast_3, + check_array_constraint, check_constraint, disc, discrete_broadcast_iii, + ) + +cdef extern from "numpy/random/distributions.h": + struct s_binomial_t: + int has_binomial + double psave + int64_t nsave + double r + double q + double fm + int64_t m + double p1 + double xm + double xl + double xr + double c + double laml + double lamr + double p2 + double p3 + double p4 + + ctypedef s_binomial_t binomial_t + + void random_standard_uniform_fill(bitgen_t* bitgen_state, np.npy_intp cnt, double *out) nogil + int64_t random_positive_int(bitgen_t *bitgen_state) nogil + double random_uniform(bitgen_t *bitgen_state, double lower, double range) nogil + double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) nogil + double random_laplace(bitgen_t *bitgen_state, double loc, double scale) nogil + double random_gumbel(bitgen_t *bitgen_state, double loc, double scale) nogil + double random_logistic(bitgen_t *bitgen_state, double loc, double scale) nogil + double random_rayleigh(bitgen_t *bitgen_state, double mode) nogil + double random_triangular(bitgen_t *bitgen_state, double left, double mode, + double right) nogil + uint64_t random_interval(bitgen_t *bitgen_state, uint64_t max) nogil + +cdef extern from "include/legacy-distributions.h": + struct aug_bitgen: + bitgen_t *bit_generator + int has_gauss + double gauss + + ctypedef aug_bitgen aug_bitgen_t + + double legacy_gauss(aug_bitgen_t *aug_state) nogil + double legacy_pareto(aug_bitgen_t *aug_state, double a) nogil + double legacy_weibull(aug_bitgen_t *aug_state, double a) nogil + double legacy_standard_gamma(aug_bitgen_t *aug_state, double shape) nogil + double legacy_normal(aug_bitgen_t *aug_state, double loc, double scale) nogil + double legacy_standard_t(aug_bitgen_t *aug_state, double df) nogil + + double legacy_standard_exponential(aug_bitgen_t *aug_state) nogil + double legacy_power(aug_bitgen_t *aug_state, double a) nogil + double legacy_gamma(aug_bitgen_t *aug_state, double shape, double scale) nogil + double legacy_power(aug_bitgen_t *aug_state, double a) nogil + double legacy_chisquare(aug_bitgen_t *aug_state, double df) nogil + double legacy_noncentral_chisquare(aug_bitgen_t *aug_state, double df, + double nonc) nogil + double legacy_noncentral_f(aug_bitgen_t *aug_state, double dfnum, double dfden, + double nonc) nogil + double legacy_wald(aug_bitgen_t *aug_state, double mean, double scale) nogil + double legacy_lognormal(aug_bitgen_t *aug_state, double mean, double sigma) nogil + int64_t legacy_random_binomial(bitgen_t *bitgen_state, double p, + int64_t n, binomial_t *binomial) nogil + int64_t legacy_negative_binomial(aug_bitgen_t *aug_state, double n, double p) nogil + int64_t legacy_random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad, int64_t sample) nogil + int64_t legacy_random_logseries(bitgen_t *bitgen_state, double p) nogil + int64_t legacy_random_poisson(bitgen_t *bitgen_state, double lam) nogil + int64_t legacy_random_zipf(bitgen_t *bitgen_state, double a) nogil + int64_t legacy_random_geometric(bitgen_t *bitgen_state, double p) nogil + void legacy_random_multinomial(bitgen_t *bitgen_state, long n, long *mnix, double *pix, np.npy_intp d, binomial_t *binomial) nogil + double legacy_standard_cauchy(aug_bitgen_t *state) nogil + double legacy_beta(aug_bitgen_t *aug_state, double a, double b) nogil + double legacy_f(aug_bitgen_t *aug_state, double dfnum, double dfden) nogil + double legacy_exponential(aug_bitgen_t *aug_state, double scale) nogil + double legacy_power(aug_bitgen_t *state, double a) nogil np.import_array() @@ -84,7 +165,7 @@ cdef class RandomState: -------- Generator MT19937 - :ref:`bit_generator` + numpy.random.BitGenerator """ cdef public object _bit_generator @@ -298,6 +379,10 @@ cdef class RandomState: (b - a) * random_sample() + a + .. note:: + New code should use the ``random`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- size : int or tuple of ints, optional @@ -311,6 +396,10 @@ cdef class RandomState: Array of random floats of shape `size` (unless ``size=None``, in which case a single float is returned). + See Also + -------- + Generator.random: which should be used for new code. + Examples -------- >>> np.random.random_sample() @@ -329,7 +418,7 @@ cdef class RandomState: """ cdef double temp - return double_fill(&random_double_fill, &self._bitgen, size, self.lock, None) + return double_fill(&random_standard_uniform_fill, &self._bitgen, size, self.lock, None) def random(self, size=None): """ @@ -360,6 +449,10 @@ cdef class RandomState: It is often seen in Bayesian inference and order statistics. + .. note:: + New code should use the ``beta`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- a : float or array_like of floats @@ -377,6 +470,9 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized beta distribution. + See Also + -------- + Generator.beta: which should be used for new code. """ return cont(&legacy_beta, &self._aug_state, size, self.lock, 2, a, 'a', CONS_POSITIVE, @@ -403,6 +499,10 @@ cdef class RandomState: the size of raindrops measured over many rainstorms [1]_, or the time between page requests to Wikipedia [2]_. + .. note:: + New code should use the ``exponential`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- scale : float or array_like of floats @@ -419,6 +519,10 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized exponential distribution. + See Also + -------- + Generator.exponential: which should be used for new code. + References ---------- .. [1] Peyton Z. Peebles Jr., "Probability, Random Variables and @@ -444,6 +548,10 @@ cdef class RandomState: `standard_exponential` is identical to the exponential distribution with a scale parameter of 1. + .. note:: + New code should use the ``standard_exponential`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- size : int or tuple of ints, optional @@ -456,6 +564,10 @@ cdef class RandomState: out : float or ndarray Drawn samples. + See Also + -------- + Generator.standard_exponential: which should be used for new code. + Examples -------- Output a 3x8000 array: @@ -474,7 +586,7 @@ cdef class RandomState: tomaxint(size=None) Return a sample of uniformly distributed random integers in the interval - [0, ``np.iinfo(np.int).max``]. The np.int type translates to the C long + [0, ``np.iinfo(np.int_).max``]. The `np.int_` type translates to the C long integer type and its precision is platform dependent. Parameters @@ -503,7 +615,7 @@ cdef class RandomState: [ 739731006, 1947757578]], [[1871712945, 752307660], [1601631370, 1479324245]]]) - >>> rs.tomaxint((2,2,2)) < np.iinfo(np.int).max + >>> rs.tomaxint((2,2,2)) < np.iinfo(np.int_).max array([[[ True, True], [ True, True]], [[ True, True], @@ -537,6 +649,10 @@ cdef class RandomState: the specified dtype in the "half-open" interval [`low`, `high`). If `high` is None (the default), then results are from [0, `low`). + .. note:: + New code should use the ``integers`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- low : int or array-like of ints @@ -555,7 +671,7 @@ cdef class RandomState: Desired dtype of the result. All dtypes are determined by their name, i.e., 'int64', 'int', etc, so byteorder is not available and a specific precision may have different C types depending - on the platform. The default value is 'np.int'. + on the platform. The default value is `np.int_`. .. versionadded:: 1.11.0 @@ -567,9 +683,10 @@ cdef class RandomState: See Also -------- - random.random_integers : similar to `randint`, only for the closed + random_integers : similar to `randint`, only for the closed interval [`low`, `high`], and 1 is the lowest value if `high` is omitted. + Generator.integers: which should be used for new code. Examples -------- @@ -643,7 +760,7 @@ cdef class RandomState: elif key == 'bool': ret = _rand_bool(low, high, size, _masked, _endpoint, &self._bitgen, self.lock) - if size is None and dtype in (np.bool, np.int, np.long): + if size is None and dtype in (bool, int, np.compat.long): if np.array(ret).shape == (): return dtype(ret) return ret @@ -654,6 +771,10 @@ cdef class RandomState: Return random bytes. + .. note:: + New code should use the ``bytes`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- length : int @@ -664,11 +785,14 @@ cdef class RandomState: out : str String of length `length`. + See Also + -------- + Generator.bytes: which should be used for new code. + Examples -------- >>> np.random.bytes(10) ' eh\\x85\\x022SZ\\xbf\\xa4' #random - """ cdef Py_ssize_t n_uint32 = ((length - 1) // 4 + 1) # Interpret the uint32s as little-endian to convert them to bytes @@ -685,6 +809,10 @@ cdef class RandomState: .. versionadded:: 1.7.0 + .. note:: + New code should use the ``choice`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- a : 1-D array-like or int @@ -718,6 +846,7 @@ cdef class RandomState: See Also -------- randint, shuffle, permutation + Generator.choice: which should be used in new code Examples -------- @@ -877,6 +1006,10 @@ cdef class RandomState: any value within the given interval is equally likely to be drawn by `uniform`. + .. note:: + New code should use the ``uniform`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- low : float or array_like of floats, optional @@ -906,6 +1039,7 @@ cdef class RandomState: rand : Convenience function that accepts dimensions as input, e.g., ``rand(2,2)`` would generate a 2-by-2 array of floats, uniformly distributed over ``[0, 1)``. + Generator.uniform: which should be used for new code. Notes ----- @@ -985,7 +1119,7 @@ cdef class RandomState: .. note:: This is a convenience function for users porting code from Matlab, - and wraps `numpy.random.random_sample`. That function takes a + and wraps `random_sample`. That function takes a tuple to specify the size of the output, which is consistent with other NumPy functions like `numpy.zeros` and `numpy.ones`. @@ -1029,10 +1163,14 @@ cdef class RandomState: .. note:: This is a convenience function for users porting code from Matlab, - and wraps `numpy.random.standard_normal`. That function takes a + and wraps `standard_normal`. That function takes a tuple to specify the size of the output, which is consistent with other NumPy functions like `numpy.zeros` and `numpy.ones`. + .. note:: + New code should use the ``standard_normal`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + If positive int_like arguments are provided, `randn` generates an array of shape ``(d0, d1, ..., dn)``, filled with random floats sampled from a univariate "normal" (Gaussian) @@ -1056,6 +1194,7 @@ cdef class RandomState: -------- standard_normal : Similar, but takes a tuple as its argument. normal : Also accepts mu and sigma arguments. + Generator.standard_normal: which should be used for new code. Notes ----- @@ -1084,11 +1223,11 @@ cdef class RandomState: """ random_integers(low, high=None, size=None) - Random integers of type np.int between `low` and `high`, inclusive. + Random integers of type `np.int_` between `low` and `high`, inclusive. - Return random integers of type np.int from the "discrete uniform" + Return random integers of type `np.int_` from the "discrete uniform" distribution in the closed interval [`low`, `high`]. If `high` is - None (the default), then results are from [1, `low`]. The np.int + None (the default), then results are from [1, `low`]. The `np.int_` type translates to the C long integer type and its precision is platform dependent. @@ -1182,6 +1321,10 @@ cdef class RandomState: Draw samples from a standard Normal distribution (mean=0, stdev=1). + .. note:: + New code should use the ``standard_normal`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- size : int or tuple of ints, optional @@ -1195,6 +1338,13 @@ cdef class RandomState: A floating-point array of shape ``size`` of drawn samples, or a single sample if ``size`` was not specified. + See Also + -------- + normal : + Equivalent function with additional ``loc`` and ``scale`` arguments + for setting the mean and standard deviation. + Generator.standard_normal: which should be used for new code. + Notes ----- For random samples from :math:`N(\\mu, \\sigma^2)`, use one of:: @@ -1202,12 +1352,6 @@ cdef class RandomState: mu + sigma * np.random.standard_normal(size=...) np.random.normal(mu, sigma, size=...) - See Also - -------- - normal : - Equivalent function with additional ``loc`` and ``scale`` arguments - for setting the mean and standard deviation. - Examples -------- >>> np.random.standard_normal() @@ -1252,6 +1396,10 @@ cdef class RandomState: by a large number of tiny, random disturbances, each with its own unique distribution [2]_. + .. note:: + New code should use the ``normal`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- loc : float or array_like of floats @@ -1274,6 +1422,7 @@ cdef class RandomState: -------- scipy.stats.norm : probability density function, distribution or cumulative density function, etc. + Generator.normal: which should be used for new code. Notes ----- @@ -1289,8 +1438,8 @@ cdef class RandomState: The function has its peak at the mean, and its "spread" increases with the standard deviation (the function reaches 0.607 times its maximum at :math:`x + \\sigma` and :math:`x - \\sigma` [2]_). This implies that - `numpy.random.normal` is more likely to return samples lying close to - the mean, rather than those far away. + normal is more likely to return samples lying close to the mean, rather + than those far away. References ---------- @@ -1347,6 +1496,10 @@ cdef class RandomState: Samples are drawn from a Gamma distribution with specified parameters, shape (sometimes designated "k") and scale=1. + .. note:: + New code should use the ``standard_gamma`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- shape : float or array_like of floats @@ -1366,6 +1519,7 @@ cdef class RandomState: -------- scipy.stats.gamma : probability density function, distribution or cumulative density function, etc. + Generator.standard_gamma: which should be used for new code. Notes ----- @@ -1423,6 +1577,10 @@ cdef class RandomState: `shape` (sometimes designated "k") and `scale` (sometimes designated "theta"), where both parameters are > 0. + .. note:: + New code should use the ``gamma`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- shape : float or array_like of floats @@ -1445,6 +1603,7 @@ cdef class RandomState: -------- scipy.stats.gamma : probability density function, distribution or cumulative density function, etc. + Generator.gamma: which should be used for new code. Notes ----- @@ -1507,6 +1666,10 @@ cdef class RandomState: that arises in ANOVA tests, and is the ratio of two chi-square variates. + .. note:: + New code should use the ``f`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- dfnum : float or array_like of floats @@ -1528,6 +1691,7 @@ cdef class RandomState: -------- scipy.stats.f : probability density function, distribution or cumulative density function, etc. + Generator.f: which should be used for new code. Notes ----- @@ -1590,6 +1754,10 @@ cdef class RandomState: freedom in denominator), where both parameters > 1. `nonc` is the non-centrality parameter. + .. note:: + New code should use the ``noncentral_f`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- dfnum : float or array_like of floats @@ -1614,6 +1782,10 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized noncentral Fisher distribution. + See Also + -------- + Generator.noncentral_f: which should be used for new code. + Notes ----- When calculating the power of an experiment (power = probability of @@ -1667,6 +1839,10 @@ cdef class RandomState: resulting distribution is chi-square (see Notes). This distribution is often used in hypothesis testing. + .. note:: + New code should use the ``chisquare`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- df : float or array_like of floats @@ -1688,6 +1864,10 @@ cdef class RandomState: When `df` <= 0 or when an inappropriate `size` (e.g. ``size=-1``) is given. + See Also + -------- + Generator.chisquare: which should be used for new code. + Notes ----- The variable obtained by summing the squares of `df` independent, @@ -1717,7 +1897,6 @@ cdef class RandomState: -------- >>> np.random.chisquare(2,4) array([ 1.89920014, 9.00867716, 3.13710533, 5.62318272]) # random - """ return cont(&legacy_chisquare, &self._aug_state, size, self.lock, 1, df, 'df', CONS_POSITIVE, @@ -1733,6 +1912,10 @@ cdef class RandomState: The noncentral :math:`\\chi^2` distribution is a generalization of the :math:`\\chi^2` distribution. + .. note:: + New code should use the ``noncentral_chisquare`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- df : float or array_like of floats @@ -1753,6 +1936,10 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized noncentral chi-square distribution. + See Also + -------- + Generator.noncentral_chisquare: which should be used for new code. + Notes ----- The probability density function for the noncentral Chi-square @@ -1811,6 +1998,10 @@ cdef class RandomState: Also known as the Lorentz distribution. + .. note:: + New code should use the ``standard_cauchy`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- size : int or tuple of ints, optional @@ -1823,6 +2014,10 @@ cdef class RandomState: samples : ndarray or scalar The drawn samples. + See Also + -------- + Generator.standard_cauchy: which should be used for new code. + Notes ----- The probability density function for the full Cauchy distribution is @@ -1879,6 +2074,10 @@ cdef class RandomState: large, the result resembles that of the standard normal distribution (`standard_normal`). + .. note:: + New code should use the ``standard_t`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- df : float or array_like of floats @@ -1894,6 +2093,10 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized standard Student's t distribution. + See Also + -------- + Generator.standard_t: which should be used for new code. + Notes ----- The probability density function for the t distribution is @@ -1976,6 +2179,10 @@ cdef class RandomState: circle. It may be thought of as the circular analogue of the normal distribution. + .. note:: + New code should use the ``vonmises`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- mu : float or array_like of floats @@ -1997,6 +2204,7 @@ cdef class RandomState: -------- scipy.stats.vonmises : probability density function, distribution, or cumulative density function, etc. + Generator.vonmises: which should be used for new code. Notes ----- @@ -2069,6 +2277,10 @@ cdef class RandomState: 20 percent of the range, while the other 20 percent fill the remaining 80 percent of the range. + .. note:: + New code should use the ``pareto`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- a : float or array_like of floats @@ -2090,6 +2302,7 @@ cdef class RandomState: cumulative density function, etc. scipy.stats.genpareto : probability density function, distribution or cumulative density function, etc. + Generator.pareto: which should be used for new code. Notes ----- @@ -2158,6 +2371,10 @@ cdef class RandomState: The more common 2-parameter Weibull, including a scale parameter :math:`\\lambda` is just :math:`X = \\lambda(-ln(U))^{1/a}`. + .. note:: + New code should use the ``weibull`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- a : float or array_like of floats @@ -2179,6 +2396,7 @@ cdef class RandomState: scipy.stats.weibull_min scipy.stats.genextreme gumbel + Generator.weibull: which should be used for new code. Notes ----- @@ -2249,6 +2467,10 @@ cdef class RandomState: Also known as the power function distribution. + .. note:: + New code should use the ``power`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- a : float or array_like of floats @@ -2269,6 +2491,10 @@ cdef class RandomState: ValueError If a < 1. + See Also + -------- + Generator.power: which should be used for new code. + Notes ----- The probability density function is @@ -2352,6 +2578,10 @@ cdef class RandomState: difference between two independent, identically distributed exponential random variables. + .. note:: + New code should use the ``laplace`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- loc : float or array_like of floats, optional @@ -2370,6 +2600,10 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized Laplace distribution. + See Also + -------- + Generator.laplace: which should be used for new code. + Notes ----- It has the probability density function @@ -2435,6 +2669,10 @@ cdef class RandomState: scale. For more information on the Gumbel distribution, see Notes and References below. + .. note:: + New code should use the ``gumbel`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- loc : float or array_like of floats, optional @@ -2459,6 +2697,7 @@ cdef class RandomState: scipy.stats.gumbel_r scipy.stats.genextreme weibull + Generator.gumbel: which should be used for new code. Notes ----- @@ -2552,6 +2791,10 @@ cdef class RandomState: Samples are drawn from a logistic distribution with specified parameters, loc (location or mean, also median), and scale (>0). + .. note:: + New code should use the ``logistic`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- loc : float or array_like of floats, optional @@ -2574,6 +2817,7 @@ cdef class RandomState: -------- scipy.stats.logistic : probability density function, distribution or cumulative density function, etc. + Generator.logistic: which should be used for new code. Notes ----- @@ -2634,6 +2878,10 @@ cdef class RandomState: deviation are not the values for the distribution itself, but of the underlying normal distribution it is derived from. + .. note:: + New code should use the ``lognormal`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- mean : float or array_like of floats, optional @@ -2656,6 +2904,7 @@ cdef class RandomState: -------- scipy.stats.lognorm : probability density function, distribution, cumulative density function, etc. + Generator.lognormal: which should be used for new code. Notes ----- @@ -2742,6 +2991,10 @@ cdef class RandomState: The :math:`\\chi` and Weibull distributions are generalizations of the Rayleigh. + .. note:: + New code should use the ``rayleigh`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- scale : float or array_like of floats, optional @@ -2757,6 +3010,10 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized Rayleigh distribution. + See Also + -------- + Generator.rayleigh: which should be used for new code. + Notes ----- The probability density function for the Rayleigh distribution is @@ -2816,6 +3073,10 @@ cdef class RandomState: because there is an inverse relationship between the time to cover a unit distance and distance covered in unit time. + .. note:: + New code should use the ``wald`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- mean : float or array_like of floats @@ -2833,6 +3094,10 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized Wald distribution. + See Also + -------- + Generator.wald: which should be used for new code. + Notes ----- The probability density function for the Wald distribution is @@ -2881,6 +3146,10 @@ cdef class RandomState: limit right. Unlike the other distributions, these parameters directly define the shape of the pdf. + .. note:: + New code should use the ``triangular`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- left : float or array_like of floats @@ -2902,6 +3171,10 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized triangular distribution. + See Also + -------- + Generator.triangular: which should be used for new code. + Notes ----- The probability density function for the triangular distribution is @@ -2980,6 +3253,10 @@ cdef class RandomState: n an integer >= 0 and p is in the interval [0,1]. (n may be input as a float, but it is truncated to an integer in use) + .. note:: + New code should use the ``binomial`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- n : int or array_like of ints @@ -3003,6 +3280,7 @@ cdef class RandomState: -------- scipy.stats.binom : probability density function, distribution or cumulative density function, etc. + Generator.binomial: which should be used for new code. Notes ----- @@ -3125,6 +3403,10 @@ cdef class RandomState: parameters, `n` successes and `p` probability of success where `n` is > 0 and `p` is in the interval [0, 1]. + .. note:: + New code should use the ``negative_binomial`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- n : float or array_like of floats @@ -3144,6 +3426,10 @@ cdef class RandomState: where each sample is equal to N, the number of failures that occurred before a total of n successes was reached. + See Also + -------- + Generator.negative_binomial: which should be used for new code. + Notes ----- The probability mass function of the negative binomial distribution is @@ -3202,6 +3488,10 @@ cdef class RandomState: The Poisson distribution is the limit of the binomial distribution for large N. + .. note:: + New code should use the ``poisson`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- lam : float or array_like of floats @@ -3218,6 +3508,10 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized Poisson distribution. + See Also + -------- + Generator.poisson: which should be used for new code. + Notes ----- The Poisson distribution @@ -3280,6 +3574,10 @@ cdef class RandomState: frequency of an item is inversely proportional to its rank in a frequency table. + .. note:: + New code should use the ``zipf`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- a : float or array_like of floats @@ -3299,6 +3597,7 @@ cdef class RandomState: -------- scipy.stats.zipf : probability density function, distribution, or cumulative density function, etc. + Generator.zipf: which should be used for new code. Notes ----- @@ -3365,6 +3664,10 @@ cdef class RandomState: where `p` is the probability of success of an individual trial. + .. note:: + New code should use the ``geometric`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- p : float or array_like of floats @@ -3380,6 +3683,10 @@ cdef class RandomState: out : ndarray or scalar Drawn samples from the parameterized geometric distribution. + See Also + -------- + Generator.geometric: which should be used for new code. + Examples -------- Draw ten thousand values from the geometric distribution, @@ -3411,6 +3718,10 @@ cdef class RandomState: a bad selection), and `nsample` (number of items sampled, which is less than or equal to the sum ``ngood + nbad``). + .. note:: + New code should use the ``hypergeometric`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- ngood : int or array_like of ints @@ -3438,6 +3749,7 @@ cdef class RandomState: -------- scipy.stats.hypergeom : probability density function, distribution or cumulative density function, etc. + Generator.hypergeometric: which should be used for new code. Notes ----- @@ -3537,6 +3849,10 @@ cdef class RandomState: Samples are drawn from a log series distribution with specified shape parameter, 0 < ``p`` < 1. + .. note:: + New code should use the ``logseries`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- p : float or array_like of floats @@ -3556,6 +3872,7 @@ cdef class RandomState: -------- scipy.stats.logser : probability density function, distribution or cumulative density function, etc. + Generator.logseries: which should be used for new code. Notes ----- @@ -3625,6 +3942,10 @@ cdef class RandomState: (average or "center") and variance (standard deviation, or "width," squared) of the one-dimensional normal distribution. + .. note:: + New code should use the ``multivariate_normal`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- mean : 1-D array_like, of length N @@ -3652,6 +3973,10 @@ cdef class RandomState: In other words, each entry ``out[i,j,...,:]`` is an N-dimensional value drawn from the distribution. + See Also + -------- + Generator.multivariate_normal: which should be used for new code. + Notes ----- The mean is a coordinate in N-dimensional space, which represents the @@ -3791,6 +4116,10 @@ cdef class RandomState: ``X_i = [X_0, X_1, ..., X_p]``, represent the number of times the outcome was ``i``. + .. note:: + New code should use the ``multinomial`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- n : int @@ -3814,6 +4143,10 @@ cdef class RandomState: In other words, each entry ``out[i,j,...,:]`` is an N-dimensional value drawn from the distribution. + See Also + -------- + Generator.multinomial: which should be used for new code. + Examples -------- Throw a dice 20 times: @@ -3901,6 +4234,10 @@ cdef class RandomState: is a conjugate prior of a multinomial distribution in Bayesian inference. + .. note:: + New code should use the ``dirichlet`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- alpha : array @@ -3921,6 +4258,10 @@ cdef class RandomState: ValueError If any value in alpha is less than or equal to zero + See Also + -------- + Generator.dirichlet: which should be used for new code. + Notes ----- The Dirichlet distribution is a distribution over vectors @@ -4038,6 +4379,10 @@ cdef class RandomState: multi-dimensional array. The order of sub-arrays is changed but their contents remains the same. + .. note:: + New code should use the ``shuffle`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- x : array_like @@ -4047,6 +4392,10 @@ cdef class RandomState: ------- None + See Also + -------- + Generator.shuffle: which should be used for new code. + Examples -------- >>> arr = np.arange(10) @@ -4073,7 +4422,7 @@ cdef class RandomState: # Fast, statically typed path: shuffle the underlying buffer. # Only for non-empty, 1d objects of class ndarray (subclasses such # as MaskedArrays may not support this approach). - x_ptr = <char*><size_t>x.ctypes.data + x_ptr = <char*><size_t>np.PyArray_DATA(x) stride = x.strides[0] itemsize = x.dtype.itemsize # As the array x could contain python objects we use a buffer @@ -4081,7 +4430,7 @@ cdef class RandomState: # within the buffer and erroneously decrementing it's refcount # when the function exits. buf = np.empty(itemsize, dtype=np.int8) # GC'd at function exit - buf_ptr = <char*><size_t>buf.ctypes.data + buf_ptr = <char*><size_t>np.PyArray_DATA(buf) with self.lock: # We trick gcc into providing a specialized implementation for # the most common case, yielding a ~33% performance improvement. @@ -4125,6 +4474,10 @@ cdef class RandomState: If `x` is a multi-dimensional array, it is only shuffled along its first index. + .. note:: + New code should use the ``permutation`` method of a ``default_rng()`` + instance instead; see `random-quick-start`. + Parameters ---------- x : int or array_like @@ -4137,6 +4490,9 @@ cdef class RandomState: out : ndarray Permuted sequence or array range. + See Also + -------- + Generator.permutation: which should be used for new code. Examples -------- diff --git a/numpy/random/setup.py b/numpy/random/setup.py index 52e020e58..1b093d6d3 100644 --- a/numpy/random/setup.py +++ b/numpy/random/setup.py @@ -34,6 +34,7 @@ def configuration(parent_package='', top_path=None): defs.append(('NPY_NO_DEPRECATED_API', 0)) config.add_data_dir('tests') + config.add_data_dir('_examples') EXTRA_LINK_ARGS = [] # Math lib @@ -56,32 +57,32 @@ def configuration(parent_package='', top_path=None): for gen in ['mt19937']: # gen.pyx, src/gen/gen.c, src/gen/gen-jump.c - config.add_extension(gen, - sources=['{0}.c'.format(gen), + config.add_extension('_{0}'.format(gen), + sources=['_{0}.c'.format(gen), 'src/{0}/{0}.c'.format(gen), 'src/{0}/{0}-jump.c'.format(gen)], include_dirs=['.', 'src', join('src', gen)], libraries=EXTRA_LIBRARIES, extra_compile_args=EXTRA_COMPILE_ARGS, extra_link_args=EXTRA_LINK_ARGS, - depends=['%s.pyx' % gen], + depends=['_%s.pyx' % gen], define_macros=defs, ) for gen in ['philox', 'pcg64', 'sfc64']: # gen.pyx, src/gen/gen.c _defs = defs + PCG64_DEFS if gen == 'pcg64' else defs - config.add_extension(gen, - sources=['{0}.c'.format(gen), + config.add_extension('_{0}'.format(gen), + sources=['_{0}.c'.format(gen), 'src/{0}/{0}.c'.format(gen)], include_dirs=['.', 'src', join('src', gen)], libraries=EXTRA_LIBRARIES, extra_compile_args=EXTRA_COMPILE_ARGS, extra_link_args=EXTRA_LINK_ARGS, - depends=['%s.pyx' % gen, 'bit_generator.pyx', - 'bit_generator.pxd'], + depends=['_%s.pyx' % gen, '_bit_generator.pyx', + '_bit_generator.pxd'], define_macros=_defs, ) - for gen in ['common', 'bit_generator']: + for gen in ['_common', '_bit_generator']: # gen.pyx config.add_extension(gen, sources=['{0}.c'.format(gen)], @@ -92,12 +93,15 @@ def configuration(parent_package='', top_path=None): depends=['%s.pyx' % gen, '%s.pxd' % gen,], define_macros=defs, ) + config.add_data_files('{0}.pxd'.format(gen)) other_srcs = [ 'src/distributions/logfactorial.c', 'src/distributions/distributions.c', + 'src/distributions/random_mvhg_count.c', + 'src/distributions/random_mvhg_marginals.c', 'src/distributions/random_hypergeometric.c', ] - for gen in ['generator', 'bounded_integers']: + for gen in ['_generator', '_bounded_integers']: # gen.pyx, src/distributions/distributions.c config.add_extension(gen, sources=['{0}.c'.format(gen)] + other_srcs, @@ -108,8 +112,8 @@ def configuration(parent_package='', top_path=None): depends=['%s.pyx' % gen], define_macros=defs, ) + config.add_data_files('_bounded_integers.pxd') config.add_extension('mtrand', - # mtrand does not depend on random_hypergeometric.c. sources=['mtrand.c', 'src/legacy/legacy-distributions.c', 'src/distributions/logfactorial.c', @@ -121,6 +125,7 @@ def configuration(parent_package='', top_path=None): depends=['mtrand.pyx'], define_macros=defs + LEGACY_DEFS, ) + config.add_data_files('__init__.pxd') return config diff --git a/numpy/random/src/aligned_malloc/aligned_malloc.c b/numpy/random/src/aligned_malloc/aligned_malloc.c deleted file mode 100644 index 6e8192cfb..000000000 --- a/numpy/random/src/aligned_malloc/aligned_malloc.c +++ /dev/null @@ -1,9 +0,0 @@ -#include "aligned_malloc.h" - -static NPY_INLINE void *PyArray_realloc_aligned(void *p, size_t n); - -static NPY_INLINE void *PyArray_malloc_aligned(size_t n); - -static NPY_INLINE void *PyArray_calloc_aligned(size_t n, size_t s); - -static NPY_INLINE void PyArray_free_aligned(void *p);
\ No newline at end of file diff --git a/numpy/random/src/distributions/distributions.c b/numpy/random/src/distributions/distributions.c index 45f062c06..0b46dc6d8 100644 --- a/numpy/random/src/distributions/distributions.c +++ b/numpy/random/src/distributions/distributions.c @@ -1,4 +1,4 @@ -#include "distributions.h" +#include "numpy/random/distributions.h" #include "ziggurat_constants.h" #include "logfactorial.h" @@ -6,92 +6,42 @@ #include <intrin.h> #endif -/* Random generators for external use */ -float random_float(bitgen_t *bitgen_state) { return next_float(bitgen_state); } - -double random_double(bitgen_t *bitgen_state) { - return next_double(bitgen_state); +/* Inline generators for internal use */ +static NPY_INLINE uint32_t next_uint32(bitgen_t *bitgen_state) { + return bitgen_state->next_uint32(bitgen_state->state); } - -static NPY_INLINE double next_standard_exponential(bitgen_t *bitgen_state) { - return -log(1.0 - next_double(bitgen_state)); +static NPY_INLINE uint64_t next_uint64(bitgen_t *bitgen_state) { + return bitgen_state->next_uint64(bitgen_state->state); } -double random_standard_exponential(bitgen_t *bitgen_state) { - return next_standard_exponential(bitgen_state); +static NPY_INLINE float next_float(bitgen_t *bitgen_state) { + return (next_uint32(bitgen_state) >> 9) * (1.0f / 8388608.0f); } -void random_standard_exponential_fill(bitgen_t *bitgen_state, npy_intp cnt, - double *out) { - npy_intp i; - for (i = 0; i < cnt; i++) { - out[i] = next_standard_exponential(bitgen_state); - } +/* Random generators for external use */ +float random_standard_uniform_f(bitgen_t *bitgen_state) { + return next_float(bitgen_state); } -float random_standard_exponential_f(bitgen_t *bitgen_state) { - return -logf(1.0f - next_float(bitgen_state)); +double random_standard_uniform(bitgen_t *bitgen_state) { + return next_double(bitgen_state); } -void random_double_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) { +void random_standard_uniform_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) { npy_intp i; for (i = 0; i < cnt; i++) { out[i] = next_double(bitgen_state); } } -#if 0 -double random_gauss(bitgen_t *bitgen_state) { - if (bitgen_state->has_gauss) { - const double temp = bitgen_state->gauss; - bitgen_state->has_gauss = false; - bitgen_state->gauss = 0.0; - return temp; - } else { - double f, x1, x2, r2; - - do { - x1 = 2.0 * next_double(bitgen_state) - 1.0; - x2 = 2.0 * next_double(bitgen_state) - 1.0; - r2 = x1 * x1 + x2 * x2; - } while (r2 >= 1.0 || r2 == 0.0); - - /* Polar method, a more efficient version of the Box-Muller approach. */ - f = sqrt(-2.0 * log(r2) / r2); - /* Keep for next call */ - bitgen_state->gauss = f * x1; - bitgen_state->has_gauss = true; - return f * x2; - } -} -float random_gauss_f(bitgen_t *bitgen_state) { - if (bitgen_state->has_gauss_f) { - const float temp = bitgen_state->gauss_f; - bitgen_state->has_gauss_f = false; - bitgen_state->gauss_f = 0.0f; - return temp; - } else { - float f, x1, x2, r2; - - do { - x1 = 2.0f * next_float(bitgen_state) - 1.0f; - x2 = 2.0f * next_float(bitgen_state) - 1.0f; - r2 = x1 * x1 + x2 * x2; - } while (r2 >= 1.0 || r2 == 0.0); - - /* Polar method, a more efficient version of the Box-Muller approach. */ - f = sqrtf(-2.0f * logf(r2) / r2); - /* Keep for next call */ - bitgen_state->gauss_f = f * x1; - bitgen_state->has_gauss_f = true; - return f * x2; +void random_standard_uniform_fill_f(bitgen_t *bitgen_state, npy_intp cnt, float *out) { + npy_intp i; + for (i = 0; i < cnt; i++) { + out[i] = next_float(bitgen_state); } } -#endif - -static NPY_INLINE double standard_exponential_zig(bitgen_t *bitgen_state); -static double standard_exponential_zig_unlikely(bitgen_t *bitgen_state, +static double standard_exponential_unlikely(bitgen_t *bitgen_state, uint8_t idx, double x) { if (idx == 0) { /* Switch to 1.0 - U to avoid log(0.0), see GH 13361 */ @@ -101,11 +51,11 @@ static double standard_exponential_zig_unlikely(bitgen_t *bitgen_state, exp(-x)) { return x; } else { - return standard_exponential_zig(bitgen_state); + return random_standard_exponential(bitgen_state); } } -static NPY_INLINE double standard_exponential_zig(bitgen_t *bitgen_state) { +double random_standard_exponential(bitgen_t *bitgen_state) { uint64_t ri; uint8_t idx; double x; @@ -117,24 +67,18 @@ static NPY_INLINE double standard_exponential_zig(bitgen_t *bitgen_state) { if (ri < ke_double[idx]) { return x; /* 98.9% of the time we return here 1st try */ } - return standard_exponential_zig_unlikely(bitgen_state, idx, x); + return standard_exponential_unlikely(bitgen_state, idx, x); } -double random_standard_exponential_zig(bitgen_t *bitgen_state) { - return standard_exponential_zig(bitgen_state); -} - -void random_standard_exponential_zig_fill(bitgen_t *bitgen_state, npy_intp cnt, - double *out) { +void random_standard_exponential_fill(bitgen_t * bitgen_state, npy_intp cnt, double * out) +{ npy_intp i; for (i = 0; i < cnt; i++) { - out[i] = standard_exponential_zig(bitgen_state); + out[i] = random_standard_exponential(bitgen_state); } } -static NPY_INLINE float standard_exponential_zig_f(bitgen_t *bitgen_state); - -static float standard_exponential_zig_unlikely_f(bitgen_t *bitgen_state, +static float standard_exponential_unlikely_f(bitgen_t *bitgen_state, uint8_t idx, float x) { if (idx == 0) { /* Switch to 1.0 - U to avoid log(0.0), see GH 13361 */ @@ -144,11 +88,11 @@ static float standard_exponential_zig_unlikely_f(bitgen_t *bitgen_state, expf(-x)) { return x; } else { - return standard_exponential_zig_f(bitgen_state); + return random_standard_exponential_f(bitgen_state); } } -static NPY_INLINE float standard_exponential_zig_f(bitgen_t *bitgen_state) { +float random_standard_exponential_f(bitgen_t *bitgen_state) { uint32_t ri; uint8_t idx; float x; @@ -160,14 +104,35 @@ static NPY_INLINE float standard_exponential_zig_f(bitgen_t *bitgen_state) { if (ri < ke_float[idx]) { return x; /* 98.9% of the time we return here 1st try */ } - return standard_exponential_zig_unlikely_f(bitgen_state, idx, x); + return standard_exponential_unlikely_f(bitgen_state, idx, x); } -float random_standard_exponential_zig_f(bitgen_t *bitgen_state) { - return standard_exponential_zig_f(bitgen_state); +void random_standard_exponential_fill_f(bitgen_t * bitgen_state, npy_intp cnt, float * out) +{ + npy_intp i; + for (i = 0; i < cnt; i++) { + out[i] = random_standard_exponential_f(bitgen_state); + } } -static NPY_INLINE double next_gauss_zig(bitgen_t *bitgen_state) { +void random_standard_exponential_inv_fill(bitgen_t * bitgen_state, npy_intp cnt, double * out) +{ + npy_intp i; + for (i = 0; i < cnt; i++) { + out[i] = -log(1.0 - next_double(bitgen_state)); + } +} + +void random_standard_exponential_inv_fill_f(bitgen_t * bitgen_state, npy_intp cnt, float * out) +{ + npy_intp i; + for (i = 0; i < cnt; i++) { + out[i] = -log(1.0 - next_float(bitgen_state)); + } +} + + +double random_standard_normal(bitgen_t *bitgen_state) { uint64_t r; int sign; uint64_t rabs; @@ -202,18 +167,14 @@ static NPY_INLINE double next_gauss_zig(bitgen_t *bitgen_state) { } } -double random_gauss_zig(bitgen_t *bitgen_state) { - return next_gauss_zig(bitgen_state); -} - -void random_gauss_zig_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) { +void random_standard_normal_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) { npy_intp i; for (i = 0; i < cnt; i++) { - out[i] = next_gauss_zig(bitgen_state); + out[i] = random_standard_normal(bitgen_state); } } -float random_gauss_zig_f(bitgen_t *bitgen_state) { +float random_standard_normal_f(bitgen_t *bitgen_state) { uint32_t r; int sign; uint32_t rabs; @@ -247,113 +208,26 @@ float random_gauss_zig_f(bitgen_t *bitgen_state) { } } -/* -static NPY_INLINE double standard_gamma(bitgen_t *bitgen_state, double shape) { - double b, c; - double U, V, X, Y; - - if (shape == 1.0) { - return random_standard_exponential(bitgen_state); - } else if (shape < 1.0) { - for (;;) { - U = next_double(bitgen_state); - V = random_standard_exponential(bitgen_state); - if (U <= 1.0 - shape) { - X = pow(U, 1. / shape); - if (X <= V) { - return X; - } - } else { - Y = -log((1 - U) / shape); - X = pow(1.0 - shape + shape * Y, 1. / shape); - if (X <= (V + Y)) { - return X; - } - } - } - } else { - b = shape - 1. / 3.; - c = 1. / sqrt(9 * b); - for (;;) { - do { - X = random_gauss(bitgen_state); - V = 1.0 + c * X; - } while (V <= 0.0); - - V = V * V * V; - U = next_double(bitgen_state); - if (U < 1.0 - 0.0331 * (X * X) * (X * X)) - return (b * V); - if (log(U) < 0.5 * X * X + b * (1. - V + log(V))) - return (b * V); - } - } -} - -static NPY_INLINE float standard_gamma_float(bitgen_t *bitgen_state, float -shape) { float b, c; float U, V, X, Y; - - if (shape == 1.0f) { - return random_standard_exponential_f(bitgen_state); - } else if (shape < 1.0f) { - for (;;) { - U = next_float(bitgen_state); - V = random_standard_exponential_f(bitgen_state); - if (U <= 1.0f - shape) { - X = powf(U, 1.0f / shape); - if (X <= V) { - return X; - } - } else { - Y = -logf((1.0f - U) / shape); - X = powf(1.0f - shape + shape * Y, 1.0f / shape); - if (X <= (V + Y)) { - return X; - } - } - } - } else { - b = shape - 1.0f / 3.0f; - c = 1.0f / sqrtf(9.0f * b); - for (;;) { - do { - X = random_gauss_f(bitgen_state); - V = 1.0f + c * X; - } while (V <= 0.0f); - - V = V * V * V; - U = next_float(bitgen_state); - if (U < 1.0f - 0.0331f * (X * X) * (X * X)) - return (b * V); - if (logf(U) < 0.5f * X * X + b * (1.0f - V + logf(V))) - return (b * V); - } +void random_standard_normal_fill_f(bitgen_t *bitgen_state, npy_intp cnt, float *out) { + npy_intp i; + for (i = 0; i < cnt; i++) { + out[i] = random_standard_normal_f(bitgen_state); } } - -double random_standard_gamma(bitgen_t *bitgen_state, double shape) { - return standard_gamma(bitgen_state, shape); -} - -float random_standard_gamma_f(bitgen_t *bitgen_state, float shape) { - return standard_gamma_float(bitgen_state, shape); -} -*/ - -static NPY_INLINE double standard_gamma_zig(bitgen_t *bitgen_state, +double random_standard_gamma(bitgen_t *bitgen_state, double shape) { double b, c; double U, V, X, Y; if (shape == 1.0) { - return random_standard_exponential_zig(bitgen_state); + return random_standard_exponential(bitgen_state); } else if (shape == 0.0) { return 0.0; } else if (shape < 1.0) { for (;;) { U = next_double(bitgen_state); - V = random_standard_exponential_zig(bitgen_state); + V = random_standard_exponential(bitgen_state); if (U <= 1.0 - shape) { X = pow(U, 1. / shape); if (X <= V) { @@ -372,7 +246,7 @@ static NPY_INLINE double standard_gamma_zig(bitgen_t *bitgen_state, c = 1. / sqrt(9 * b); for (;;) { do { - X = random_gauss_zig(bitgen_state); + X = random_standard_normal(bitgen_state); V = 1.0 + c * X; } while (V <= 0.0); @@ -387,19 +261,19 @@ static NPY_INLINE double standard_gamma_zig(bitgen_t *bitgen_state, } } -static NPY_INLINE float standard_gamma_zig_f(bitgen_t *bitgen_state, +float random_standard_gamma_f(bitgen_t *bitgen_state, float shape) { float b, c; float U, V, X, Y; if (shape == 1.0f) { - return random_standard_exponential_zig_f(bitgen_state); + return random_standard_exponential_f(bitgen_state); } else if (shape == 0.0) { return 0.0; } else if (shape < 1.0f) { for (;;) { U = next_float(bitgen_state); - V = random_standard_exponential_zig_f(bitgen_state); + V = random_standard_exponential_f(bitgen_state); if (U <= 1.0f - shape) { X = powf(U, 1.0f / shape); if (X <= V) { @@ -418,7 +292,7 @@ static NPY_INLINE float standard_gamma_zig_f(bitgen_t *bitgen_state, c = 1.0f / sqrtf(9.0f * b); for (;;) { do { - X = random_gauss_zig_f(bitgen_state); + X = random_standard_normal_f(bitgen_state); V = 1.0f + c * X; } while (V <= 0.0f); @@ -433,14 +307,6 @@ static NPY_INLINE float standard_gamma_zig_f(bitgen_t *bitgen_state, } } -double random_standard_gamma_zig(bitgen_t *bitgen_state, double shape) { - return standard_gamma_zig(bitgen_state, shape); -} - -float random_standard_gamma_zig_f(bitgen_t *bitgen_state, float shape) { - return standard_gamma_zig_f(bitgen_state, shape); -} - int64_t random_positive_int64(bitgen_t *bitgen_state) { return next_uint64(bitgen_state) >> 1; } @@ -470,10 +336,10 @@ uint64_t random_uint(bitgen_t *bitgen_state) { * algorithm comes from SPECFUN by Shanjie Zhang and Jianming Jin and their * book "Computation of Special Functions", 1996, John Wiley & Sons, Inc. * - * If loggam(k+1) is being used to compute log(k!) for an integer k, consider + * If random_loggam(k+1) is being used to compute log(k!) for an integer k, consider * using logfactorial(k) instead. */ -double loggam(double x) { +double random_loggam(double x) { double x0, x2, xp, gl, gl0; RAND_INT_TYPE k, n; @@ -513,12 +379,12 @@ double random_normal(bitgen_t *bitgen_state, double loc, double scale) { } */ -double random_normal_zig(bitgen_t *bitgen_state, double loc, double scale) { - return loc + scale * random_gauss_zig(bitgen_state); +double random_normal(bitgen_t *bitgen_state, double loc, double scale) { + return loc + scale * random_standard_normal(bitgen_state); } double random_exponential(bitgen_t *bitgen_state, double scale) { - return scale * standard_exponential_zig(bitgen_state); + return scale * random_standard_exponential(bitgen_state); } double random_uniform(bitgen_t *bitgen_state, double lower, double range) { @@ -526,11 +392,11 @@ double random_uniform(bitgen_t *bitgen_state, double lower, double range) { } double random_gamma(bitgen_t *bitgen_state, double shape, double scale) { - return scale * random_standard_gamma_zig(bitgen_state, shape); + return scale * random_standard_gamma(bitgen_state, shape); } -float random_gamma_float(bitgen_t *bitgen_state, float shape, float scale) { - return scale * random_standard_gamma_zig_f(bitgen_state, shape); +float random_gamma_f(bitgen_t *bitgen_state, float shape, float scale) { + return scale * random_standard_gamma_f(bitgen_state, shape); } double random_beta(bitgen_t *bitgen_state, double a, double b) { @@ -562,14 +428,14 @@ double random_beta(bitgen_t *bitgen_state, double a, double b) { } } } else { - Ga = random_standard_gamma_zig(bitgen_state, a); - Gb = random_standard_gamma_zig(bitgen_state, b); + Ga = random_standard_gamma(bitgen_state, a); + Gb = random_standard_gamma(bitgen_state, b); return Ga / (Ga + Gb); } } double random_chisquare(bitgen_t *bitgen_state, double df) { - return 2.0 * random_standard_gamma_zig(bitgen_state, df / 2.0); + return 2.0 * random_standard_gamma(bitgen_state, df / 2.0); } double random_f(bitgen_t *bitgen_state, double dfnum, double dfden) { @@ -578,22 +444,22 @@ double random_f(bitgen_t *bitgen_state, double dfnum, double dfden) { } double random_standard_cauchy(bitgen_t *bitgen_state) { - return random_gauss_zig(bitgen_state) / random_gauss_zig(bitgen_state); + return random_standard_normal(bitgen_state) / random_standard_normal(bitgen_state); } double random_pareto(bitgen_t *bitgen_state, double a) { - return exp(standard_exponential_zig(bitgen_state) / a) - 1; + return exp(random_standard_exponential(bitgen_state) / a) - 1; } double random_weibull(bitgen_t *bitgen_state, double a) { if (a == 0.0) { return 0.0; } - return pow(standard_exponential_zig(bitgen_state), 1. / a); + return pow(random_standard_exponential(bitgen_state), 1. / a); } double random_power(bitgen_t *bitgen_state, double a) { - return pow(1 - exp(-standard_exponential_zig(bitgen_state)), 1. / a); + return pow(1 - exp(-random_standard_exponential(bitgen_state)), 1. / a); } double random_laplace(bitgen_t *bitgen_state, double loc, double scale) { @@ -634,7 +500,7 @@ double random_logistic(bitgen_t *bitgen_state, double loc, double scale) { } double random_lognormal(bitgen_t *bitgen_state, double mean, double sigma) { - return exp(random_normal_zig(bitgen_state, mean, sigma)); + return exp(random_normal(bitgen_state, mean, sigma)); } double random_rayleigh(bitgen_t *bitgen_state, double mode) { @@ -644,8 +510,8 @@ double random_rayleigh(bitgen_t *bitgen_state, double mode) { double random_standard_t(bitgen_t *bitgen_state, double df) { double num, denom; - num = random_gauss_zig(bitgen_state); - denom = random_standard_gamma_zig(bitgen_state, df / 2); + num = random_standard_normal(bitgen_state); + denom = random_standard_gamma(bitgen_state, df / 2); return sqrt(df / 2) * num / sqrt(denom); } @@ -699,7 +565,7 @@ static RAND_INT_TYPE random_poisson_ptrs(bitgen_t *bitgen_state, double lam) { /* log(V) == log(0.0) ok here */ /* if U==0.0 so that us==0.0, log is ok since always returns */ if ((log(V) + log(invalpha) - log(a / (us * us) + b)) <= - (-lam + k * loglam - loggam(k + 1))) { + (-lam + k * loglam - random_loggam(k + 1))) { return k; } } @@ -934,7 +800,7 @@ double random_noncentral_chisquare(bitgen_t *bitgen_state, double df, } if (1 < df) { const double Chi2 = random_chisquare(bitgen_state, df - 1); - const double n = random_gauss_zig(bitgen_state) + sqrt(nonc); + const double n = random_standard_normal(bitgen_state) + sqrt(nonc); return Chi2 + n * n; } else { const RAND_INT_TYPE i = random_poisson(bitgen_state, nonc / 2.0); @@ -953,7 +819,7 @@ double random_wald(bitgen_t *bitgen_state, double mean, double scale) { double mu_2l; mu_2l = mean / (2 * scale); - Y = random_gauss_zig(bitgen_state); + Y = random_standard_normal(bitgen_state); Y = mean * Y * Y; X = mean + mu_2l * (Y - sqrt(4 * scale * Y + Y * Y)); U = next_double(bitgen_state); @@ -1092,8 +958,8 @@ RAND_INT_TYPE random_zipf(bitgen_t *bitgen_state, double a) { while (1) { double T, U, V, X; - U = 1.0 - random_double(bitgen_state); - V = random_double(bitgen_state); + U = 1.0 - next_double(bitgen_state); + V = next_double(bitgen_state); X = floor(pow(U, -1.0 / am1)); /* * The real result may be above what can be represented in a signed diff --git a/numpy/random/src/distributions/random_hypergeometric.c b/numpy/random/src/distributions/random_hypergeometric.c index 59a3a4b9b..0da49bd62 100644 --- a/numpy/random/src/distributions/random_hypergeometric.c +++ b/numpy/random/src/distributions/random_hypergeometric.c @@ -1,6 +1,6 @@ -#include <stdint.h> -#include "distributions.h" +#include "numpy/random/distributions.h" #include "logfactorial.h" +#include <stdint.h> /* * Generate a sample from the hypergeometric distribution. @@ -188,8 +188,8 @@ static int64_t hypergeometric_hrua(bitgen_t *bitgen_state, while (1) { double U, V, X, T; double gp; - U = random_double(bitgen_state); - V = random_double(bitgen_state); // "U star" in Stadlober (1989) + U = next_double(bitgen_state); + V = next_double(bitgen_state); // "U star" in Stadlober (1989) X = a + h*(V - 0.5) / U; // fast rejection: diff --git a/numpy/random/src/distributions/random_mvhg_count.c b/numpy/random/src/distributions/random_mvhg_count.c new file mode 100644 index 000000000..7cbed1f9e --- /dev/null +++ b/numpy/random/src/distributions/random_mvhg_count.c @@ -0,0 +1,131 @@ +#include <stdint.h> +#include <stdlib.h> +#include <stdbool.h> + +#include "numpy/random/distributions.h" + +/* + * random_multivariate_hypergeometric_count + * + * Draw variates from the multivariate hypergeometric distribution-- + * the "count" algorithm. + * + * Parameters + * ---------- + * bitgen_t *bitgen_state + * Pointer to a `bitgen_t` instance. + * int64_t total + * The sum of the values in the array `colors`. (This is redundant + * information, but we know the caller has already computed it, so + * we might as well use it.) + * size_t num_colors + * The length of the `colors` array. + * int64_t *colors + * The array of colors (i.e. the number of each type in the collection + * from which the random variate is drawn). + * int64_t nsample + * The number of objects drawn without replacement for each variate. + * `nsample` must not exceed sum(colors). This condition is not checked; + * it is assumed that the caller has already validated the value. + * size_t num_variates + * The number of variates to be produced and put in the array + * pointed to by `variates`. One variate is a vector of length + * `num_colors`, so the array pointed to by `variates` must have length + * `num_variates * num_colors`. + * int64_t *variates + * The array that will hold the result. It must have length + * `num_variates * num_colors`. + * The array is not initialized in the function; it is expected that the + * array has been initialized with zeros when the function is called. + * + * Notes + * ----- + * The "count" algorithm for drawing one variate is roughly equivalent to the + * following numpy code: + * + * choices = np.repeat(np.arange(len(colors)), colors) + * selection = np.random.choice(choices, nsample, replace=False) + * variate = np.bincount(selection, minlength=len(colors)) + * + * This function uses a temporary array with length sum(colors). + * + * Assumptions on the arguments (not checked in the function): + * * colors[k] >= 0 for k in range(num_colors) + * * total = sum(colors) + * * 0 <= nsample <= total + * * the product total * sizeof(size_t) does not exceed SIZE_MAX + * * the product num_variates * num_colors does not overflow + */ + +int random_multivariate_hypergeometric_count(bitgen_t *bitgen_state, + int64_t total, + size_t num_colors, int64_t *colors, + int64_t nsample, + size_t num_variates, int64_t *variates) +{ + size_t *choices; + bool more_than_half; + + if ((total == 0) || (nsample == 0) || (num_variates == 0)) { + // Nothing to do. + return 0; + } + + choices = malloc(total * (sizeof *choices)); + if (choices == NULL) { + return -1; + } + + /* + * If colors contains, for example, [3 2 5], then choices + * will contain [0 0 0 1 1 2 2 2 2 2]. + */ + for (size_t i = 0, k = 0; i < num_colors; ++i) { + for (int64_t j = 0; j < colors[i]; ++j) { + choices[k] = i; + ++k; + } + } + + more_than_half = nsample > (total / 2); + if (more_than_half) { + nsample = total - nsample; + } + + for (size_t i = 0; i < num_variates * num_colors; i += num_colors) { + /* + * Fisher-Yates shuffle, but only loop through the first + * `nsample` entries of `choices`. After the loop, + * choices[:nsample] contains a random sample from the + * the full array. + */ + for (size_t j = 0; j < (size_t) nsample; ++j) { + size_t tmp, k; + // Note: nsample is not greater than total, so there is no danger + // of integer underflow in `(size_t) total - j - 1`. + k = j + (size_t) random_interval(bitgen_state, + (size_t) total - j - 1); + tmp = choices[k]; + choices[k] = choices[j]; + choices[j] = tmp; + } + /* + * Count the number of occurrences of each value in choices[:nsample]. + * The result, stored in sample[i:i+num_colors], is the sample from + * the multivariate hypergeometric distribution. + */ + for (size_t j = 0; j < (size_t) nsample; ++j) { + variates[i + choices[j]] += 1; + } + + if (more_than_half) { + for (size_t k = 0; k < num_colors; ++k) { + variates[i + k] = colors[k] - variates[i + k]; + } + } + } + + free(choices); + + return 0; +} diff --git a/numpy/random/src/distributions/random_mvhg_marginals.c b/numpy/random/src/distributions/random_mvhg_marginals.c new file mode 100644 index 000000000..809d129de --- /dev/null +++ b/numpy/random/src/distributions/random_mvhg_marginals.c @@ -0,0 +1,138 @@ +#include <stdint.h> +#include <stddef.h> +#include <stdbool.h> +#include <math.h> + +#include "numpy/random/distributions.h" +#include "logfactorial.h" + + +/* + * random_multivariate_hypergeometric_marginals + * + * Draw samples from the multivariate hypergeometric distribution-- + * the "marginals" algorithm. + * + * This version generates the sample by iteratively calling + * hypergeometric() (the univariate hypergeometric distribution). + * + * Parameters + * ---------- + * bitgen_t *bitgen_state + * Pointer to a `bitgen_t` instance. + * int64_t total + * The sum of the values in the array `colors`. (This is redundant + * information, but we know the caller has already computed it, so + * we might as well use it.) + * size_t num_colors + * The length of the `colors` array. The functions assumes + * num_colors > 0. + * int64_t *colors + * The array of colors (i.e. the number of each type in the collection + * from which the random variate is drawn). + * int64_t nsample + * The number of objects drawn without replacement for each variate. + * `nsample` must not exceed sum(colors). This condition is not checked; + * it is assumed that the caller has already validated the value. + * size_t num_variates + * The number of variates to be produced and put in the array + * pointed to by `variates`. One variate is a vector of length + * `num_colors`, so the array pointed to by `variates` must have length + * `num_variates * num_colors`. + * int64_t *variates + * The array that will hold the result. It must have length + * `num_variates * num_colors`. + * The array is not initialized in the function; it is expected that the + * array has been initialized with zeros when the function is called. + * + * Notes + * ----- + * Here's an example that demonstrates the idea of this algorithm. + * + * Suppose the urn contains red, green, blue and yellow marbles. + * Let nred be the number of red marbles, and define the quantities for + * the other colors similarly. The total number of marbles is + * + * total = nred + ngreen + nblue + nyellow. + * + * To generate a sample using rk_hypergeometric: + * + * red_sample = hypergeometric(ngood=nred, nbad=total - nred, + * nsample=nsample) + * + * This gives us the number of red marbles in the sample. The number of + * marbles in the sample that are *not* red is nsample - red_sample. + * To figure out the distribution of those marbles, we again use + * rk_hypergeometric: + * + * green_sample = hypergeometric(ngood=ngreen, + * nbad=total - nred - ngreen, + * nsample=nsample - red_sample) + * + * Similarly, + * + * blue_sample = hypergeometric( + * ngood=nblue, + * nbad=total - nred - ngreen - nblue, + * nsample=nsample - red_sample - green_sample) + * + * Finally, + * + * yellow_sample = total - (red_sample + green_sample + blue_sample). + * + * The above sequence of steps is implemented as a loop for an arbitrary + * number of colors in the innermost loop in the code below. `remaining` + * is the value passed to `nbad`; it is `total - colors[0]` in the first + * call to random_hypergeometric(), and then decreases by `colors[j]` in + * each iteration. `num_to_sample` is the `nsample` argument. It + * starts at this function's `nsample` input, and is decreased by the + * result of the call to random_hypergeometric() in each iteration. + * + * Assumptions on the arguments (not checked in the function): + * * colors[k] >= 0 for k in range(num_colors) + * * total = sum(colors) + * * 0 <= nsample <= total + * * the product num_variates * num_colors does not overflow + */ + +void random_multivariate_hypergeometric_marginals(bitgen_t *bitgen_state, + int64_t total, + size_t num_colors, int64_t *colors, + int64_t nsample, + size_t num_variates, int64_t *variates) +{ + bool more_than_half; + + if ((total == 0) || (nsample == 0) || (num_variates == 0)) { + // Nothing to do. + return; + } + + more_than_half = nsample > (total / 2); + if (more_than_half) { + nsample = total - nsample; + } + + for (size_t i = 0; i < num_variates * num_colors; i += num_colors) { + int64_t num_to_sample = nsample; + int64_t remaining = total; + for (size_t j = 0; (num_to_sample > 0) && (j + 1 < num_colors); ++j) { + int64_t r; + remaining -= colors[j]; + r = random_hypergeometric(bitgen_state, + colors[j], remaining, num_to_sample); + variates[i + j] = r; + num_to_sample -= r; + } + + if (num_to_sample > 0) { + variates[i + num_colors - 1] = num_to_sample; + } + + if (more_than_half) { + for (size_t k = 0; k < num_colors; ++k) { + variates[i + k] = colors[k] - variates[i + k]; + } + } + } +} diff --git a/numpy/random/src/legacy/legacy-distributions.c b/numpy/random/src/legacy/legacy-distributions.c index 684b3d762..fd067fe8d 100644 --- a/numpy/random/src/legacy/legacy-distributions.c +++ b/numpy/random/src/legacy/legacy-distributions.c @@ -1,4 +1,4 @@ -#include "legacy-distributions.h" +#include "include/legacy-distributions.h" static NPY_INLINE double legacy_double(aug_bitgen_t *aug_state) { @@ -294,8 +294,8 @@ static RAND_INT_TYPE random_hypergeometric_hrua(bitgen_t *bitgen_state, d7 = sqrt((double)(popsize - m) * sample * d4 * d5 / (popsize - 1) + 0.5); d8 = D1 * d7 + D2; d9 = (RAND_INT_TYPE)floor((double)(m + 1) * (mingoodbad + 1) / (popsize + 2)); - d10 = (loggam(d9 + 1) + loggam(mingoodbad - d9 + 1) + loggam(m - d9 + 1) + - loggam(maxgoodbad - m + d9 + 1)); + d10 = (random_loggam(d9 + 1) + random_loggam(mingoodbad - d9 + 1) + + random_loggam(m - d9 + 1) + random_loggam(maxgoodbad - m + d9 + 1)); d11 = MIN(MIN(m, mingoodbad) + 1.0, floor(d6 + 16 * d7)); /* 16 for 16-decimal-digit precision in D1 and D2 */ @@ -309,8 +309,8 @@ static RAND_INT_TYPE random_hypergeometric_hrua(bitgen_t *bitgen_state, continue; Z = (RAND_INT_TYPE)floor(W); - T = d10 - (loggam(Z + 1) + loggam(mingoodbad - Z + 1) + loggam(m - Z + 1) + - loggam(maxgoodbad - m + Z + 1)); + T = d10 - (random_loggam(Z + 1) + random_loggam(mingoodbad - Z + 1) + + random_loggam(m - Z + 1) + random_loggam(maxgoodbad - m + Z + 1)); /* fast acceptance: */ if ((X * (4.0 - X) - 3.0) <= T) diff --git a/numpy/random/src/philox/philox.h b/numpy/random/src/philox/philox.h index 309d89eae..c72424a97 100644 --- a/numpy/random/src/philox/philox.h +++ b/numpy/random/src/philox/philox.h @@ -1,8 +1,8 @@ #ifndef _RANDOMDGEN__PHILOX_H_ #define _RANDOMDGEN__PHILOX_H_ -#include <inttypes.h> #include "numpy/npy_common.h" +#include <inttypes.h> #define PHILOX_BUFFER_SIZE 4L diff --git a/numpy/random/src/sfc64/sfc64.h b/numpy/random/src/sfc64/sfc64.h index 6674ae69c..75c4118d3 100644 --- a/numpy/random/src/sfc64/sfc64.h +++ b/numpy/random/src/sfc64/sfc64.h @@ -1,11 +1,11 @@ #ifndef _RANDOMDGEN__SFC64_H_ #define _RANDOMDGEN__SFC64_H_ +#include "numpy/npy_common.h" #include <inttypes.h> #ifdef _WIN32 #include <stdlib.h> #endif -#include "numpy/npy_common.h" typedef struct s_sfc64_state { uint64_t s[4]; diff --git a/numpy/random/tests/test_direct.py b/numpy/random/tests/test_direct.py index 0f57c4bd4..9f77f0ad2 100644 --- a/numpy/random/tests/test_direct.py +++ b/numpy/random/tests/test_direct.py @@ -1,5 +1,6 @@ import os from os.path import join +import sys import numpy as np from numpy.testing import (assert_equal, assert_allclose, assert_array_equal, @@ -10,7 +11,7 @@ from numpy.random import ( Generator, MT19937, PCG64, Philox, RandomState, SeedSequence, SFC64, default_rng ) -from numpy.random.common import interface +from numpy.random._common import interface try: import cffi # noqa: F401 @@ -26,6 +27,12 @@ try: except ImportError: MISSING_CTYPES = False +if sys.flags.optimize > 1: + # no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1 + # cffi cannot succeed + MISSING_CFFI = True + + pwd = os.path.dirname(os.path.abspath(__file__)) @@ -120,7 +127,7 @@ def gauss_from_uint(x, n, bits): return gauss[:n] def test_seedsequence(): - from numpy.random.bit_generator import (ISeedSequence, + from numpy.random._bit_generator import (ISeedSequence, ISpawnableSeedSequence, SeedlessSeedSequence) diff --git a/numpy/random/tests/test_extending.py b/numpy/random/tests/test_extending.py new file mode 100644 index 000000000..807de1a25 --- /dev/null +++ b/numpy/random/tests/test_extending.py @@ -0,0 +1,51 @@ +import os, sys +import pytest +import warnings + +try: + import cffi +except ImportError: + cffi = None + +if sys.flags.optimize > 1: + # no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1 + # cffi cannot succeed + cffi = None + +try: + with warnings.catch_warnings(record=True) as w: + # numba issue gh-4733 + warnings.filterwarnings('always', '', DeprecationWarning) + import numba +except ImportError: + numba = None + +try: + import cython +except ImportError: + cython = None + +@pytest.mark.skipif(cython is None, reason="requires cython") +def test_cython(): + curdir = os.getcwd() + argv = sys.argv + examples = (os.path.dirname(__file__), '..', '_examples') + try: + os.chdir(os.path.join(*examples)) + sys.argv = argv[:1] + ['build'] + with warnings.catch_warnings(record=True) as w: + # setuptools issue gh-1885 + warnings.filterwarnings('always', '', DeprecationWarning) + from numpy.random._examples.cython import setup + finally: + sys.argv = argv + os.chdir(curdir) + +@pytest.mark.skipif(numba is None or cffi is None, + reason="requires numba and cffi") +def test_numba(): + from numpy.random._examples.numba import extending + +@pytest.mark.skipif(cffi is None, reason="requires cffi") +def test_cffi(): + from numpy.random._examples.cffi import extending diff --git a/numpy/random/tests/test_generator_mt19937.py b/numpy/random/tests/test_generator_mt19937.py index 1f18c9ca1..d835f16bd 100644 --- a/numpy/random/tests/test_generator_mt19937.py +++ b/numpy/random/tests/test_generator_mt19937.py @@ -3,8 +3,10 @@ import sys import pytest import numpy as np +from numpy.dual import cholesky, eigh, svd +from numpy.linalg import LinAlgError from numpy.testing import ( - assert_, assert_raises, assert_equal, + assert_, assert_raises, assert_equal, assert_allclose, assert_warns, assert_no_warnings, assert_array_equal, assert_array_almost_equal, suppress_warnings) @@ -115,6 +117,140 @@ class TestMultinomial(object): assert_array_equal(non_contig, contig) +class TestMultivariateHypergeometric(object): + + def setup(self): + self.seed = 8675309 + + def test_argument_validation(self): + # Error cases... + + # `colors` must be a 1-d sequence + assert_raises(ValueError, random.multivariate_hypergeometric, + 10, 4) + + # Negative nsample + assert_raises(ValueError, random.multivariate_hypergeometric, + [2, 3, 4], -1) + + # Negative color + assert_raises(ValueError, random.multivariate_hypergeometric, + [-1, 2, 3], 2) + + # nsample exceeds sum(colors) + assert_raises(ValueError, random.multivariate_hypergeometric, + [2, 3, 4], 10) + + # nsample exceeds sum(colors) (edge case of empty colors) + assert_raises(ValueError, random.multivariate_hypergeometric, + [], 1) + + # Validation errors associated with very large values in colors. + assert_raises(ValueError, random.multivariate_hypergeometric, + [999999999, 101], 5, 1, 'marginals') + + int64_info = np.iinfo(np.int64) + max_int64 = int64_info.max + max_int64_index = max_int64 // int64_info.dtype.itemsize + assert_raises(ValueError, random.multivariate_hypergeometric, + [max_int64_index - 100, 101], 5, 1, 'count') + + @pytest.mark.parametrize('method', ['count', 'marginals']) + def test_edge_cases(self, method): + # Set the seed, but in fact, all the results in this test are + # deterministic, so we don't really need this. + random = Generator(MT19937(self.seed)) + + x = random.multivariate_hypergeometric([0, 0, 0], 0, method=method) + assert_array_equal(x, [0, 0, 0]) + + x = random.multivariate_hypergeometric([], 0, method=method) + assert_array_equal(x, []) + + x = random.multivariate_hypergeometric([], 0, size=1, method=method) + assert_array_equal(x, np.empty((1, 0), dtype=np.int64)) + + x = random.multivariate_hypergeometric([1, 2, 3], 0, method=method) + assert_array_equal(x, [0, 0, 0]) + + x = random.multivariate_hypergeometric([9, 0, 0], 3, method=method) + assert_array_equal(x, [3, 0, 0]) + + colors = [1, 1, 0, 1, 1] + x = random.multivariate_hypergeometric(colors, sum(colors), + method=method) + assert_array_equal(x, colors) + + x = random.multivariate_hypergeometric([3, 4, 5], 12, size=3, + method=method) + assert_array_equal(x, [[3, 4, 5]]*3) + + # Cases for nsample: + # nsample < 10 + # 10 <= nsample < colors.sum()/2 + # colors.sum()/2 < nsample < colors.sum() - 10 + # colors.sum() - 10 < nsample < colors.sum() + @pytest.mark.parametrize('nsample', [8, 25, 45, 55]) + @pytest.mark.parametrize('method', ['count', 'marginals']) + @pytest.mark.parametrize('size', [5, (2, 3), 150000]) + def test_typical_cases(self, nsample, method, size): + random = Generator(MT19937(self.seed)) + + colors = np.array([10, 5, 20, 25]) + sample = random.multivariate_hypergeometric(colors, nsample, size, + method=method) + if isinstance(size, int): + expected_shape = (size,) + colors.shape + else: + expected_shape = size + colors.shape + assert_equal(sample.shape, expected_shape) + assert_((sample >= 0).all()) + assert_((sample <= colors).all()) + assert_array_equal(sample.sum(axis=-1), + np.full(size, fill_value=nsample, dtype=int)) + if isinstance(size, int) and size >= 100000: + # This sample is large enough to compare its mean to + # the expected values. + assert_allclose(sample.mean(axis=0), + nsample * colors / colors.sum(), + rtol=1e-3, atol=0.005) + + def test_repeatability1(self): + random = Generator(MT19937(self.seed)) + sample = random.multivariate_hypergeometric([3, 4, 5], 5, size=5, + method='count') + expected = np.array([[2, 1, 2], + [2, 1, 2], + [1, 1, 3], + [2, 0, 3], + [2, 1, 2]]) + assert_array_equal(sample, expected) + + def test_repeatability2(self): + random = Generator(MT19937(self.seed)) + sample = random.multivariate_hypergeometric([20, 30, 50], 50, + size=5, + method='marginals') + expected = np.array([[ 9, 17, 24], + [ 7, 13, 30], + [ 9, 15, 26], + [ 9, 17, 24], + [12, 14, 24]]) + assert_array_equal(sample, expected) + + def test_repeatability3(self): + random = Generator(MT19937(self.seed)) + sample = random.multivariate_hypergeometric([20, 30, 50], 12, + size=5, + method='marginals') + expected = np.array([[2, 3, 7], + [5, 3, 4], + [2, 5, 5], + [5, 3, 4], + [1, 5, 6]]) + assert_array_equal(sample, expected) + + class TestSetState(object): def setup(self): self.seed = 1234567890 @@ -358,9 +494,8 @@ class TestIntegers(object): def test_repeatability_broadcasting(self, endpoint): for dt in self.itype: - lbnd = 0 if dt in (np.bool, bool, np.bool_) else np.iinfo(dt).min - ubnd = 2 if dt in ( - np.bool, bool, np.bool_) else np.iinfo(dt).max + 1 + lbnd = 0 if dt in (bool, np.bool_) else np.iinfo(dt).min + ubnd = 2 if dt in (bool, np.bool_) else np.iinfo(dt).max + 1 ubnd = ubnd - 1 if endpoint else ubnd # view as little endian for hash @@ -399,8 +534,8 @@ class TestIntegers(object): assert_raises(ValueError, random.integers, low_a, high_a, endpoint=endpoint, dtype=dtype) - low_o = np.array([[low]*10], dtype=np.object) - high_o = np.array([high] * 10, dtype=np.object) + low_o = np.array([[low]*10], dtype=object) + high_o = np.array([high] * 10, dtype=object) assert_raises(ValueError, random.integers, low_o, high, endpoint=endpoint, dtype=dtype) assert_raises(ValueError, random.integers, low, high_o, @@ -442,7 +577,7 @@ class TestIntegers(object): sample = self.rfunc(lbnd, ubnd, endpoint=endpoint, dtype=dt) assert_equal(sample.dtype, dt) - for dt in (bool, int, np.long): + for dt in (bool, int, np.compat.long): lbnd = 0 if dt is bool else np.iinfo(dt).min ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 ubnd = ubnd - 1 if endpoint else ubnd @@ -750,6 +885,29 @@ class TestRandomDist(object): desired = conv([4, 1, 9, 8, 0, 5, 3, 6, 2, 7]) assert_array_equal(actual, desired) + def test_shuffle_custom_axis(self): + random = Generator(MT19937(self.seed)) + actual = np.arange(16).reshape((4, 4)) + random.shuffle(actual, axis=1) + desired = np.array([[ 0, 3, 1, 2], + [ 4, 7, 5, 6], + [ 8, 11, 9, 10], + [12, 15, 13, 14]]) + assert_array_equal(actual, desired) + random = Generator(MT19937(self.seed)) + actual = np.arange(16).reshape((4, 4)) + random.shuffle(actual, axis=-1) + assert_array_equal(actual, desired) + + def test_shuffle_axis_nonsquare(self): + y1 = np.arange(20).reshape(2, 10) + y2 = y1.copy() + random = Generator(MT19937(self.seed)) + random.shuffle(y1, axis=1) + random = Generator(MT19937(self.seed)) + random.shuffle(y2.T) + assert_array_equal(y1, y2) + def test_shuffle_masked(self): # gh-3263 a = np.ma.masked_values(np.reshape(range(20), (5, 4)) % 3 - 1, -1) @@ -764,6 +922,16 @@ class TestRandomDist(object): assert_equal( sorted(b.data[~b.mask]), sorted(b_orig.data[~b_orig.mask])) + def test_shuffle_exceptions(self): + random = Generator(MT19937(self.seed)) + arr = np.arange(10) + assert_raises(np.AxisError, random.shuffle, arr, 1) + arr = np.arange(9).reshape((3, 3)) + assert_raises(np.AxisError, random.shuffle, arr, 3) + assert_raises(TypeError, random.shuffle, arr, slice(1, 2, None)) + arr = [[1, 2, 3], [4, 5, 6]] + assert_raises(NotImplementedError, random.shuffle, arr, 1) + def test_permutation(self): random = Generator(MT19937(self.seed)) alist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] @@ -789,6 +957,27 @@ class TestRandomDist(object): actual = random.permutation(integer_val) assert_array_equal(actual, desired) + def test_permutation_custom_axis(self): + a = np.arange(16).reshape((4, 4)) + desired = np.array([[ 0, 3, 1, 2], + [ 4, 7, 5, 6], + [ 8, 11, 9, 10], + [12, 15, 13, 14]]) + random = Generator(MT19937(self.seed)) + actual = random.permutation(a, axis=1) + assert_array_equal(actual, desired) + random = Generator(MT19937(self.seed)) + actual = random.permutation(a, axis=-1) + assert_array_equal(actual, desired) + + def test_permutation_exceptions(self): + random = Generator(MT19937(self.seed)) + arr = np.arange(10) + assert_raises(np.AxisError, random.permutation, arr, 1) + arr = np.arange(9).reshape((3, 3)) + assert_raises(np.AxisError, random.permutation, arr, 3) + assert_raises(TypeError, random.permutation, arr, slice(1, 2, None)) + def test_beta(self): random = Generator(MT19937(self.seed)) actual = random.beta(.1, .9, size=(3, 2)) @@ -1008,12 +1197,13 @@ class TestRandomDist(object): [5, 5, 3, 1, 2, 4]]]) assert_array_equal(actual, desired) - def test_multivariate_normal(self): + @pytest.mark.parametrize("method", ["svd", "eigh", "cholesky"]) + def test_multivariate_normal(self, method): random = Generator(MT19937(self.seed)) mean = (.123456789, 10) cov = [[1, 0], [0, 1]] size = (3, 2) - actual = random.multivariate_normal(mean, cov, size) + actual = random.multivariate_normal(mean, cov, size, method=method) desired = np.array([[[-1.747478062846581, 11.25613495182354 ], [-0.9967333370066214, 10.342002097029821 ]], [[ 0.7850019631242964, 11.181113712443013 ], @@ -1024,15 +1214,24 @@ class TestRandomDist(object): assert_array_almost_equal(actual, desired, decimal=15) # Check for default size, was raising deprecation warning - actual = random.multivariate_normal(mean, cov) + actual = random.multivariate_normal(mean, cov, method=method) desired = np.array([0.233278563284287, 9.424140804347195]) assert_array_almost_equal(actual, desired, decimal=15) + # Check that non symmetric covariance input raises exception when + # check_valid='raises' if using default svd method. + mean = [0, 0] + cov = [[1, 2], [1, 2]] + assert_raises(ValueError, random.multivariate_normal, mean, cov, + check_valid='raise') # Check that non positive-semidefinite covariance warns with # RuntimeWarning - mean = [0, 0] cov = [[1, 2], [2, 1]] assert_warns(RuntimeWarning, random.multivariate_normal, mean, cov) + assert_warns(RuntimeWarning, random.multivariate_normal, mean, cov, + method='eigh') + assert_raises(LinAlgError, random.multivariate_normal, mean, cov, + method='cholesky') # and that it doesn't warn with RuntimeWarning check_valid='ignore' assert_no_warnings(random.multivariate_normal, mean, cov, @@ -1041,10 +1240,12 @@ class TestRandomDist(object): # and that it raises with RuntimeWarning check_valid='raises' assert_raises(ValueError, random.multivariate_normal, mean, cov, check_valid='raise') + assert_raises(ValueError, random.multivariate_normal, mean, cov, + check_valid='raise', method='eigh') cov = np.array([[1, 0.1], [0.1, 1]], dtype=np.float32) with suppress_warnings() as sup: - random.multivariate_normal(mean, cov) + random.multivariate_normal(mean, cov, method=method) w = sup.record(RuntimeWarning) assert len(w) == 0 @@ -2018,7 +2219,7 @@ class TestSingleEltArrayInput(object): assert_equal(out.shape, self.tgtShape) def test_integers(self, endpoint): - itype = [np.bool, np.int8, np.uint8, np.int16, np.uint16, + itype = [np.bool_, np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64] func = random.integers high = np.array([1]) diff --git a/numpy/random/tests/test_random.py b/numpy/random/tests/test_random.py index 37bd121f3..2e2ecedf8 100644 --- a/numpy/random/tests/test_random.py +++ b/numpy/random/tests/test_random.py @@ -269,7 +269,7 @@ class TestRandint(object): sample = self.rfunc(lbnd, ubnd, dtype=dt) assert_equal(sample.dtype, np.dtype(dt)) - for dt in (bool, int, np.long): + for dt in (bool, int, np.compat.long): lbnd = 0 if dt is bool else np.iinfo(dt).min ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 diff --git a/numpy/random/tests/test_randomstate.py b/numpy/random/tests/test_randomstate.py index a0edc5c23..c12b685ad 100644 --- a/numpy/random/tests/test_randomstate.py +++ b/numpy/random/tests/test_randomstate.py @@ -11,7 +11,8 @@ from numpy.testing import ( suppress_warnings ) -from numpy.random import MT19937, PCG64, mtrand as random +from numpy.random import MT19937, PCG64 +from numpy import random INT_FUNCS = {'binomial': (100.0, 0.6), 'geometric': (.5,), @@ -228,7 +229,7 @@ class TestSetState(object): new_state = ('Unknown', ) + state[1:] assert_raises(ValueError, self.random_state.set_state, new_state) assert_raises(TypeError, self.random_state.set_state, - np.array(new_state, dtype=np.object)) + np.array(new_state, dtype=object)) state = self.random_state.get_state(legacy=False) del state['bit_generator'] assert_raises(ValueError, self.random_state.set_state, state) @@ -381,7 +382,7 @@ class TestRandint(object): sample = self.rfunc(lbnd, ubnd, dtype=dt) assert_equal(sample.dtype, np.dtype(dt)) - for dt in (bool, int, np.long): + for dt in (bool, int, np.compat.long): lbnd = 0 if dt is bool else np.iinfo(dt).min ubnd = 2 if dt is bool else np.iinfo(dt).max + 1 @@ -454,7 +455,7 @@ class TestRandomDist(object): random.seed(self.seed) rs = random.RandomState(self.seed) actual = rs.tomaxint(size=(3, 2)) - if np.iinfo(np.int).max == 2147483647: + if np.iinfo(int).max == 2147483647: desired = np.array([[1328851649, 731237375], [1270502067, 320041495], [1908433478, 499156889]], dtype=np.int64) diff --git a/numpy/random/tests/test_randomstate_regression.py b/numpy/random/tests/test_randomstate_regression.py index edf32ea97..bdc2214b6 100644 --- a/numpy/random/tests/test_randomstate_regression.py +++ b/numpy/random/tests/test_randomstate_regression.py @@ -8,7 +8,7 @@ from numpy.testing import ( from numpy.compat import long import numpy as np -from numpy.random import mtrand as random +from numpy import random class TestRegression(object): diff --git a/numpy/random/tests/test_seed_sequence.py b/numpy/random/tests/test_seed_sequence.py index 8d6d604a2..fe23680ed 100644 --- a/numpy/random/tests/test_seed_sequence.py +++ b/numpy/random/tests/test_seed_sequence.py @@ -1,7 +1,7 @@ import numpy as np from numpy.testing import assert_array_equal -from numpy.random.bit_generator import SeedSequence +from numpy.random import SeedSequence def test_reference_data(): diff --git a/numpy/random/tests/test_smoke.py b/numpy/random/tests/test_smoke.py index 6e641b5f4..58ef6a09a 100644 --- a/numpy/random/tests/test_smoke.py +++ b/numpy/random/tests/test_smoke.py @@ -8,7 +8,7 @@ from numpy.testing import assert_equal, assert_, assert_array_equal from numpy.random import (Generator, MT19937, PCG64, Philox, SFC64) @pytest.fixture(scope='module', - params=(np.bool, np.int8, np.int16, np.int32, np.int64, + params=(np.bool_, np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64)) def dtype(request): return request.param @@ -655,7 +655,7 @@ class RNG(object): rg.standard_gamma(1.0, out=existing[::3]) def test_integers_broadcast(self, dtype): - if dtype == np.bool: + if dtype == np.bool_: upper = 2 lower = 0 else: @@ -672,7 +672,7 @@ class RNG(object): assert_equal(a, c) self._reset_state() d = self.rg.integers(np.array( - [lower] * 10), np.array([upper], dtype=np.object), size=10, + [lower] * 10), np.array([upper], dtype=object), size=10, dtype=dtype) assert_equal(a, d) self._reset_state() @@ -701,7 +701,7 @@ class RNG(object): assert out.shape == (1,) def test_integers_broadcast_errors(self, dtype): - if dtype == np.bool: + if dtype == np.bool_: upper = 2 lower = 0 else: diff --git a/numpy/testing/_private/parameterized.py b/numpy/testing/_private/parameterized.py index a5fa4fb5e..489d8e09a 100644 --- a/numpy/testing/_private/parameterized.py +++ b/numpy/testing/_private/parameterized.py @@ -45,11 +45,18 @@ except ImportError: from unittest import TestCase -PY3 = sys.version_info[0] == 3 PY2 = sys.version_info[0] == 2 -if PY3: +if PY2: + from types import InstanceType + lzip = zip + text_type = unicode + bytes_type = str + string_types = basestring, + def make_method(func, instance, type): + return MethodType(func, instance, type) +else: # Python 3 doesn't have an InstanceType, so just use a dummy type. class InstanceType(): pass @@ -61,14 +68,6 @@ if PY3: if instance is None: return func return MethodType(func, instance) -else: - from types import InstanceType - lzip = zip - text_type = unicode - bytes_type = str - string_types = basestring, - def make_method(func, instance, type): - return MethodType(func, instance, type) _param = namedtuple("param", "args kwargs") diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 306967216..23267a9e1 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -21,7 +21,7 @@ import pprint from numpy.core import( intp, float32, empty, arange, array_repr, ndarray, isnat, array) -from numpy.lib.utils import deprecate +import numpy.__config__ if sys.version_info[0] >= 3: from io import StringIO @@ -33,14 +33,14 @@ __all__ = [ 'assert_array_equal', 'assert_array_less', 'assert_string_equal', 'assert_array_almost_equal', 'assert_raises', 'build_err_msg', 'decorate_methods', 'jiffies', 'memusage', 'print_assert_equal', - 'raises', 'rand', 'rundocs', 'runstring', 'verbose', 'measure', + 'raises', 'rundocs', 'runstring', 'verbose', 'measure', 'assert_', 'assert_array_almost_equal_nulp', 'assert_raises_regex', 'assert_array_max_ulp', 'assert_warns', 'assert_no_warnings', 'assert_allclose', 'IgnoreException', 'clear_and_catch_warnings', 'SkipTest', 'KnownFailureException', 'temppath', 'tempdir', 'IS_PYPY', 'HAS_REFCOUNT', 'suppress_warnings', 'assert_array_compare', '_assert_valid_refcount', '_gen_alignment_data', 'assert_no_gc_cycles', - 'break_cycles', + 'break_cycles', 'HAS_LAPACK64' ] @@ -54,6 +54,7 @@ verbose = 0 IS_PYPY = platform.python_implementation() == 'PyPy' HAS_REFCOUNT = getattr(sys, 'getrefcount', None) is not None +HAS_LAPACK64 = hasattr(numpy.__config__, 'lapack_ilp64_opt_info') def import_nose(): @@ -154,22 +155,6 @@ def gisinf(x): return st -@deprecate(message="numpy.testing.rand is deprecated in numpy 1.11. " - "Use numpy.random.rand instead.") -def rand(*args): - """Returns an array of random numbers with the given shape. - - This only uses the standard library, so it is useful for testing purposes. - """ - import random - from numpy.core import zeros, float64 - results = zeros(args, float64) - f = results.flat - for i in range(len(f)): - f[i] = random.random() - return results - - if os.name == 'nt': # Code "stolen" from enthought/debug/memusage.py def GetPerformanceAttributes(object, counter, instance=None, @@ -2397,3 +2382,97 @@ def break_cycles(): gc.collect() # one more, just to make sure gc.collect() + + +def requires_memory(free_bytes): + """Decorator to skip a test if not enough memory is available""" + import pytest + + def decorator(func): + @wraps(func) + def wrapper(*a, **kw): + msg = check_free_memory(free_bytes) + if msg is not None: + pytest.skip(msg) + + try: + return func(*a, **kw) + except MemoryError: + # Probably ran out of memory regardless: don't regard as failure + pytest.xfail("MemoryError raised") + + return wrapper + + return decorator + + +def check_free_memory(free_bytes): + """ + Check whether `free_bytes` amount of memory is currently free. + Returns: None if enough memory available, otherwise error message + """ + env_var = 'NPY_AVAILABLE_MEM' + env_value = os.environ.get(env_var) + if env_value is not None: + try: + mem_free = _parse_size(env_value) + except ValueError as exc: + raise ValueError('Invalid environment variable {}: {!s}'.format( + env_var, exc)) + + msg = ('{0} GB memory required, but environment variable ' + 'NPY_AVAILABLE_MEM={1} set'.format( + free_bytes/1e9, env_value)) + else: + mem_free = _get_mem_available() + + if mem_free is None: + msg = ("Could not determine available memory; set NPY_AVAILABLE_MEM " + "environment variable (e.g. NPY_AVAILABLE_MEM=16GB) to run " + "the test.") + mem_free = -1 + else: + msg = '{0} GB memory required, but {1} GB available'.format( + free_bytes/1e9, mem_free/1e9) + + return msg if mem_free < free_bytes else None + + +def _parse_size(size_str): + """Convert memory size strings ('12 GB' etc.) to float""" + suffixes = {'': 1, 'b': 1, + 'k': 1000, 'm': 1000**2, 'g': 1000**3, 't': 1000**4, + 'kb': 1000, 'mb': 1000**2, 'gb': 1000**3, 'tb': 1000**4, + 'kib': 1024, 'mib': 1024**2, 'gib': 1024**3, 'tib': 1024**4} + + size_re = re.compile(r'^\s*(\d+|\d+\.\d+)\s*({0})\s*$'.format( + '|'.join(suffixes.keys())), re.I) + + m = size_re.match(size_str.lower()) + if not m or m.group(2) not in suffixes: + raise ValueError("value {!r} not a valid size".format(size_str)) + return int(float(m.group(1)) * suffixes[m.group(2)]) + + +def _get_mem_available(): + """Return available memory in bytes, or None if unknown.""" + try: + import psutil + return psutil.virtual_memory().available + except (ImportError, AttributeError): + pass + + if sys.platform.startswith('linux'): + info = {} + with open('/proc/meminfo', 'r') as f: + for line in f: + p = line.split() + info[p[0].strip(':').lower()] = int(p[1]) * 1024 + + if 'memavailable' in info: + # Linux >= 3.14 + return info['memavailable'] + else: + return info['memfree'] + info['cached'] + + return None diff --git a/numpy/testing/decorators.py b/numpy/testing/decorators.py deleted file mode 100644 index bf78be500..000000000 --- a/numpy/testing/decorators.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Back compatibility decorators module. It will import the appropriate -set of tools - -""" -from __future__ import division, absolute_import, print_function - -import warnings - -# 2018-04-04, numpy 1.15.0 -warnings.warn("Importing from numpy.testing.decorators is deprecated " - "since numpy 1.15.0, import from numpy.testing instead.", - DeprecationWarning, stacklevel=2) - -from ._private.decorators import * diff --git a/numpy/testing/noseclasses.py b/numpy/testing/noseclasses.py deleted file mode 100644 index 5748a9a0f..000000000 --- a/numpy/testing/noseclasses.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Back compatibility noseclasses module. It will import the appropriate -set of tools -""" -from __future__ import division, absolute_import, print_function - -import warnings - -# 2018-04-04, numpy 1.15.0 -warnings.warn("Importing from numpy.testing.noseclasses is deprecated " - "since 1.15.0, import from numpy.testing instead", - DeprecationWarning, stacklevel=2) - -from ._private.noseclasses import * diff --git a/numpy/testing/nosetester.py b/numpy/testing/nosetester.py deleted file mode 100644 index 2ac212eee..000000000 --- a/numpy/testing/nosetester.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Back compatibility nosetester module. It will import the appropriate -set of tools - -""" -from __future__ import division, absolute_import, print_function - -import warnings - -# 2018-04-04, numpy 1.15.0 -warnings.warn("Importing from numpy.testing.nosetester is deprecated " - "since 1.15.0, import from numpy.testing instead.", - DeprecationWarning, stacklevel=2) - -from ._private.nosetester import * - -__all__ = ['get_package_name', 'run_module_suite', 'NoseTester', - '_numpy_tester', 'get_package_name', 'import_nose', - 'suppress_warnings'] diff --git a/numpy/testing/print_coercion_tables.py b/numpy/testing/print_coercion_tables.py index 3a359f472..72b22cee1 100755 --- a/numpy/testing/print_coercion_tables.py +++ b/numpy/testing/print_coercion_tables.py @@ -70,22 +70,24 @@ def print_coercion_table(ntypes, inputfirstvalue, inputsecondvalue, firstarray, print(char, end=' ') print() -print("can cast") -print_cancast_table(np.typecodes['All']) -print() -print("In these tables, ValueError is '!', OverflowError is '@', TypeError is '#'") -print() -print("scalar + scalar") -print_coercion_table(np.typecodes['All'], 0, 0, False) -print() -print("scalar + neg scalar") -print_coercion_table(np.typecodes['All'], 0, -1, False) -print() -print("array + scalar") -print_coercion_table(np.typecodes['All'], 0, 0, True) -print() -print("array + neg scalar") -print_coercion_table(np.typecodes['All'], 0, -1, True) -print() -print("promote_types") -print_coercion_table(np.typecodes['All'], 0, 0, False, True) + +if __name__ == '__main__': + print("can cast") + print_cancast_table(np.typecodes['All']) + print() + print("In these tables, ValueError is '!', OverflowError is '@', TypeError is '#'") + print() + print("scalar + scalar") + print_coercion_table(np.typecodes['All'], 0, 0, False) + print() + print("scalar + neg scalar") + print_coercion_table(np.typecodes['All'], 0, -1, False) + print() + print("array + scalar") + print_coercion_table(np.typecodes['All'], 0, 0, True) + print() + print("array + neg scalar") + print_coercion_table(np.typecodes['All'], 0, -1, True) + print() + print("promote_types") + print_coercion_table(np.typecodes['All'], 0, 0, False, True) diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py index ad72b9199..44f93a693 100644 --- a/numpy/testing/tests/test_utils.py +++ b/numpy/testing/tests/test_utils.py @@ -90,21 +90,6 @@ class TestArrayEqual(_GenericTest): for t in ['S1', 'U1']: foo(t) - def test_0_ndim_array(self): - x = np.array(473963742225900817127911193656584771) - y = np.array(18535119325151578301457182298393896) - assert_raises(AssertionError, self._assert_func, x, y) - - y = x - self._assert_func(x, y) - - x = np.array(43) - y = np.array(10) - assert_raises(AssertionError, self._assert_func, x, y) - - y = x - self._assert_func(x, y) - def test_generic_rank3(self): """Test rank 3 array for all dtypes.""" def foo(t): @@ -623,9 +608,9 @@ class TestApproxEqual(object): def setup(self): self._assert_func = assert_approx_equal - def test_simple_0d_arrays(self): - x = np.array(1234.22) - y = np.array(1234.23) + def test_simple_arrays(self): + x = np.array([1234.22]) + y = np.array([1234.23]) self._assert_func(x, y, significant=5) self._assert_func(x, y, significant=6) diff --git a/numpy/testing/utils.py b/numpy/testing/utils.py index 98f19e348..975f6ad5d 100644 --- a/numpy/testing/utils.py +++ b/numpy/testing/utils.py @@ -7,10 +7,11 @@ from __future__ import division, absolute_import, print_function import warnings -# 2018-04-04, numpy 1.15.0 +# 2018-04-04, numpy 1.15.0 ImportWarning +# 2019-09-18, numpy 1.18.0 DeprecatonWarning (changed) warnings.warn("Importing from numpy.testing.utils is deprecated " "since 1.15.0, import from numpy.testing instead.", - ImportWarning, stacklevel=2) + DeprecationWarning, stacklevel=2) from ._private.utils import * @@ -19,7 +20,7 @@ __all__ = [ 'assert_array_equal', 'assert_array_less', 'assert_string_equal', 'assert_array_almost_equal', 'assert_raises', 'build_err_msg', 'decorate_methods', 'jiffies', 'memusage', 'print_assert_equal', - 'raises', 'rand', 'rundocs', 'runstring', 'verbose', 'measure', + 'raises', 'rundocs', 'runstring', 'verbose', 'measure', 'assert_', 'assert_array_almost_equal_nulp', 'assert_raises_regex', 'assert_array_max_ulp', 'assert_warns', 'assert_no_warnings', 'assert_allclose', 'IgnoreException', 'clear_and_catch_warnings', diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index 807c98652..0484bb8cd 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -1,14 +1,22 @@ from __future__ import division, absolute_import, print_function import sys +import subprocess +import pkgutil +import types +import importlib +import warnings import numpy as np +import numpy import pytest + try: import ctypes except ImportError: ctypes = None + def check_dir(module, module_name=None): """Returns a mapping of all objects with the wrong __module__ attribute.""" if module_name is None: @@ -26,7 +34,8 @@ def check_dir(module, module_name=None): sys.version_info[0] < 3, reason="NumPy exposes slightly different functions on Python 2") def test_numpy_namespace(): - # None of these objects are publicly documented. + # None of these objects are publicly documented to be part of the main + # NumPy namespace (some are useful though, others need to be cleaned up) undocumented = { 'Tester': 'numpy.testing._private.nosetester.NoseTester', '_add_newdoc_ufunc': 'numpy.core._multiarray_umath._add_newdoc_ufunc', @@ -69,6 +78,28 @@ def test_numpy_namespace(): assert bad_results == whitelist +@pytest.mark.parametrize('name', ['testing', 'Tester']) +def test_import_lazy_import(name): + """Make sure we can actually use the modules we lazy load. + + While not exported as part of the public API, it was accessible. With the + use of __getattr__ and __dir__, this isn't always true It can happen that + an infinite recursion may happen. + + This is the only way I found that would force the failure to appear on the + badly implemented code. + + We also test for the presence of the lazily imported modules in dir + + """ + exe = (sys.executable, '-c', "import numpy; numpy." + name) + result = subprocess.check_output(exe) + assert not result + + # Make sure they are still in the __dir__ + assert name in dir(np) + + def test_numpy_linalg(): bad_results = check_dir(np.linalg) assert bad_results == {} @@ -78,6 +109,7 @@ def test_numpy_fft(): bad_results = check_dir(np.fft) assert bad_results == {} + @pytest.mark.skipif(ctypes is None, reason="ctypes not available in this python") def test_NPY_NO_EXPORT(): @@ -86,3 +118,373 @@ def test_NPY_NO_EXPORT(): f = getattr(cdll, 'test_not_exported', None) assert f is None, ("'test_not_exported' is mistakenly exported, " "NPY_NO_EXPORT does not work") + + +# Historically NumPy has not used leading underscores for private submodules +# much. This has resulted in lots of things that look like public modules +# (i.e. things that can be imported as `import numpy.somesubmodule.somefile`), +# but were never intended to be public. The PUBLIC_MODULES list contains +# modules that are either public because they were meant to be, or because they +# contain public functions/objects that aren't present in any other namespace +# for whatever reason and therefore should be treated as public. +# +# The PRIVATE_BUT_PRESENT_MODULES list contains modules that look public (lack +# of underscores) but should not be used. For many of those modules the +# current status is fine. For others it may make sense to work on making them +# private, to clean up our public API and avoid confusion. +PUBLIC_MODULES = ['numpy.' + s for s in [ + "ctypeslib", + "distutils", + "distutils.cpuinfo", + "distutils.exec_command", + "distutils.misc_util", + "distutils.log", + "distutils.system_info", + "doc", + "doc.basics", + "doc.broadcasting", + "doc.byteswapping", + "doc.constants", + "doc.creation", + "doc.dispatch", + "doc.glossary", + "doc.indexing", + "doc.internals", + "doc.misc", + "doc.structured_arrays", + "doc.subclassing", + "doc.ufuncs", + "dual", + "f2py", + "fft", + "lib", + "lib.format", # was this meant to be public? + "lib.mixins", + "lib.recfunctions", + "lib.scimath", + "linalg", + "ma", + "ma.extras", + "ma.mrecords", + "matlib", + "polynomial", + "polynomial.chebyshev", + "polynomial.hermite", + "polynomial.hermite_e", + "polynomial.laguerre", + "polynomial.legendre", + "polynomial.polynomial", + "polynomial.polyutils", + "random", + "testing", + "version", +]] + + +PUBLIC_ALIASED_MODULES = [ + "numpy.char", + "numpy.emath", + "numpy.rec", +] + + +PRIVATE_BUT_PRESENT_MODULES = ['numpy.' + s for s in [ + "compat", + "compat.py3k", + "conftest", + "core", + "core.arrayprint", + "core.defchararray", + "core.einsumfunc", + "core.fromnumeric", + "core.function_base", + "core.getlimits", + "core.machar", + "core.memmap", + "core.multiarray", + "core.numeric", + "core.numerictypes", + "core.overrides", + "core.records", + "core.shape_base", + "core.umath", + "core.umath_tests", + "distutils.ccompiler", + "distutils.command", + "distutils.command.autodist", + "distutils.command.bdist_rpm", + "distutils.command.build", + "distutils.command.build_clib", + "distutils.command.build_ext", + "distutils.command.build_py", + "distutils.command.build_scripts", + "distutils.command.build_src", + "distutils.command.config", + "distutils.command.config_compiler", + "distutils.command.develop", + "distutils.command.egg_info", + "distutils.command.install", + "distutils.command.install_clib", + "distutils.command.install_data", + "distutils.command.install_headers", + "distutils.command.sdist", + "distutils.compat", + "distutils.conv_template", + "distutils.core", + "distutils.extension", + "distutils.fcompiler", + "distutils.fcompiler.absoft", + "distutils.fcompiler.compaq", + "distutils.fcompiler.environment", + "distutils.fcompiler.g95", + "distutils.fcompiler.gnu", + "distutils.fcompiler.hpux", + "distutils.fcompiler.ibm", + "distutils.fcompiler.intel", + "distutils.fcompiler.lahey", + "distutils.fcompiler.mips", + "distutils.fcompiler.nag", + "distutils.fcompiler.none", + "distutils.fcompiler.pathf95", + "distutils.fcompiler.pg", + "distutils.fcompiler.sun", + "distutils.fcompiler.vast", + "distutils.from_template", + "distutils.intelccompiler", + "distutils.lib2def", + "distutils.line_endings", + "distutils.mingw32ccompiler", + "distutils.msvccompiler", + "distutils.npy_pkg_config", + "distutils.numpy_distribution", + "distutils.pathccompiler", + "distutils.unixccompiler", + "f2py.auxfuncs", + "f2py.capi_maps", + "f2py.cb_rules", + "f2py.cfuncs", + "f2py.common_rules", + "f2py.crackfortran", + "f2py.diagnose", + "f2py.f2py2e", + "f2py.f2py_testing", + "f2py.f90mod_rules", + "f2py.func2subr", + "f2py.rules", + "f2py.use_rules", + "fft.helper", + "lib.arraypad", + "lib.arraysetops", + "lib.arrayterator", + "lib.financial", + "lib.function_base", + "lib.histograms", + "lib.index_tricks", + "lib.nanfunctions", + "lib.npyio", + "lib.polynomial", + "lib.shape_base", + "lib.stride_tricks", + "lib.twodim_base", + "lib.type_check", + "lib.ufunclike", + "lib.user_array", # note: not in np.lib, but probably should just be deleted + "lib.utils", + "linalg.lapack_lite", + "linalg.linalg", + "ma.bench", + "ma.core", + "ma.testutils", + "ma.timer_comparison", + "matrixlib", + "matrixlib.defmatrix", + "random.mtrand", + "testing.print_coercion_tables", + "testing.utils", +]] + + +def is_unexpected(name): + """Check if this needs to be considered.""" + if '._' in name or '.tests' in name or '.setup' in name: + return False + + if name in PUBLIC_MODULES: + return False + + if name in PUBLIC_ALIASED_MODULES: + return False + + if name in PRIVATE_BUT_PRESENT_MODULES: + return False + + return True + + +# These are present in a directory with an __init__.py but cannot be imported +# code_generators/ isn't installed, but present for an inplace build +SKIP_LIST = [ + "numpy.core.code_generators", + "numpy.core.code_generators.genapi", + "numpy.core.code_generators.generate_umath", + "numpy.core.code_generators.ufunc_docstrings", + "numpy.core.code_generators.generate_numpy_api", + "numpy.core.code_generators.generate_ufunc_api", + "numpy.core.code_generators.numpy_api", + "numpy.core.cversions", + "numpy.core.generate_numpy_api", + "numpy.distutils.msvc9compiler", +] + + +def test_all_modules_are_expected(): + """ + Test that we don't add anything that looks like a new public module by + accident. Check is based on filenames. + """ + + modnames = [] + for _, modname, ispkg in pkgutil.walk_packages(path=np.__path__, + prefix=np.__name__ + '.', + onerror=None): + if is_unexpected(modname) and modname not in SKIP_LIST: + # We have a name that is new. If that's on purpose, add it to + # PUBLIC_MODULES. We don't expect to have to add anything to + # PRIVATE_BUT_PRESENT_MODULES. Use an underscore in the name! + modnames.append(modname) + + if modnames: + raise AssertionError("Found unexpected modules: {}".format(modnames)) + + +# Stuff that clearly shouldn't be in the API and is detected by the next test +# below +SKIP_LIST_2 = [ + 'numpy.math', + 'numpy.distutils.log.sys', + 'numpy.distutils.system_info.copy', + 'numpy.distutils.system_info.distutils', + 'numpy.distutils.system_info.log', + 'numpy.distutils.system_info.os', + 'numpy.distutils.system_info.platform', + 'numpy.distutils.system_info.re', + 'numpy.distutils.system_info.shutil', + 'numpy.distutils.system_info.subprocess', + 'numpy.distutils.system_info.sys', + 'numpy.distutils.system_info.tempfile', + 'numpy.distutils.system_info.textwrap', + 'numpy.distutils.system_info.warnings', + 'numpy.doc.constants.re', + 'numpy.doc.constants.textwrap', + 'numpy.lib.emath', + 'numpy.lib.math', + 'numpy.matlib.char', + 'numpy.matlib.rec', + 'numpy.matlib.emath', + 'numpy.matlib.math', + 'numpy.matlib.linalg', + 'numpy.matlib.fft', + 'numpy.matlib.random', + 'numpy.matlib.ctypeslib', + 'numpy.matlib.ma', +] + + +def test_all_modules_are_expected_2(): + """ + Method checking all objects. The pkgutil-based method in + `test_all_modules_are_expected` does not catch imports into a namespace, + only filenames. So this test is more thorough, and checks this like: + + import .lib.scimath as emath + + To check if something in a module is (effectively) public, one can check if + there's anything in that namespace that's a public function/object but is + not exposed in a higher-level namespace. For example for a `numpy.lib` + submodule:: + + mod = np.lib.mixins + for obj in mod.__all__: + if obj in np.__all__: + continue + elif obj in np.lib.__all__: + continue + + else: + print(obj) + + """ + + def find_unexpected_members(mod_name): + members = [] + module = importlib.import_module(mod_name) + if hasattr(module, '__all__'): + objnames = module.__all__ + else: + objnames = dir(module) + + for objname in objnames: + if not objname.startswith('_'): + fullobjname = mod_name + '.' + objname + if isinstance(getattr(module, objname), types.ModuleType): + if is_unexpected(fullobjname): + if fullobjname not in SKIP_LIST_2: + members.append(fullobjname) + + return members + + unexpected_members = find_unexpected_members("numpy") + for modname in PUBLIC_MODULES: + unexpected_members.extend(find_unexpected_members(modname)) + + if unexpected_members: + raise AssertionError("Found unexpected object(s) that look like " + "modules: {}".format(unexpected_members)) + + +def test_api_importable(): + """ + Check that all submodules listed higher up in this file can be imported + + Note that if a PRIVATE_BUT_PRESENT_MODULES entry goes missing, it may + simply need to be removed from the list (deprecation may or may not be + needed - apply common sense). + """ + def check_importable(module_name): + try: + importlib.import_module(module_name) + except (ImportError, AttributeError): + return False + + return True + + module_names = [] + for module_name in PUBLIC_MODULES: + if not check_importable(module_name): + module_names.append(module_name) + + if module_names: + raise AssertionError("Modules in the public API that cannot be " + "imported: {}".format(module_names)) + + for module_name in PUBLIC_ALIASED_MODULES: + try: + eval(module_name) + except AttributeError: + module_names.append(module_name) + + if module_names: + raise AssertionError("Modules in the public API that were not " + "found: {}".format(module_names)) + + with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings('always', category=DeprecationWarning) + warnings.filterwarnings('always', category=ImportWarning) + for module_name in PRIVATE_BUT_PRESENT_MODULES: + if not check_importable(module_name): + module_names.append(module_name) + + if module_names: + raise AssertionError("Modules that are not really public but looked " + "public and can not be imported: " + "{}".format(module_names)) diff --git a/pavement.py b/pavement.py index 427fbc96d..889a552f6 100644 --- a/pavement.py +++ b/pavement.py @@ -12,8 +12,7 @@ Assumes you have git and the binaries/tarballs in installers/:: paver write_release paver write_note -This automatically put the checksum into README.rst, and write the Changelog -which can be uploaded to sourceforge. +This automatically put the checksum into README.rst, and writes the Changelog. TODO ==== @@ -42,7 +41,7 @@ from paver.easy import Bunch, options, task, sh #----------------------------------- # Path to the release notes -RELEASE_NOTES = 'doc/release/1.17.5-notes.rst' +RELEASE_NOTES = 'doc/source/release/1.18.0-notes.rst' #------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..918cbb278 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +[build-system] +# Minimum requirements for the build system to execute. +requires = [ + "setuptools", + "wheel", + "Cython>=0.29.13", # Note: keep in sync with tools/cythonize.py +] + + +[tool.towncrier] + # Do no set this since it is hard to import numpy inside the source directory + # the name is hardcoded. Use "--version 1.18.0" to set the version + single_file = true + filename = "doc/source/release/{version}-notes.rst" + directory = "doc/release/upcoming_changes/" + issue_format = "`gh-{issue} <https://github.com/numpy/numpy/pull/{issue}>`__" + template = "doc/release/upcoming_changes/template.rst" + underlines = "~=" + all_bullets = false + + + [[tool.towncrier.type]] + directory = "highlight" + name = "Highlights" + showcontent = true + + [[tool.towncrier.type]] + directory = "new_function" + name = "New functions" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "future" + name = "Future Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "expired" + name = "Expired deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "compatibility" + name = "Compatibility notes" + showcontent = true + + [[tool.towncrier.type]] + directory = "c_api" + name = "C API changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "new_feature" + name = "New Features" + showcontent = true + + [[tool.towncrier.type]] + directory = "improvement" + name = "Improvements" + showcontent = true + + [[tool.towncrier.type]] + directory = "change" + name = "Changes" + showcontent = true + diff --git a/runtests.py b/runtests.py index 23245aeac..a38054f86 100755 --- a/runtests.py +++ b/runtests.py @@ -18,6 +18,10 @@ Run a debugger: $ gdb --args python runtests.py [...other args...] +Disable pytest capturing of output by using its '-s' option: + + $ python runtests.py -- -s + Generate C code coverage listing under build/lcov/: (requires http://ltp.sourceforge.net/coverage/lcov.php) @@ -67,6 +71,10 @@ def main(argv): parser = ArgumentParser(usage=__doc__.lstrip()) parser.add_argument("--verbose", "-v", action="count", default=1, help="more verbosity") + parser.add_argument("--debug-info", action="store_true", + help=("add --verbose-cfg to build_src to show compiler " + "configuration output while creating " + "_numpyconfig.h and config.h")) parser.add_argument("--no-build", "-n", action="store_true", default=False, help="do not build the project (use system installed version)") parser.add_argument("--build-only", "-b", action="store_true", default=False, @@ -106,6 +114,8 @@ def main(argv): help="Debug build") parser.add_argument("--parallel", "-j", type=int, default=0, help="Number of parallel jobs during build") + parser.add_argument("--warn-error", action="store_true", + help="Set -Werror to convert all compiler warnings to errors") parser.add_argument("--show-build-log", action="store_true", help="Show build output rather than using a log file") parser.add_argument("--bench", action="store_true", @@ -366,6 +376,10 @@ def build_project(args): cmd += ["build"] if args.parallel > 1: cmd += ["-j", str(args.parallel)] + if args.debug_info: + cmd += ["build_src", "--verbose-cfg"] + if args.warn_error: + cmd += ["--warn-error"] # Install; avoid producing eggs so numpy can be imported from dst_dir. cmd += ['install', '--prefix=' + dst_dir, '--single-version-externally-managed', @@ -45,6 +45,7 @@ Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 +Programming Language :: Python :: 3 :: Only Programming Language :: Python :: Implementation :: CPython Topic :: Software Development Topic :: Scientific/Engineering @@ -55,8 +56,8 @@ Operating System :: MacOS """ MAJOR = 1 -MINOR = 17 -MICRO = 5 +MINOR = 18 +MICRO = 0 ISRELEASED = True VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) @@ -83,6 +84,10 @@ def git_version(): except (subprocess.SubprocessError, OSError): GIT_REVISION = "Unknown" + if not GIT_REVISION: + # this shouldn't happen but apparently can (see gh-8512) + GIT_REVISION = "Unknown" + return GIT_REVISION # BEFORE importing setuptools, remove MANIFEST. Otherwise it may not be @@ -109,8 +114,8 @@ def get_version_info(): try: from numpy.version import git_revision as GIT_REVISION except ImportError: - raise ImportError("Unable to import git_revision. Try removing " \ - "numpy/version.py and the build directory " \ + raise ImportError("Unable to import git_revision. Try removing " + "numpy/version.py and the build directory " "before building.") else: GIT_REVISION = "Unknown" @@ -263,7 +268,7 @@ def parse_setuppy_commands(): # below and not standalone. Hence they're not added to good_commands. good_commands = ('develop', 'sdist', 'build', 'build_ext', 'build_py', 'build_clib', 'build_scripts', 'bdist_wheel', 'bdist_rpm', - 'bdist_wininst', 'bdist_msi', 'bdist_mpkg') + 'bdist_wininst', 'bdist_msi', 'bdist_mpkg', 'build_src') for command in good_commands: if command in args: @@ -365,7 +370,7 @@ def parse_setuppy_commands(): def setup_package(): - src_path = os.path.dirname(os.path.abspath(sys.argv[0])) + src_path = os.path.dirname(os.path.abspath(__file__)) old_path = os.getcwd() os.chdir(src_path) sys.path.insert(0, src_path) @@ -403,7 +408,8 @@ def setup_package(): classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f], platforms = ["Windows", "Linux", "Solaris", "Mac OS-X", "Unix"], test_suite='nose.collector', - cmdclass={"sdist": sdist_checked}, + cmdclass={"sdist": sdist_checked, + }, python_requires='>=3.5', zip_safe=False, entry_points={ @@ -422,8 +428,8 @@ def setup_package(): if run_build: from numpy.distutils.core import setup cwd = os.path.abspath(os.path.dirname(__file__)) - if not os.path.exists(os.path.join(cwd, 'PKG-INFO')): - # Generate Cython sources, unless building from source release + if not 'sdist' in sys.argv: + # Generate Cython sources, unless we're generating an sdist generate_cython() metadata['configuration'] = configuration diff --git a/shippable.yml b/shippable.yml index e281a4c40..b5ce17751 100644 --- a/shippable.yml +++ b/shippable.yml @@ -32,9 +32,7 @@ build: # we will pay the ~13 minute cost of compiling Cython only when a new # version is scraped in by pip; otherwise, use the cached # wheel shippable places on Amazon S3 after we build it once - - pip install cython --cache-dir=/root/.cache/pip/wheels/$SHIPPABLE_PYTHON_VERSION - # install pytz for datetime testing - - pip install pytz + - pip install -r test_requirements.txt --cache-dir=/root/.cache/pip/wheels/$SHIPPABLE_PYTHON_VERSION # install pytest-xdist to leverage a second core # for unit tests - pip install pytest-xdist @@ -44,14 +42,14 @@ build: # build first and adjust PATH so f2py is found in scripts dir # use > 1 core for build sometimes slows down a fair bit, # other times modestly speeds up, so avoid for now - - python setup.py install + - pip install . - extra_directories=($SHIPPABLE_REPO_DIR/build/*scripts*) - extra_path=$(printf "%s:" "${extra_directories[@]}") - export PATH="${extra_path}${PATH}" # check OpenBLAS version - - python tools/openblas_support.py --check_version 0.3.5 + - python tools/openblas_support.py --check_version 0.3.7 # run the test suite - - python runtests.py -- -rsx --junit-xml=$SHIPPABLE_REPO_DIR/shippable/testresults/tests.xml -n 2 --durations=10 + - python runtests.py --debug-info --show-build-log -- -rsx --junit-xml=$SHIPPABLE_REPO_DIR/shippable/testresults/tests.xml -n 2 --durations=10 cache: true cache_dir_list: diff --git a/site.cfg.example b/site.cfg.example index b6b0175d6..cff076381 100644 --- a/site.cfg.example +++ b/site.cfg.example @@ -132,6 +132,51 @@ # include_dirs = /opt/OpenBLAS/include # runtime_library_dirs = /opt/OpenBLAS/lib +# OpenBLAS (64-bit with suffix) +# ----------------------------- +# OpenBLAS can be compiled with 64-bit integer size and symbol suffix '64_' +# (INTERFACE64=1 SYMBOLSUFFIX=64_). OpenBLAS built with this setting are also +# provided by some Linux distributions (e.g. Fedora's 64-bit openblas packages). +# This is an emerging "standard" for 64-bit BLAS/LAPACK, avoiding symbol clashes +# with 32-bit BLAS/LAPACK. +# +# To build Numpy with such 64-bit BLAS/LAPACK, set environment +# variables NPY_USE_BLAS_ILP64=1, NPY_BLAS_ILP64_ORDER=openblas64_, +# NPY_LAPACK_ILP64_ORDER=openblas64_ at build time. +# +# See: +# https://github.com/xianyi/OpenBLAS/issues/646 +# +# [openblas64_] +# libraries = openblas64_ +# library_dirs = /opt/OpenBLAS/lib +# include_dirs = /opt/OpenBLAS/include +# runtime_library_dirs = /opt/OpenBLAS/lib + +# OpenBLAS (64-bit ILP64) +# ----------------------- +# It is possible to also use OpenBLAS compiled with 64-bit integer +# size (ILP64) but no symbol name changes. To do that, set the +# environment variables NPY_USE_BLAS_ILP64=1, +# NPY_BLAS_ILP64_ORDER=openblas_ilp64, +# NPY_LAPACK_ILP64_ORDER=openblas_ilp64 at build time. +# +# Note that mixing both 64-bit and 32-bit BLAS without symbol suffixes +# in the same application may cause problems due to symbol name +# clashes, especially with embedded Python interpreters. +# +# The name of the library file may vary on different systems, so you +# may need to check your specific OpenBLAS installation and +# uncomment and e.g. set ``libraries = openblas`` below. +# +# [openblas_ilp64] +# libraries = openblas64 +# library_dirs = /opt/OpenBLAS/lib +# include_dirs = /opt/OpenBLAS/include +# runtime_library_dirs = /opt/OpenBLAS/lib +# symbol_prefix = +# symbol_suffix = + # BLIS # ---- # BLIS (https://github.com/flame/blis) also provides a BLAS interface. It's a diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 000000000..627150673 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,9 @@ +cython==0.29.14 +pytest==5.3.1 +pytz==2019.3 +pytest-cov==2.8.1 +pickle5; python_version == '3.7' +pickle5; python_version == '3.6' and platform_python_implementation != 'PyPy' +nose +# for numpy.random.test.test_extending +cffi diff --git a/tools/ci/appveyor/requirements.txt b/tools/ci/appveyor/requirements.txt deleted file mode 100644 index fba8260da..000000000 --- a/tools/ci/appveyor/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -cython -nose -pytest-timeout -pytest-xdist -pytest-env -pytest-faulthandler diff --git a/tools/ci/test_all_newsfragments_used.py b/tools/ci/test_all_newsfragments_used.py new file mode 100755 index 000000000..6c4591fd8 --- /dev/null +++ b/tools/ci/test_all_newsfragments_used.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import sys +import toml +import os + +path = toml.load("pyproject.toml")["tool"]["towncrier"]["directory"] + +fragments = os.listdir(path) +fragments.remove("README.rst") +fragments.remove("template.rst") + +if fragments: + print("The following files were not found by towncrier:") + print(" " + " \n".join(fragments)) + sys.exit(1) diff --git a/tools/cythonize.py b/tools/cythonize.py index c81b72d25..5bea2d4ec 100755 --- a/tools/cythonize.py +++ b/tools/cythonize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ cythonize Cythonize pyx files into C files as needed. @@ -54,30 +54,25 @@ except NameError: def process_pyx(fromfile, tofile): flags = ['-3', '--fast-fail'] if tofile.endswith('.cxx'): - flags += ['--cplus'] + flags.append('--cplus') try: # try the cython in the installed python first (somewhat related to scipy/scipy#2397) from Cython.Compiler.Version import version as cython_version except ImportError: - # if that fails, use the one on the path, which might be the wrong version - try: - # Try the one on the path as a last resort - subprocess.check_call( - ['cython'] + flags + ["-o", tofile, fromfile]) - except OSError: - raise OSError('Cython needs to be installed') + # The `cython` command need not point to the version installed in the + # Python running this script, so raise an error to avoid the chance of + # using the wrong version of Cython. + raise OSError('Cython needs to be installed in Python as a module') else: # check the version, and invoke through python from distutils.version import LooseVersion - # requiring the newest version on all pythons doesn't work, since - # we're relying on the version of the distribution cython. Add new - # versions as they become required for new python versions. - if sys.version_info[:2] < (3, 7): - required_version = LooseVersion('0.19') - else: - required_version = LooseVersion('0.28') + # Cython 0.29.13 is required for Python 3.8 and there are + # other fixes in the 0.29 series that are needed even for earlier + # Python versions. + # Note: keep in sync with that in pyproject.toml + required_version = LooseVersion('0.29.13') if LooseVersion(cython_version) < required_version: raise RuntimeError('Building {} requires Cython >= {}'.format( diff --git a/tools/npy_tempita/__init__.py b/tools/npy_tempita/__init__.py index dfb40e965..f75f23a21 100644 --- a/tools/npy_tempita/__init__.py +++ b/tools/npy_tempita/__init__.py @@ -105,21 +105,21 @@ class Template(object): def __init__(self, content, name=None, namespace=None, stacklevel=None, get_template=None, default_inherit=None, line_offset=0, - delimeters=None): + delimiters=None): self.content = content - # set delimeters - if delimeters is None: - delimeters = (self.default_namespace['start_braces'], + # set delimiters + if delimiters is None: + delimiters = (self.default_namespace['start_braces'], self.default_namespace['end_braces']) else: - assert len(delimeters) == 2 and all( - [isinstance(delimeter, basestring_) - for delimeter in delimeters]) + assert len(delimiters) == 2 and all( + [isinstance(delimiter, basestring_) + for delimiter in delimiters]) self.default_namespace = self.__class__.default_namespace.copy() - self.default_namespace['start_braces'] = delimeters[0] - self.default_namespace['end_braces'] = delimeters[1] - self.delimeters = delimeters + self.default_namespace['start_braces'] = delimiters[0] + self.default_namespace['end_braces'] = delimiters[1] + self.delimiters = delimiters self._unicode = is_unicode(content) if name is None and stacklevel is not None: @@ -143,7 +143,7 @@ class Template(object): self.name = name self._parsed = parse( content, name=name, line_offset=line_offset, - delimeters=self.delimeters) + delimiters=self.delimiters) if namespace is None: namespace = {} self.namespace = namespace @@ -392,9 +392,9 @@ class Template(object): return msg -def sub(content, delimeters=None, **kw): +def sub(content, delimiters=None, **kw): name = kw.get('__name') - tmpl = Template(content, name=name, delimeters=delimeters) + tmpl = Template(content, name=name, delimiters=delimiters) return tmpl.substitute(kw) @@ -652,28 +652,28 @@ del _Empty ############################################################ -def lex(s, name=None, trim_whitespace=True, line_offset=0, delimeters=None): - if delimeters is None: - delimeters = (Template.default_namespace['start_braces'], +def lex(s, name=None, trim_whitespace=True, line_offset=0, delimiters=None): + if delimiters is None: + delimiters = (Template.default_namespace['start_braces'], Template.default_namespace['end_braces']) in_expr = False chunks = [] last = 0 last_pos = (line_offset + 1, 1) - token_re = re.compile(r'%s|%s' % (re.escape(delimeters[0]), - re.escape(delimeters[1]))) + token_re = re.compile(r'%s|%s' % (re.escape(delimiters[0]), + re.escape(delimiters[1]))) for match in token_re.finditer(s): expr = match.group(0) pos = find_position(s, match.end(), last, last_pos) - if expr == delimeters[0] and in_expr: - raise TemplateError('%s inside expression' % delimeters[0], + if expr == delimiters[0] and in_expr: + raise TemplateError('%s inside expression' % delimiters[0], position=pos, name=name) - elif expr == delimeters[1] and not in_expr: - raise TemplateError('%s outside expression' % delimeters[1], + elif expr == delimiters[1] and not in_expr: + raise TemplateError('%s outside expression' % delimiters[1], position=pos, name=name) - if expr == delimeters[0]: + if expr == delimiters[0]: part = s[last:match.start()] if part: chunks.append(part) @@ -684,7 +684,7 @@ def lex(s, name=None, trim_whitespace=True, line_offset=0, delimeters=None): last = match.end() last_pos = pos if in_expr: - raise TemplateError('No %s to finish last expression' % delimeters[1], + raise TemplateError('No %s to finish last expression' % delimiters[1], name=name, position=last_pos) part = s[last:] if part: @@ -822,12 +822,12 @@ def find_position(string, index, last_index, last_pos): return (last_pos[0] + lines, column) -def parse(s, name=None, line_offset=0, delimeters=None): +def parse(s, name=None, line_offset=0, delimiters=None): - if delimeters is None: - delimeters = (Template.default_namespace['start_braces'], + if delimiters is None: + delimiters = (Template.default_namespace['start_braces'], Template.default_namespace['end_braces']) - tokens = lex(s, name=name, line_offset=line_offset, delimeters=delimeters) + tokens = lex(s, name=name, line_offset=line_offset, delimiters=delimiters) result = [] while tokens: next_chunk, tokens = parse_expr(tokens, name) diff --git a/tools/npy_tempita/compat3.py b/tools/npy_tempita/compat3.py index eb890ca14..01d771345 100644 --- a/tools/npy_tempita/compat3.py +++ b/tools/npy_tempita/compat3.py @@ -5,7 +5,7 @@ import sys __all__ = ['PY3', 'b', 'basestring_', 'bytes', 'next', 'is_unicode', 'iteritems'] -PY3 = True if sys.version_info[0] == 3 else False +PY3 = True if sys.version_info[0] >= 3 else False if sys.version_info[0] < 3: diff --git a/tools/openblas_support.py b/tools/openblas_support.py index 297dc6436..964adce6e 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -14,8 +14,8 @@ from tempfile import mkstemp, gettempdir import zipfile import tarfile -OPENBLAS_V = 'v0.3.5' -OPENBLAS_LONG = 'v0.3.5-274-g6a8b4269' +OPENBLAS_V = 'v0.3.7' +OPENBLAS_LONG = 'v0.3.7' BASE_LOC = '' RACKSPACE = 'https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com' ARCHITECTURES = ['', 'windows', 'darwin', 'arm', 'x86', 'ppc64'] @@ -47,7 +47,7 @@ def download_openblas(target, arch): # https://github.com/tylerjereddy/openblas-static-gcc/tree/master/ARMv8 # build done on GCC compile farm machine named gcc115 # tarball uploaded manually to an unshared Dropbox location - filename = ('https://www.dropbox.com/s/pbqkxzlmih4cky1/' + filename = ('https://www.dropbox.com/s/vdeckao4omss187/' 'openblas-{}-armv8.tar.gz?dl=1'.format(OPENBLAS_V)) typ = 'tar.gz' elif arch == 'ppc64': @@ -55,7 +55,7 @@ def download_openblas(target, arch): # https://github.com/tylerjereddy/openblas-static-gcc/blob/master/power8 # built on GCC compile farm machine named gcc112 # manually uploaded tarball to an unshared Dropbox location - filename = ('https://www.dropbox.com/s/zcwhk7c2zptwy0s/' + filename = ('https://www.dropbox.com/s/yt0d2j86x1j8nh1/' 'openblas-{}-ppc64le-power8.tar.gz?dl=1'.format(OPENBLAS_V)) typ = 'tar.gz' elif arch == 'darwin': diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh index 038748af9..f4d56ba1a 100755 --- a/tools/pypy-test.sh +++ b/tools/pypy-test.sh @@ -32,14 +32,14 @@ mkdir -p pypy3 (cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2) pypy3/bin/pypy3 -mensurepip pypy3/bin/pypy3 -m pip install --upgrade pip setuptools -pypy3/bin/pypy3 -m pip install --user cython==0.29.0 pytest pytz --no-warn-script-location +pypy3/bin/pypy3 -m pip install --user -r test_requirements.txt --no-warn-script-location echo echo pypy3 version pypy3/bin/pypy3 -c "import sys; print(sys.version)" echo -pypy3/bin/pypy3 runtests.py --show-build-log -- -rsx \ +pypy3/bin/pypy3 runtests.py --debug-info --show-build-log -v -- -rsx \ --junitxml=junit/test-results.xml --durations 10 echo Make sure the correct openblas has been linked in diff --git a/tools/refguide_check.py b/tools/refguide_check.py index c20807267..2c628091f 100644 --- a/tools/refguide_check.py +++ b/tools/refguide_check.py @@ -2,8 +2,10 @@ """ refguide_check.py [OPTIONS] [-- ARGS] -Check for a NumPy submodule whether the objects in its __all__ dict -correspond to the objects included in the reference guide. +- Check for a NumPy submodule whether the objects in its __all__ dict + correspond to the objects included in the reference guide. +- Check docstring examples +- Check example blocks in RST files Example of usage:: @@ -15,12 +17,13 @@ objects are left out of the refguide for a good reason (it's an alias of another function, or deprecated, or ...) Another use of this helper script is to check validity of code samples -in docstrings. This is different from doctesting [we do not aim to have -numpy docstrings doctestable!], this is just to make sure that code in -docstrings is valid python:: +in docstrings:: - $ python refguide_check.py --doctests optimize + $ python refguide_check.py --doctests ma +or in RST-based documentations:: + + $ python refguide_check.py --rst docs """ from __future__ import print_function @@ -47,9 +50,11 @@ import numpy as np sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'doc', 'sphinxext')) from numpydoc.docscrape_sphinx import get_doc_object +SKIPBLOCK = doctest.register_optionflag('SKIPBLOCK') + if parse_version(sphinx.__version__) >= parse_version('1.5'): # Enable specific Sphinx directives - from sphinx.directives import SeeAlso, Only + from sphinx.directives.other import SeeAlso, Only directives.register_directive('seealso', SeeAlso) directives.register_directive('only', Only) else: @@ -102,6 +107,21 @@ DOCTEST_SKIPLIST = set([ 'numpy.lib.Repository', ]) +# Skip non-numpy RST files, historical release notes +# Any single-directory exact match will skip the directory and all subdirs. +# Any exact match (like 'doc/release') will scan subdirs but skip files in +# the matched directory. +# Any filename will skip that file +RST_SKIPLIST = [ + 'scipy-sphinx-theme', + 'sphinxext', + 'neps', + 'changelog', + 'doc/release', + 'doc/source/release', + 'c-info.ufunc-tutorial.rst', +] + # these names are not required to be present in ALL despite being in # autosummary:: listing REFGUIDE_ALL_SKIPLIST = [ @@ -243,6 +263,7 @@ def compare(all_dict, others, names, module_name): return only_all, only_ref, missing + def is_deprecated(f): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("error") @@ -254,7 +275,12 @@ def is_deprecated(f): pass return False + def check_items(all_dict, names, deprecated, others, module_name, dots=True): + """ + Check that `all_dict` is consistent with the `names` in `module_name` + For instance, that there are no deprecated or extra objects. + """ num_all = len(all_dict) num_ref = len(names) @@ -450,6 +476,7 @@ DEFAULT_NAMESPACE = {'np': np} # the namespace to do checks in CHECK_NAMESPACE = { 'np': np, + 'numpy': np, 'assert_allclose': np.testing.assert_allclose, 'assert_equal': np.testing.assert_equal, # recognize numpy repr's @@ -465,7 +492,9 @@ CHECK_NAMESPACE = { 'nan': np.nan, 'NaN': np.nan, 'inf': np.inf, - 'Inf': np.inf,} + 'Inf': np.inf, + 'StringIO': io.StringIO, +} class DTRunner(doctest.DocTestRunner): @@ -517,7 +546,7 @@ class Checker(doctest.OutputChecker): self.parse_namedtuples = parse_namedtuples self.atol, self.rtol = atol, rtol if ns is None: - self.ns = dict(CHECK_NAMESPACE) + self.ns = CHECK_NAMESPACE else: self.ns = ns @@ -581,9 +610,9 @@ class Checker(doctest.OutputChecker): # and then compare the tuples. try: num = len(a_want) - regex = ('[\w\d_]+\(' + - ', '.join(['[\w\d_]+=(.+)']*num) + - '\)') + regex = (r'[\w\d_]+\(' + + ', '.join([r'[\w\d_]+=(.+)']*num) + + r'\)') grp = re.findall(regex, got.replace('\n', ' ')) if len(grp) > 1: # no more than one for now return False @@ -655,11 +684,21 @@ def _run_doctests(tests, full_name, verbose, doctest_warnings): # try to ensure random seed is NOT reproducible np.random.seed(None) + ns = {} for t in tests: + # We broke the tests up into chunks to try to avoid PSEUDOCODE + # This has the unfortunate side effect of restarting the global + # namespace for each test chunk, so variables will be "lost" after + # a chunk. Chain the globals to avoid this + t.globs.update(ns) t.filename = short_path(t.filename, cwd) - fails, successes = runner.run(t, out=out) + # Process our options + if any([SKIPBLOCK in ex.options for ex in t.examples]): + continue + fails, successes = runner.run(t, out=out, clear_globs=False) if fails > 0: success = False + ns = t.globs finally: sys.stderr = old_stderr os.chdir(cwd) @@ -757,10 +796,9 @@ def check_doctests_testfile(fname, verbose, ns=None, 5 """ - results = [] - if ns is None: - ns = dict(DEFAULT_NAMESPACE) + ns = CHECK_NAMESPACE + results = [] _, short_name = os.path.split(fname) if short_name in DOCTEST_SKIPLIST: @@ -782,20 +820,32 @@ def check_doctests_testfile(fname, verbose, ns=None, # split the text into "blocks" and try to detect and omit pseudocode blocks. parser = doctest.DocTestParser() good_parts = [] + base_line_no = 0 for part in text.split('\n\n'): - tests = parser.get_doctest(part, ns, fname, fname, 0) + try: + tests = parser.get_doctest(part, ns, fname, fname, base_line_no) + except ValueError as e: + if e.args[0].startswith('line '): + # fix line number since `parser.get_doctest` does not increment + # the reported line number by base_line_no in the error message + parts = e.args[0].split() + parts[1] = str(int(parts[1]) + base_line_no) + e.args = (' '.join(parts),) + e.args[1:] + raise if any(word in ex.source for word in PSEUDOCODE for ex in tests.examples): # omit it pass else: # `part` looks like a good code, let's doctest it - good_parts += [part] + good_parts.append((part, base_line_no)) + base_line_no += part.count('\n') + 2 # Reassemble the good bits and doctest them: - good_text = '\n\n'.join(good_parts) - tests = parser.get_doctest(good_text, ns, fname, fname, 0) - success, output = _run_doctests([tests], full_name, verbose, + tests = [] + for good_text, line_no in good_parts: + tests.append(parser.get_doctest(good_text, ns, fname, fname, line_no)) + success, output = _run_doctests(tests, full_name, verbose, doctest_warnings) if dots: @@ -810,6 +860,59 @@ def check_doctests_testfile(fname, verbose, ns=None, return results +def iter_included_files(base_path, verbose=0, suffixes=('.rst',)): + """ + Generator function to walk `base_path` and its subdirectories, skipping + files or directories in RST_SKIPLIST, and yield each file with a suffix in + `suffixes` + """ + if os.path.exists(base_path) and os.path.isfile(base_path): + yield base_path + for dir_name, subdirs, files in os.walk(base_path, topdown=True): + if dir_name in RST_SKIPLIST: + if verbose > 0: + sys.stderr.write('skipping files in %s' % dir_name) + files = [] + for p in RST_SKIPLIST: + if p in subdirs: + if verbose > 0: + sys.stderr.write('skipping %s and subdirs' % p) + subdirs.remove(p) + for f in files: + if (os.path.splitext(f)[1] in suffixes and + f not in RST_SKIPLIST): + yield os.path.join(dir_name, f) + + +def check_documentation(base_path, results, args, dots): + """ + Check examples in any *.rst located inside `base_path`. + Add the output to `results`. + + See Also + -------- + check_doctests_testfile + """ + for filename in iter_included_files(base_path, args.verbose): + if dots: + sys.stderr.write(filename + ' ') + sys.stderr.flush() + + tut_results = check_doctests_testfile( + filename, + (args.verbose >= 2), dots=dots, + doctest_warnings=args.doctest_warnings) + + # stub out a "module" which is needed when reporting the result + def scratch(): + pass + scratch.__name__ = filename + results.append((scratch, tut_results)) + if dots: + sys.stderr.write('\n') + sys.stderr.flush() + + def init_matplotlib(): global HAVE_MATPLOTLIB @@ -825,20 +928,21 @@ def main(argv): parser = ArgumentParser(usage=__doc__.lstrip()) parser.add_argument("module_names", metavar="SUBMODULES", default=[], nargs='*', help="Submodules to check (default: all public)") - parser.add_argument("--doctests", action="store_true", help="Run also doctests") + parser.add_argument("--doctests", action="store_true", + help="Run also doctests on ") parser.add_argument("-v", "--verbose", action="count", default=0) parser.add_argument("--doctest-warnings", action="store_true", help="Enforce warning checking for doctests") - parser.add_argument("--skip-tutorial", action="store_true", - help="Skip running doctests in the tutorial.") + parser.add_argument("--rst", nargs='?', const='doc', default=None, + help=("Run also examples from *rst files " + "discovered walking the directory(s) specified, " + "defaults to 'doc'")) args = parser.parse_args(argv) modules = [] names_dict = {} - if args.module_names: - args.skip_tutorial = True - else: + if not args.module_names: args.module_names = list(PUBLIC_SUBMODULES) os.environ['SCIPY_PIL_IMAGE_VIEWER'] = 'true' @@ -850,6 +954,15 @@ def main(argv): if name not in module_names: module_names.append(name) + dots = True + success = True + results = [] + errormsgs = [] + + + if args.doctests or args.rst: + init_matplotlib() + for submodule_name in module_names: module_name = BASE_MODULE + '.' + submodule_name __import__(module_name) @@ -861,69 +974,55 @@ def main(argv): if submodule_name in args.module_names: modules.append(module) - dots = True - success = True - results = [] - - print("Running checks for %d modules:" % (len(modules),)) - if args.doctests or not args.skip_tutorial: - init_matplotlib() - - for module in modules: - if dots: - if module is not modules[0]: - sys.stderr.write(' ') - sys.stderr.write(module.__name__ + ' ') - sys.stderr.flush() + if args.doctests or not args.rst: + print("Running checks for %d modules:" % (len(modules),)) + for module in modules: + if dots: + sys.stderr.write(module.__name__ + ' ') + sys.stderr.flush() - all_dict, deprecated, others = get_all_dict(module) - names = names_dict.get(module.__name__, set()) + all_dict, deprecated, others = get_all_dict(module) + names = names_dict.get(module.__name__, set()) - mod_results = [] - mod_results += check_items(all_dict, names, deprecated, others, module.__name__) - mod_results += check_rest(module, set(names).difference(deprecated), - dots=dots) - if args.doctests: - mod_results += check_doctests(module, (args.verbose >= 2), dots=dots, - doctest_warnings=args.doctest_warnings) + mod_results = [] + mod_results += check_items(all_dict, names, deprecated, others, + module.__name__) + mod_results += check_rest(module, set(names).difference(deprecated), + dots=dots) + if args.doctests: + mod_results += check_doctests(module, (args.verbose >= 2), dots=dots, + doctest_warnings=args.doctest_warnings) - for v in mod_results: - assert isinstance(v, tuple), v + for v in mod_results: + assert isinstance(v, tuple), v - results.append((module, mod_results)) + results.append((module, mod_results)) - if dots: - sys.stderr.write("\n") - sys.stderr.flush() - - if not args.skip_tutorial: - base_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..') - tut_path = os.path.join(base_dir, 'doc', 'source', 'tutorial', '*.rst') - print('\nChecking tutorial files at %s:' % os.path.relpath(tut_path, os.getcwd())) - for filename in sorted(glob.glob(tut_path)): if dots: sys.stderr.write('\n') - sys.stderr.write(os.path.split(filename)[1] + ' ') sys.stderr.flush() - tut_results = check_doctests_testfile(filename, (args.verbose >= 2), - dots=dots, doctest_warnings=args.doctest_warnings) - - def scratch(): pass # stub out a "module", see below - scratch.__name__ = filename - results.append((scratch, tut_results)) + all_dict, deprecated, others = get_all_dict(module) + if args.rst: + base_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..') + rst_path = os.path.relpath(os.path.join(base_dir, args.rst)) + if os.path.exists(rst_path): + print('\nChecking files in %s:' % rst_path) + check_documentation(rst_path, results, args, dots) + else: + sys.stderr.write(f'\ninvalid --rst argument "{args.rst}"') + errormsgs.append('invalid directory argument to --rst') if dots: sys.stderr.write("\n") sys.stderr.flush() # Report results - all_success = True - for module, mod_results in results: success = all(x[1] for x in mod_results) - all_success = all_success and success + if not success: + errormsgs.append(f'failed checking {module.__name__}') if success and args.verbose == 0: continue @@ -946,11 +1045,11 @@ def main(argv): print(output.strip()) print("") - if all_success: - print("\nOK: refguide and doctests checks passed!") + if len(errormsgs) == 0: + print("\nOK: all checks passed!") sys.exit(0) else: - print("\nERROR: refguide or doctests have errors") + print('\nERROR: ', '\n '.join(errormsgs)) sys.exit(1) diff --git a/tools/swig/test/testFarray.py b/tools/swig/test/testFarray.py index 0037dc9b3..e8bf711c5 100755 --- a/tools/swig/test/testFarray.py +++ b/tools/swig/test/testFarray.py @@ -15,7 +15,7 @@ else: BadListError = ValueError # Add the distutils-generated build directory to the python search path and then # import the extension module -libDir = "lib.%s-%s" % (get_platform(), sys.version[:3]) +libDir = "lib.{}-{}.{}".format(get_platform(), *sys.version_info[:2]) sys.path.insert(0, os.path.join("build", libDir)) import Farray diff --git a/tools/test-installed-numpy.py b/tools/test-installed-numpy.py deleted file mode 100755 index 5240253b6..000000000 --- a/tools/test-installed-numpy.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -from __future__ import division, absolute_import, print_function - -# A simple script to test the installed version of numpy by calling -# 'numpy.test()'. Key features: -# -- convenient command-line syntax -# -- sets exit status appropriately, useful for automated test environments - -# It would be better to set this up as a module in the numpy namespace, so -# that it could be run as: -# python -m numpy.run_tests <args> -# But, python2.4's -m switch only works with top-level modules, not modules -# that are inside packages. So, once we drop 2.4 support, maybe... - -import sys, os -# In case we are run from the source directory, we don't want to import numpy -# from there, we want to import the installed version: -sys.path.pop(0) - -from optparse import OptionParser -parser = OptionParser("usage: %prog [options] -- [nosetests options]") -parser.add_option("-v", "--verbose", - action="count", dest="verbose", default=1, - help="increase verbosity") -parser.add_option("--doctests", - action="store_true", dest="doctests", default=False, - help="Run doctests in module") -parser.add_option("--coverage", - action="store_true", dest="coverage", default=False, - help="report coverage of NumPy code (requires 'pytest-cov' module") -parser.add_option("-m", "--mode", - action="store", dest="mode", default="fast", - help="'fast', 'full', or something that could be " - "passed to pytest [default: %default]") -parser.add_option("-n", "--durations", - dest="durations", default=-1, - help="show time to run slowest N tests [default: -1]") -(options, args) = parser.parse_args() - -import numpy - -# Check that NPY_RELAXED_STRIDES_CHECKING is active when set. -# The same flags check is also used in the tests to switch behavior. -if (os.environ.get('NPY_RELAXED_STRIDES_CHECKING', "1") != "0"): - if not numpy.ones((10, 1), order='C').flags.f_contiguous: - print('NPY_RELAXED_STRIDES_CHECKING set, but not active.') - sys.exit(1) -elif numpy.ones((10, 1), order='C').flags.f_contiguous: - print('NPY_RELAXED_STRIDES_CHECKING not set, but active.') - sys.exit(1) - -if options.coverage: - # Produce code coverage XML report for codecov.io - args += ["--cov-report=xml"] - -result = numpy.test(options.mode, - verbose=options.verbose, - extra_argv=args, - doctests=options.doctests, - durations=int(options.durations), - coverage=options.coverage) - -if result: - sys.exit(0) -else: - sys.exit(1) diff --git a/tools/travis-before-install.sh b/tools/travis-before-install.sh index b2882df69..072ad3bf6 100755 --- a/tools/travis-before-install.sh +++ b/tools/travis-before-install.sh @@ -30,12 +30,8 @@ fi source venv/bin/activate python -V -if [ -n "$INSTALL_PICKLE5" ]; then - pip install pickle5 -fi - +popd pip install --upgrade pip setuptools -pip install pytz cython pytest==5.1.2 +pip install -r test_requirements.txt if [ -n "$USE_ASV" ]; then pip install asv; fi -popd diff --git a/tools/travis-test.sh b/tools/travis-test.sh index e3ea8a5d7..241b9d913 100755 --- a/tools/travis-test.sh +++ b/tools/travis-test.sh @@ -32,12 +32,10 @@ werrors="$werrors -Werror=implicit-function-declaration" setup_base() { - # use default python flags but remoge sign-compare + # use default python flags but remove sign-compare sysflags="$($PYTHON -c "from distutils import sysconfig; \ print (sysconfig.get_config_var('CFLAGS'))")" export CFLAGS="$sysflags $werrors -Wlogical-op -Wno-sign-compare" - # use c99 - export CFLAGS=$CFLAGS" -std=c99" # We used to use 'setup.py install' here, but that has the terrible # behaviour that if a copy of the package is already installed in the # install location, then the new copy just gets dropped on top of it. @@ -52,10 +50,12 @@ setup_base() else # Python3.5-dbg on travis seems to need this export CFLAGS=$CFLAGS" -Wno-maybe-uninitialized" - $PYTHON setup.py build_ext --inplace 2>&1 | tee log + $PYTHON setup.py build build_src --verbose-cfg build_ext --inplace 2>&1 | tee log fi grep -v "_configtest" log \ - | grep -vE "ld returned 1|no previously-included files matching|manifest_maker: standard file '-c'" \ + | grep -vE "ld returned 1|no files found matching" \ + | grep -vE "no previously-included files matching" \ + | grep -vE "manifest_maker: standard file '-c'" \ | grep -E "warning\>" \ | tee warnings if [ "$LAPACK" != "None" ]; then @@ -65,12 +65,20 @@ setup_base() run_test() { + $PIP install -r test_requirements.txt + if [ -n "$USE_DEBUG" ]; then export PYTHONPATH=$PWD fi + # pytest aborts when running --durations with python3.6-dbg, so only enable + # it for non-debug tests. That is a cPython bug fixed in later versions of + # python3.7 but python3.7-dbg is not currently available on travisCI. + if [ -z "$USE_DEBUG" ]; then + DURATIONS_FLAG="--durations 10" + fi + if [ -n "$RUN_COVERAGE" ]; then - $PIP install pytest-cov COVERAGE_FLAG=--coverage fi @@ -82,15 +90,15 @@ run_test() "import os; import numpy; print(os.path.dirname(numpy.__file__))") export PYTHONWARNINGS=default - if [ -n "$PPC64_LE" ]; then + if [ -n "$CHECK_BLAS" ]; then $PYTHON ../tools/openblas_support.py --check_version $OpenBLAS_version fi if [ -n "$RUN_FULL_TESTS" ]; then export PYTHONWARNINGS="ignore::DeprecationWarning:virtualenv" - $PYTHON ../tools/test-installed-numpy.py -v --durations 10 --mode=full $COVERAGE_FLAG + $PYTHON -b ../runtests.py -n -v --mode=full $DURATIONS_FLAG $COVERAGE_FLAG else - $PYTHON ../tools/test-installed-numpy.py -v --durations 10 + $PYTHON ../runtests.py -n -v $DURATIONS_FLAG fi if [ -n "$RUN_COVERAGE" ]; then @@ -139,8 +147,6 @@ if [ -n "$USE_WHEEL" ] && [ $# -eq 0 ]; then $PIP install -U virtualenv # ensure some warnings are not issued export CFLAGS=$CFLAGS" -Wno-sign-compare -Wno-unused-result" - # use c99 - export CFLAGS=$CFLAGS" -std=c99" # adjust gcc flags if C coverage requested if [ -n "$RUN_COVERAGE" ]; then export NPY_DISTUTILS_APPEND_FLAGS=1 @@ -149,21 +155,17 @@ if [ -n "$USE_WHEEL" ] && [ $# -eq 0 ]; then export F90='gfortran --coverage' export LDFLAGS='--coverage' fi - $PYTHON setup.py bdist_wheel + $PYTHON setup.py build build_src --verbose-cfg bdist_wheel # Make another virtualenv to install into virtualenv --python=`which $PYTHON` venv-for-wheel . venv-for-wheel/bin/activate # Move out of source directory to avoid finding local numpy pushd dist $PIP install --pre --no-index --upgrade --find-links=. numpy - $PIP install nose pytest - - if [ -n "$INSTALL_PICKLE5" ]; then - $PIP install pickle5 - fi - popd + run_test + elif [ -n "$USE_SDIST" ] && [ $# -eq 0 ]; then # use an up-to-date pip / setuptools inside the venv $PIP install -U virtualenv @@ -171,8 +173,6 @@ elif [ -n "$USE_SDIST" ] && [ $# -eq 0 ]; then $PYTHON -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)" # ensure some warnings are not issued export CFLAGS=$CFLAGS" -Wno-sign-compare -Wno-unused-result" - # use c99 - export CFLAGS=$CFLAGS" -std=c99" $PYTHON setup.py sdist # Make another virtualenv to install into virtualenv --python=`which $PYTHON` venv-for-wheel @@ -180,11 +180,6 @@ elif [ -n "$USE_SDIST" ] && [ $# -eq 0 ]; then # Move out of source directory to avoid finding local numpy pushd dist $PIP install numpy* - $PIP install nose pytest - if [ -n "$INSTALL_PICKLE5" ]; then - $PIP install pickle5 - fi - popd run_test else @@ -15,7 +15,7 @@ # To run against a specific subset of Python versions, use: # tox -e py37 -# Extra arguments will be passed to test-installed-numpy.py. To run +# Extra arguments will be passed to runtests.py. To run # the full testsuite: # tox full # To run with extra verbosity: @@ -30,10 +30,9 @@ envlist = py37-not-relaxed-strides [testenv] -deps= - pytest +deps= -Ur{toxinidir}/test_requirements.txt changedir={envdir} -commands={envpython} {toxinidir}/tools/test-installed-numpy.py --mode=full {posargs:} +commands={envpython} -b {toxinidir}/runtests.py --mode=full {posargs:} [testenv:py37-not-relaxed-strides] basepython=python3.7 @@ -43,4 +42,4 @@ env=NPY_RELAXED_STRIDES_CHECKING=0 # if you want it: [testenv:debug] basepython=python-dbg -commands=gdb --args {envpython} {toxinidir}/tools/test-installed-numpy.py --mode=full {posargs:} +commands=gdb --args {envpython} {toxinidir}/runtests.py --mode=full {posargs:} |
