diff options
Diffstat (limited to 'numpy/distutils')
-rw-r--r-- | numpy/distutils/__init__.py | 27 | ||||
-rw-r--r-- | numpy/distutils/__version__.py | 6 | ||||
-rw-r--r-- | numpy/distutils/ccompiler.py | 18 | ||||
-rw-r--r-- | numpy/distutils/command/build.py | 13 | ||||
-rw-r--r-- | numpy/distutils/command/build_clib.py | 14 | ||||
-rw-r--r-- | numpy/distutils/command/build_ext.py | 15 | ||||
-rw-r--r-- | numpy/distutils/command/build_src.py | 21 | ||||
-rw-r--r-- | numpy/distutils/command/config.py | 8 | ||||
-rw-r--r-- | numpy/distutils/cpuinfo.py | 28 | ||||
-rw-r--r-- | numpy/distutils/exec_command.py | 2 | ||||
-rw-r--r-- | numpy/distutils/extension.py | 18 | ||||
-rw-r--r-- | numpy/distutils/fcompiler/compaq.py | 2 | ||||
-rw-r--r-- | numpy/distutils/fcompiler/environment.py | 12 | ||||
-rw-r--r-- | numpy/distutils/info.py | 6 | ||||
-rw-r--r-- | numpy/distutils/log.py | 2 | ||||
-rw-r--r-- | numpy/distutils/misc_util.py | 52 | ||||
-rw-r--r-- | numpy/distutils/npy_pkg_config.py | 2 | ||||
-rw-r--r-- | numpy/distutils/system_info.py | 250 | ||||
-rw-r--r-- | numpy/distutils/tests/test_fcompiler.py | 34 |
19 files changed, 389 insertions, 141 deletions
diff --git a/numpy/distutils/__init__.py b/numpy/distutils/__init__.py index 55514750e..8dbb63b28 100644 --- a/numpy/distutils/__init__.py +++ b/numpy/distutils/__init__.py @@ -1,12 +1,31 @@ +""" +An enhanced distutils, providing support for Fortran compilers, for BLAS, +LAPACK and other common libraries for numerical computing, and more. + +Public submodules are:: + + misc_util + system_info + cpu_info + log + exec_command + +For details, please see the *Packaging* and *NumPy Distutils User Guide* +sections of the NumPy Reference Guide. + +For configuring the preference for and location of libraries like BLAS and +LAPACK, and for setting include paths and similar build options, please see +``site.cfg.example`` in the root of the NumPy repository or sdist. + +""" + from __future__ import division, absolute_import, print_function -from .__version__ import version as __version__ # Must import local ccompiler ASAP in order to get # customized CCompiler.spawn effective. from . import ccompiler from . import unixccompiler -from .info import __doc__ from .npy_pkg_config import * # If numpy is installed, add distutils.test() @@ -28,7 +47,7 @@ def customized_fcompiler(plat=None, compiler=None): c.customize() return c -def customized_ccompiler(plat=None, compiler=None): - c = ccompiler.new_compiler(plat=plat, compiler=compiler) +def customized_ccompiler(plat=None, compiler=None, verbose=1): + c = ccompiler.new_compiler(plat=plat, compiler=compiler, verbose=verbose) c.customize('') return c diff --git a/numpy/distutils/__version__.py b/numpy/distutils/__version__.py deleted file mode 100644 index 969decbba..000000000 --- a/numpy/distutils/__version__.py +++ /dev/null @@ -1,6 +0,0 @@ -from __future__ import division, absolute_import, print_function - -major = 0 -minor = 4 -micro = 0 -version = '%(major)d.%(minor)d.%(micro)d' % (locals()) diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py index 14451fa66..684c7535b 100644 --- a/numpy/distutils/ccompiler.py +++ b/numpy/distutils/ccompiler.py @@ -140,7 +140,10 @@ def CCompiler_spawn(self, cmd, display=None): display = ' '.join(list(display)) log.info(display) try: - subprocess.check_output(cmd) + if self.verbose: + subprocess.check_output(cmd) + else: + subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: o = exc.output s = exc.returncode @@ -162,7 +165,8 @@ def CCompiler_spawn(self, cmd, display=None): if is_sequence(cmd): cmd = ' '.join(list(cmd)) - forward_bytes_to_stdout(o) + if self.verbose: + forward_bytes_to_stdout(o) if re.search(b'Too many open files', o): msg = '\nTry rerunning setup command until build succeeds.' @@ -528,6 +532,11 @@ def CCompiler_customize(self, dist, need_cxx=0): 'g++' in self.compiler[0] or 'clang' in self.compiler[0]): self._auto_depends = True + if 'gcc' in self.compiler[0]: + # add std=c99 flag for gcc + # TODO: does this need to be more specific? + self.compiler.append('-std=c99') + self.compiler_so.append('-std=c99') elif os.name == 'posix': import tempfile import shutil @@ -727,10 +736,12 @@ if sys.platform == 'win32': _distutils_new_compiler = new_compiler def new_compiler (plat=None, compiler=None, - verbose=0, + verbose=None, dry_run=0, force=0): # Try first C compilers from numpy.distutils. + if verbose is None: + verbose = log.get_threshold() <= log.INFO if plat is None: plat = os.name try: @@ -763,6 +774,7 @@ def new_compiler (plat=None, raise DistutilsModuleError(("can't compile C/C++ code: unable to find class '%s' " + "in module '%s'") % (class_name, module_name)) compiler = klass(None, dry_run, force) + compiler.verbose = verbose log.debug('new_compiler returns %s' % (klass)) return compiler diff --git a/numpy/distutils/command/build.py b/numpy/distutils/command/build.py index 3d7101582..5a9da1217 100644 --- a/numpy/distutils/command/build.py +++ b/numpy/distutils/command/build.py @@ -16,8 +16,8 @@ class build(old_build): user_options = old_build.user_options + [ ('fcompiler=', None, "specify the Fortran compiler type"), - ('parallel=', 'j', - "number of parallel jobs"), + ('warn-error', None, + "turn all warnings into errors (-Werror)"), ] help_options = old_build.help_options + [ @@ -28,17 +28,12 @@ class build(old_build): def initialize_options(self): old_build.initialize_options(self) self.fcompiler = None - self.parallel = None + self.warn_error = False def finalize_options(self): - if self.parallel: - try: - self.parallel = int(self.parallel) - except ValueError: - raise ValueError("--parallel/-j argument must be an integer") build_scripts = self.build_scripts old_build.finalize_options(self) - plat_specifier = ".%s-%s" % (get_platform(), sys.version[0:3]) + plat_specifier = ".{}-{}.{}".format(get_platform(), *sys.version_info[:2]) if build_scripts is None: self.build_scripts = os.path.join(self.build_base, 'scripts' + plat_specifier) diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py index 910493a77..13edf0717 100644 --- a/numpy/distutils/command/build_clib.py +++ b/numpy/distutils/command/build_clib.py @@ -33,15 +33,18 @@ class build_clib(old_build_clib): ('inplace', 'i', 'Build in-place'), ('parallel=', 'j', "number of parallel jobs"), + ('warn-error', None, + "turn all warnings into errors (-Werror)"), ] - boolean_options = old_build_clib.boolean_options + ['inplace'] + boolean_options = old_build_clib.boolean_options + ['inplace', 'warn-error'] def initialize_options(self): old_build_clib.initialize_options(self) self.fcompiler = None self.inplace = 0 self.parallel = None + self.warn_error = None def finalize_options(self): if self.parallel: @@ -50,7 +53,10 @@ class build_clib(old_build_clib): except ValueError: raise ValueError("--parallel/-j argument must be an integer") old_build_clib.finalize_options(self) - self.set_undefined_options('build', ('parallel', 'parallel')) + self.set_undefined_options('build', + ('parallel', 'parallel'), + ('warn_error', 'warn_error'), + ) def have_f_sources(self): for (lib_name, build_info) in self.libraries: @@ -86,6 +92,10 @@ class build_clib(old_build_clib): self.compiler.customize(self.distribution, need_cxx=self.have_cxx_sources()) + if self.warn_error: + self.compiler.compiler.append('-Werror') + self.compiler.compiler_so.append('-Werror') + libraries = self.libraries self.libraries = None self.compiler.customize_cmd(self) diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index ef54fb25e..cd9b1c6f1 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -33,6 +33,8 @@ class build_ext (old_build_ext): "specify the Fortran compiler type"), ('parallel=', 'j', "number of parallel jobs"), + ('warn-error', None, + "turn all warnings into errors (-Werror)"), ] help_options = old_build_ext.help_options + [ @@ -40,10 +42,13 @@ class build_ext (old_build_ext): show_fortran_compilers), ] + boolean_options = old_build_ext.boolean_options + ['warn-error'] + def initialize_options(self): old_build_ext.initialize_options(self) self.fcompiler = None self.parallel = None + self.warn_error = None def finalize_options(self): if self.parallel: @@ -69,7 +74,10 @@ class build_ext (old_build_ext): self.include_dirs.extend(incl_dirs) old_build_ext.finalize_options(self) - self.set_undefined_options('build', ('parallel', 'parallel')) + self.set_undefined_options('build', + ('parallel', 'parallel'), + ('warn_error', 'warn_error'), + ) def run(self): if not self.extensions: @@ -116,6 +124,11 @@ class build_ext (old_build_ext): force=self.force) self.compiler.customize(self.distribution) self.compiler.customize_cmd(self) + + if self.warn_error: + self.compiler.compiler.append('-Werror') + self.compiler.compiler_so.append('-Werror') + self.compiler.show_customization() # Setup directory for storing generated extra DLL files on Windows diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index 6d5d305d2..3e0522c5f 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -53,9 +53,12 @@ class build_src(build_ext.build_ext): ('inplace', 'i', "ignore build-lib and put compiled extensions into the source " + "directory alongside your pure Python modules"), + ('verbose-cfg', None, + "change logging level from WARN to INFO which will show all " + + "compiler output") ] - boolean_options = ['force', 'inplace'] + boolean_options = ['force', 'inplace', 'verbose-cfg'] help_options = [] @@ -76,6 +79,7 @@ class build_src(build_ext.build_ext): self.swig_opts = None self.swig_cpp = None self.swig = None + self.verbose_cfg = None def finalize_options(self): self.set_undefined_options('build', @@ -90,7 +94,7 @@ class build_src(build_ext.build_ext): self.data_files = self.distribution.data_files or [] if self.build_src is None: - plat_specifier = ".%s-%s" % (get_platform(), sys.version[0:3]) + plat_specifier = ".{}-{}.{}".format(get_platform(), *sys.version_info[:2]) self.build_src = os.path.join(self.build_base, 'src'+plat_specifier) # py_modules_dict is used in build_py.find_package_modules @@ -362,9 +366,16 @@ class build_src(build_ext.build_ext): # incl_dirs = extension.include_dirs #if self.build_src not in incl_dirs: # incl_dirs.append(self.build_src) - build_dir = os.path.join(*([self.build_src]\ + build_dir = os.path.join(*([self.build_src] +name.split('.')[:-1])) self.mkpath(build_dir) + + if self.verbose_cfg: + new_level = log.INFO + else: + new_level = log.WARN + old_level = log.set_threshold(new_level) + for func in func_sources: source = func(extension, build_dir) if not source: @@ -375,7 +386,7 @@ class build_src(build_ext.build_ext): else: log.info(" adding '%s' to sources." % (source,)) new_sources.append(source) - + log.set_threshold(old_level) return new_sources def filter_py_files(self, sources): @@ -540,7 +551,7 @@ class build_src(build_ext.build_ext): if is_sequence(extension): name = extension[0] else: name = extension.name - target_dir = os.path.join(*([self.build_src]\ + target_dir = os.path.join(*([self.build_src] +name.split('.')[:-1])) target_file = os.path.join(target_dir, ext_name + 'module.c') new_sources.append(target_file) diff --git a/numpy/distutils/command/config.py b/numpy/distutils/command/config.py index e9ca7bea8..b9f2fa76e 100644 --- a/numpy/distutils/command/config.py +++ b/numpy/distutils/command/config.py @@ -442,10 +442,10 @@ class config(old_config): of the program and its output. """ # 2008-11-16, RemoveMe - warnings.warn("\n+++++++++++++++++++++++++++++++++++++++++++++++++\n" \ - "Usage of get_output is deprecated: please do not \n" \ - "use it anymore, and avoid configuration checks \n" \ - "involving running executable on the target machine.\n" \ + warnings.warn("\n+++++++++++++++++++++++++++++++++++++++++++++++++\n" + "Usage of get_output is deprecated: please do not \n" + "use it anymore, and avoid configuration checks \n" + "involving running executable on the target machine.\n" "+++++++++++++++++++++++++++++++++++++++++++++++++\n", DeprecationWarning, stacklevel=2) self._check_compiler() diff --git a/numpy/distutils/cpuinfo.py b/numpy/distutils/cpuinfo.py index 580299347..bc9728335 100644 --- a/numpy/distutils/cpuinfo.py +++ b/numpy/distutils/cpuinfo.py @@ -242,16 +242,16 @@ class LinuxCPUInfo(CPUInfoBase): return self.is_PentiumIV() and self.has_sse3() def _is_Nocona(self): - return self.is_Intel() \ - and (self.info[0]['cpu family'] == '6' \ - or self.info[0]['cpu family'] == '15' ) \ - and (self.has_sse3() and not self.has_ssse3())\ - and re.match(r'.*?\blm\b', self.info[0]['flags']) is not None + return (self.is_Intel() + and (self.info[0]['cpu family'] == '6' + or self.info[0]['cpu family'] == '15') + and (self.has_sse3() and not self.has_ssse3()) + and re.match(r'.*?\blm\b', self.info[0]['flags']) is not None) def _is_Core2(self): - return self.is_64bit() and self.is_Intel() and \ - re.match(r'.*?Core\(TM\)2\b', \ - self.info[0]['model name']) is not None + return (self.is_64bit() and self.is_Intel() and + re.match(r'.*?Core\(TM\)2\b', + self.info[0]['model name']) is not None) def _is_Itanium(self): return re.match(r'.*?Itanium\b', @@ -632,13 +632,13 @@ class Win32CPUInfo(CPUInfoBase): def _has_sse(self): if self.is_Intel(): - return (self.info[0]['Family']==6 and \ - self.info[0]['Model'] in [7, 8, 9, 10, 11]) \ - or self.info[0]['Family']==15 + return ((self.info[0]['Family']==6 and + self.info[0]['Model'] in [7, 8, 9, 10, 11]) + or self.info[0]['Family']==15) elif self.is_AMD(): - return (self.info[0]['Family']==6 and \ - self.info[0]['Model'] in [6, 7, 8, 10]) \ - or self.info[0]['Family']==15 + return ((self.info[0]['Family']==6 and + self.info[0]['Model'] in [6, 7, 8, 10]) + or self.info[0]['Family']==15) else: return False diff --git a/numpy/distutils/exec_command.py b/numpy/distutils/exec_command.py index 2e7b9e463..712f22666 100644 --- a/numpy/distutils/exec_command.py +++ b/numpy/distutils/exec_command.py @@ -217,7 +217,7 @@ def exec_command(command, execute_in='', use_shell=None, use_tee=None, # 2019-01-30, 1.17 warnings.warn('exec_command is deprecated since NumPy v1.17, use ' 'subprocess.Popen instead', DeprecationWarning, stacklevel=1) - log.debug('exec_command(%r,%s)' % (command,\ + log.debug('exec_command(%r,%s)' % (command, ','.join(['%s=%r'%kv for kv in env.items()]))) if use_tee is None: diff --git a/numpy/distutils/extension.py b/numpy/distutils/extension.py index 935f3eec9..872bd5362 100644 --- a/numpy/distutils/extension.py +++ b/numpy/distutils/extension.py @@ -19,8 +19,24 @@ if sys.version_info[0] >= 3: cxx_ext_re = re.compile(r'.*[.](cpp|cxx|cc)\Z', re.I).match fortran_pyf_ext_re = re.compile(r'.*[.](f90|f95|f77|for|ftn|f|pyf)\Z', re.I).match + class Extension(old_Extension): - def __init__ ( + """ + Parameters + ---------- + name : str + Extension name. + sources : list of str + List of source file locations relative to the top directory of + the package. + extra_compile_args : list of str + Extra command line arguments to pass to the compiler. + extra_f77_compile_args : list of str + Extra command line arguments to pass to the fortran77 compiler. + extra_f90_compile_args : list of str + Extra command line arguments to pass to the fortran90 compiler. + """ + def __init__( self, name, sources, include_dirs=None, define_macros=None, diff --git a/numpy/distutils/fcompiler/compaq.py b/numpy/distutils/fcompiler/compaq.py index 07d502706..671b3a55f 100644 --- a/numpy/distutils/fcompiler/compaq.py +++ b/numpy/distutils/fcompiler/compaq.py @@ -95,7 +95,7 @@ class CompaqVisualFCompiler(FCompiler): raise e except ValueError: e = get_exception() - if not "path']" in str(e): + if not "'path'" in str(e): print("Unexpected ValueError in", __file__) raise e diff --git a/numpy/distutils/fcompiler/environment.py b/numpy/distutils/fcompiler/environment.py index 73a5e98e1..bb362d483 100644 --- a/numpy/distutils/fcompiler/environment.py +++ b/numpy/distutils/fcompiler/environment.py @@ -59,17 +59,13 @@ class EnvironmentConfig(object): if envvar_contents is not None: envvar_contents = convert(envvar_contents) if var and append: - if os.environ.get('NPY_DISTUTILS_APPEND_FLAGS', '0') == '1': + if os.environ.get('NPY_DISTUTILS_APPEND_FLAGS', '1') == '1': var.extend(envvar_contents) else: + # NPY_DISTUTILS_APPEND_FLAGS was explicitly set to 0 + # to keep old (overwrite flags rather than append to + # them) behavior var = envvar_contents - if 'NPY_DISTUTILS_APPEND_FLAGS' not in os.environ.keys(): - msg = "{} is used as is, not appended ".format(envvar) + \ - "to flags already defined " + \ - "by numpy.distutils! Use NPY_DISTUTILS_APPEND_FLAGS=1 " + \ - "to obtain appending behavior instead (this " + \ - "behavior will become default in a future release)." - warnings.warn(msg, UserWarning, stacklevel=3) else: var = envvar_contents if confvar is not None and self._conf: diff --git a/numpy/distutils/info.py b/numpy/distutils/info.py deleted file mode 100644 index 2f5310665..000000000 --- a/numpy/distutils/info.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Enhanced distutils with Fortran compilers support and more. -""" -from __future__ import division, absolute_import, print_function - -postpone_import = True diff --git a/numpy/distutils/log.py b/numpy/distutils/log.py index 37f9fe5dd..ff7de86b1 100644 --- a/numpy/distutils/log.py +++ b/numpy/distutils/log.py @@ -67,6 +67,8 @@ def set_threshold(level, force=False): ' %s to %s' % (prev_level, level)) return prev_level +def get_threshold(): + return _global_log.threshold def set_verbosity(v, force=False): prev_level = _global_log.threshold diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index 2a6472b95..d46ff8981 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -859,7 +859,7 @@ class Configuration(object): print(message) def warn(self, message): - sys.stderr.write('Warning: %s' % (message,)) + sys.stderr.write('Warning: %s\n' % (message,)) def set_options(self, **options): """ @@ -1687,6 +1687,41 @@ class Configuration(object): and will be installed as foo.ini in the 'lib' subpath. + When cross-compiling with numpy distutils, it might be necessary to + use modified npy-pkg-config files. Using the default/generated files + will link with the host libraries (i.e. libnpymath.a). For + cross-compilation you of-course need to link with target libraries, + while using the host Python installation. + + You can copy out the numpy/core/lib/npy-pkg-config directory, add a + pkgdir value to the .ini files and set NPY_PKG_CONFIG_PATH environment + variable to point to the directory with the modified npy-pkg-config + files. + + Example npymath.ini modified for cross-compilation:: + + [meta] + Name=npymath + Description=Portable, core math library implementing C99 standard + Version=0.1 + + [variables] + pkgname=numpy.core + pkgdir=/build/arm-linux-gnueabi/sysroot/usr/lib/python3.7/site-packages/numpy/core + prefix=${pkgdir} + libdir=${prefix}/lib + includedir=${prefix}/include + + [default] + Libs=-L${libdir} -lnpymath + Cflags=-I${includedir} + Requires=mlib + + [msvc] + Libs=/LIBPATH:${libdir} npymath.lib + Cflags=/INCLUDE:${includedir} + Requires=mlib + """ if subst_dict is None: subst_dict = {} @@ -2092,9 +2127,22 @@ def get_numpy_include_dirs(): return include_dirs def get_npy_pkg_dir(): - """Return the path where to find the npy-pkg-config directory.""" + """Return the path where to find the npy-pkg-config directory. + + If the NPY_PKG_CONFIG_PATH environment variable is set, the value of that + is returned. Otherwise, a path inside the location of the numpy module is + returned. + + The NPY_PKG_CONFIG_PATH can be useful when cross-compiling, maintaining + customized npy-pkg-config .ini files for the cross-compilation + environment, and using them when cross-compiling. + + """ # XXX: import here for bootstrapping reasons import numpy + d = os.environ.get('NPY_PKG_CONFIG_PATH') + if d is not None: + return d d = os.path.join(os.path.dirname(numpy.__file__), 'core', 'lib', 'npy-pkg-config') return d diff --git a/numpy/distutils/npy_pkg_config.py b/numpy/distutils/npy_pkg_config.py index bfe8b9f77..48584b4c4 100644 --- a/numpy/distutils/npy_pkg_config.py +++ b/numpy/distutils/npy_pkg_config.py @@ -426,7 +426,7 @@ if __name__ == '__main__': if options.define_variable: m = re.search(r'([\S]+)=([\S]+)', options.define_variable) if not m: - raise ValueError("--define-variable option should be of " \ + raise ValueError("--define-variable option should be of " "the form --define-variable=foo=bar") else: name = m.group(1) diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index ba2b1f46c..4786b3a0c 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -21,9 +21,17 @@ classes are available: blas_info lapack_info openblas_info + openblas64__info + openblas_ilp64_info blis_info blas_opt_info # usage recommended lapack_opt_info # usage recommended + blas_ilp64_opt_info # usage recommended (general ILP64 BLAS) + lapack_ilp64_opt_info # usage recommended (general ILP64 LAPACK) + blas_ilp64_plain_opt_info # usage recommended (general ILP64 BLAS, no symbol suffix) + lapack_ilp64_plain_opt_info # usage recommended (general ILP64 LAPACK, no symbol suffix) + blas64__opt_info # usage recommended (general ILP64 BLAS, 64_ symbol suffix) + lapack64__opt_info # usage recommended (general ILP64 LAPACK, 64_ symbol suffix) fftw_info,dfftw_info,sfftw_info fftw_threads_info,dfftw_threads_info,sfftw_threads_info djbfft_info @@ -146,7 +154,7 @@ else: from distutils.errors import DistutilsError from distutils.dist import Distribution import distutils.sysconfig -from distutils import log +from numpy.distutils import log from distutils.util import get_platform from numpy.distutils.exec_command import ( @@ -156,7 +164,7 @@ from numpy.distutils.misc_util import (is_sequence, is_string, get_shared_lib_extension) from numpy.distutils.command.config import config as cmd_config from numpy.distutils.compat import get_exception -from numpy.distutils import customized_ccompiler +from numpy.distutils import customized_ccompiler as _customized_ccompiler from numpy.distutils import _shell_utils import distutils.ccompiler import tempfile @@ -169,6 +177,15 @@ _bits = {'32bit': 32, '64bit': 64} platform_bits = _bits[platform.architecture()[0]] +global_compiler = None + +def customized_ccompiler(): + global global_compiler + if not global_compiler: + global_compiler = _customized_ccompiler() + return global_compiler + + def _c_string_literal(s): """ Convert a python string into a literal suitable for inclusion into C code @@ -397,6 +414,10 @@ def get_info(name, notfound_action=0): 'lapack_mkl': lapack_mkl_info, # use lapack_opt instead 'blas_mkl': blas_mkl_info, # use blas_opt instead 'accelerate': accelerate_info, # use blas_opt instead + 'openblas64_': openblas64__info, + 'openblas64__lapack': openblas64__lapack_info, + 'openblas_ilp64': openblas_ilp64_info, + 'openblas_ilp64_lapack': openblas_ilp64_lapack_info, 'x11': x11_info, 'fft_opt': fft_opt_info, 'fftw': fftw_info, @@ -419,7 +440,13 @@ def get_info(name, notfound_action=0): 'numarray': numarray_info, 'numerix': numerix_info, 'lapack_opt': lapack_opt_info, + 'lapack_ilp64_opt': lapack_ilp64_opt_info, + 'lapack_ilp64_plain_opt': lapack_ilp64_plain_opt_info, + 'lapack64__opt': lapack64__opt_info, 'blas_opt': blas_opt_info, + 'blas_ilp64_opt': blas_ilp64_opt_info, + 'blas_ilp64_plain_opt': blas_ilp64_plain_opt_info, + 'blas64__opt': blas64__opt_info, 'boost_python': boost_python_info, 'agg2': agg2_info, 'wx': wx_info, @@ -456,7 +483,7 @@ class AliasedOptionError(DistutilsError): class AtlasNotFoundError(NotFoundError): """ - Atlas (http://math-atlas.sourceforge.net/) libraries not found. + Atlas (http://github.com/math-atlas/math-atlas) libraries not found. Directories to search for the libraries can be specified in the numpy/distutils/site.cfg file (section [atlas]) or by setting the ATLAS environment variable.""" @@ -485,6 +512,13 @@ class LapackSrcNotFoundError(LapackNotFoundError): the LAPACK_SRC environment variable.""" +class LapackILP64NotFoundError(NotFoundError): + """ + 64-bit Lapack libraries not found. + Known libraries in numpy/distutils/site.cfg file are: + openblas64_, openblas_ilp64 + """ + class BlasOptNotFoundError(NotFoundError): """ Optimized (vendor) Blas libraries are not found. @@ -499,6 +533,12 @@ class BlasNotFoundError(NotFoundError): numpy/distutils/site.cfg file (section [blas]) or by setting the BLAS environment variable.""" +class BlasILP64NotFoundError(NotFoundError): + """ + 64-bit Blas libraries not found. + Known libraries in numpy/distutils/site.cfg file are: + openblas64_, openblas_ilp64 + """ class BlasSrcNotFoundError(BlasNotFoundError): """ @@ -550,7 +590,6 @@ class system_info(object): dir_env_var = None search_static_first = 0 # XXX: disabled by default, may disappear in # future unless it is proved to be useful. - verbosity = 1 saved_results = {} notfounderror = NotFoundError @@ -558,7 +597,6 @@ class system_info(object): def __init__(self, default_lib_dirs=default_lib_dirs, default_include_dirs=default_include_dirs, - verbosity=1, ): self.__class__.info = {} self.local_prefixes = [] @@ -704,7 +742,7 @@ class system_info(object): log.info(' FOUND:') res = self.saved_results.get(self.__class__.__name__) - if self.verbosity > 0 and flag: + if log.get_threshold() <= log.INFO and flag: for k, v in res.items(): v = str(v) if k in ['sources', 'libraries'] and len(v) > 270: @@ -914,7 +952,7 @@ class system_info(object): """Return a list of existing paths composed by all combinations of items from the arguments. """ - return combine_paths(*args, **{'verbosity': self.verbosity}) + return combine_paths(*args) class fft_opt_info(system_info): @@ -1531,12 +1569,12 @@ def get_atlas_version(**config): try: s, o = c.get_output(atlas_version_c_text, libraries=libraries, library_dirs=library_dirs, - use_tee=(system_info.verbosity > 0)) + ) if s and re.search(r'undefined reference to `_gfortran', o, re.M): s, o = c.get_output(atlas_version_c_text, libraries=libraries + ['gfortran'], library_dirs=library_dirs, - use_tee=(system_info.verbosity > 0)) + ) if not s: warnings.warn(textwrap.dedent(""" ***************************************************** @@ -1582,7 +1620,7 @@ def get_atlas_version(**config): log.info('Status: %d', s) log.info('Output: %s', o) - if atlas_version == '3.2.1_pre3.3.6': + elif atlas_version == '3.2.1_pre3.3.6': dict_append(info, define_macros=[('NO_ATLAS_INFO', -2)]) else: dict_append(info, define_macros=[( @@ -1593,10 +1631,10 @@ def get_atlas_version(**config): class lapack_opt_info(system_info): - notfounderror = LapackNotFoundError - # Default order of LAPACK checks + # List of all known BLAS libraries, in the default order lapack_order = ['mkl', 'openblas', 'flame', 'atlas', 'accelerate', 'lapack'] + order_env_var_name = 'NPY_LAPACK_ORDER' def _calc_info_mkl(self): info = get_info('lapack_mkl') @@ -1688,8 +1726,11 @@ class lapack_opt_info(system_info): return True return False + def _calc_info(self, name): + return getattr(self, '_calc_info_{}'.format(name))() + def calc_info(self): - user_order = os.environ.get('NPY_LAPACK_ORDER', None) + user_order = os.environ.get(self.order_env_var_name, None) if user_order is None: lapack_order = self.lapack_order else: @@ -1709,7 +1750,7 @@ class lapack_opt_info(system_info): "values: {}".format(non_existing)) for lapack in lapack_order: - if getattr(self, '_calc_info_{}'.format(lapack))(): + if self._calc_info(lapack): return if 'lapack' not in lapack_order: @@ -1719,11 +1760,53 @@ class lapack_opt_info(system_info): warnings.warn(LapackSrcNotFoundError.__doc__ or '', stacklevel=2) -class blas_opt_info(system_info): +class _ilp64_opt_info_mixin: + symbol_suffix = None + symbol_prefix = None + + def _check_info(self, info): + macros = dict(info.get('define_macros', [])) + prefix = macros.get('BLAS_SYMBOL_PREFIX', '') + suffix = macros.get('BLAS_SYMBOL_SUFFIX', '') + + if self.symbol_prefix not in (None, prefix): + return False + + if self.symbol_suffix not in (None, suffix): + return False + + return bool(info) + + +class lapack_ilp64_opt_info(lapack_opt_info, _ilp64_opt_info_mixin): + notfounderror = LapackILP64NotFoundError + lapack_order = ['openblas64_', 'openblas_ilp64'] + order_env_var_name = 'NPY_LAPACK_ILP64_ORDER' + + def _calc_info(self, name): + info = get_info(name + '_lapack') + if self._check_info(info): + self.set_info(**info) + return True + return False + + +class lapack_ilp64_plain_opt_info(lapack_ilp64_opt_info): + # Same as lapack_ilp64_opt_info, but fix symbol names + symbol_prefix = '' + symbol_suffix = '' + + +class lapack64__opt_info(lapack_ilp64_opt_info): + symbol_prefix = '' + symbol_suffix = '64_' + +class blas_opt_info(system_info): notfounderror = BlasNotFoundError - # Default order of BLAS checks + # List of all known BLAS libraries, in the default order blas_order = ['mkl', 'blis', 'openblas', 'atlas', 'accelerate', 'blas'] + order_env_var_name = 'NPY_BLAS_ORDER' def _calc_info_mkl(self): info = get_info('blas_mkl') @@ -1788,8 +1871,11 @@ class blas_opt_info(system_info): self.set_info(**info) return True + def _calc_info(self, name): + return getattr(self, '_calc_info_{}'.format(name))() + def calc_info(self): - user_order = os.environ.get('NPY_BLAS_ORDER', None) + user_order = os.environ.get(self.order_env_var_name, None) if user_order is None: blas_order = self.blas_order else: @@ -1807,7 +1893,7 @@ class blas_opt_info(system_info): raise ValueError("blas_opt_info user defined BLAS order has unacceptable values: {}".format(non_existing)) for blas in blas_order: - if getattr(self, '_calc_info_{}'.format(blas))(): + if self._calc_info(blas): return if 'blas' not in blas_order: @@ -1817,6 +1903,29 @@ class blas_opt_info(system_info): warnings.warn(BlasSrcNotFoundError.__doc__ or '', stacklevel=2) +class blas_ilp64_opt_info(blas_opt_info, _ilp64_opt_info_mixin): + notfounderror = BlasILP64NotFoundError + blas_order = ['openblas64_', 'openblas_ilp64'] + order_env_var_name = 'NPY_BLAS_ILP64_ORDER' + + def _calc_info(self, name): + info = get_info(name) + if self._check_info(info): + self.set_info(**info) + return True + return False + + +class blas_ilp64_plain_opt_info(blas_ilp64_opt_info): + symbol_prefix = '' + symbol_suffix = '' + + +class blas64__opt_info(blas_ilp64_opt_info): + symbol_prefix = '' + symbol_suffix = '64_' + + class blas_info(system_info): section = 'blas' dir_env_var = 'BLAS' @@ -1905,8 +2014,6 @@ class blas_info(system_info): library_dirs=info['library_dirs'], extra_postargs=info.get('extra_link_args', [])) return libs - # This breaks the for loop - break except distutils.ccompiler.LinkError: pass finally: @@ -1918,12 +2025,24 @@ class openblas_info(blas_info): section = 'openblas' dir_env_var = 'OPENBLAS' _lib_names = ['openblas'] + _require_symbols = [] notfounderror = BlasNotFoundError - def check_embedded_lapack(self, info): - return True + @property + def symbol_prefix(self): + try: + return self.cp.get(self.section, 'symbol_prefix') + except NoOptionError: + return '' - def calc_info(self): + @property + def symbol_suffix(self): + try: + return self.cp.get(self.section, 'symbol_suffix') + except NoOptionError: + return '' + + def _calc_info(self): c = customized_ccompiler() lib_dirs = self.get_lib_dirs() @@ -1941,23 +2060,33 @@ class openblas_info(blas_info): # Try gfortran-compatible library files info = self.check_msvc_gfortran_libs(lib_dirs, openblas_libs) # Skip lapack check, we'd need build_ext to do it - assume_lapack = True + skip_symbol_check = True elif info: - assume_lapack = False + skip_symbol_check = False info['language'] = 'c' if info is None: - return + return None # Add extra info for OpenBLAS extra_info = self.calc_extra_info() dict_append(info, **extra_info) - if not (assume_lapack or self.check_embedded_lapack(info)): - return + if not (skip_symbol_check or self.check_symbols(info)): + return None info['define_macros'] = [('HAVE_CBLAS', None)] - self.set_info(**info) + if self.symbol_prefix: + info['define_macros'] += [('BLAS_SYMBOL_PREFIX', self.symbol_prefix)] + if self.symbol_suffix: + info['define_macros'] += [('BLAS_SYMBOL_SUFFIX', self.symbol_suffix)] + + return info + + def calc_info(self): + info = self._calc_info() + if info is not None: + self.set_info(**info) def check_msvc_gfortran_libs(self, library_dirs, libraries): # First, find the full path to each library directory @@ -1990,24 +2119,27 @@ class openblas_info(blas_info): return info -class openblas_lapack_info(openblas_info): - section = 'openblas' - dir_env_var = 'OPENBLAS' - _lib_names = ['openblas'] - notfounderror = BlasNotFoundError - - def check_embedded_lapack(self, info): + def check_symbols(self, info): res = False c = customized_ccompiler() tmpdir = tempfile.mkdtemp() + + prototypes = "\n".join("void %s%s%s();" % (self.symbol_prefix, + symbol_name, + self.symbol_suffix) + for symbol_name in self._require_symbols) + calls = "\n".join("%s%s%s();" % (self.symbol_prefix, + symbol_name, + self.symbol_suffix) + for symbol_name in self._require_symbols) s = textwrap.dedent("""\ - void zungqr_(); + %(prototypes)s int main(int argc, const char *argv[]) { - zungqr_(); + %(calls)s return 0; - }""") + }""") % dict(prototypes=prototypes, calls=calls) src = os.path.join(tmpdir, 'source.c') out = os.path.join(tmpdir, 'a.out') # Add the additional "extra" arguments @@ -2032,9 +2164,49 @@ class openblas_lapack_info(openblas_info): shutil.rmtree(tmpdir) return res +class openblas_lapack_info(openblas_info): + section = 'openblas' + dir_env_var = 'OPENBLAS' + _lib_names = ['openblas'] + _require_symbols = ['zungqr_'] + notfounderror = BlasNotFoundError + class openblas_clapack_info(openblas_lapack_info): _lib_names = ['openblas', 'lapack'] +class openblas_ilp64_info(openblas_info): + section = 'openblas_ilp64' + dir_env_var = 'OPENBLAS_ILP64' + _lib_names = ['openblas64'] + _require_symbols = ['dgemm_', 'cblas_dgemm'] + notfounderror = BlasILP64NotFoundError + + def _calc_info(self): + info = super()._calc_info() + if info is not None: + info['define_macros'] += [('HAVE_BLAS_ILP64', None)] + return info + +class openblas_ilp64_lapack_info(openblas_ilp64_info): + _require_symbols = ['dgemm_', 'cblas_dgemm', 'zungqr_', 'LAPACKE_zungqr'] + + def _calc_info(self): + info = super()._calc_info() + if info: + info['define_macros'] += [('HAVE_LAPACKE', None)] + return info + +class openblas64__info(openblas_ilp64_info): + # ILP64 Openblas, with default symbol suffix + section = 'openblas64_' + dir_env_var = 'OPENBLAS64_' + _lib_names = ['openblas64_'] + symbol_suffix = '64_' + symbol_prefix = '' + +class openblas64__lapack_info(openblas_ilp64_lapack_info, openblas64__info): + pass + class blis_info(blas_info): section = 'blis' dir_env_var = 'BLIS' diff --git a/numpy/distutils/tests/test_fcompiler.py b/numpy/distutils/tests/test_fcompiler.py index ba19a97ea..6d245fbd4 100644 --- a/numpy/distutils/tests/test_fcompiler.py +++ b/numpy/distutils/tests/test_fcompiler.py @@ -45,37 +45,3 @@ def test_fcompiler_flags(monkeypatch): else: assert_(new_flags == prev_flags + [new_flag]) - -def test_fcompiler_flags_append_warning(monkeypatch): - # Test to check that the warning for append behavior changing in future - # is triggered. Need to use a real compiler instance so that we have - # non-empty flags to start with (otherwise the "if var and append" check - # will always be false). - try: - with suppress_warnings() as sup: - sup.record() - fc = numpy.distutils.fcompiler.new_fcompiler(compiler='gnu95') - fc.customize() - except numpy.distutils.fcompiler.CompilerNotFound: - pytest.skip("gfortran not found, so can't execute this test") - - # Ensure NPY_DISTUTILS_APPEND_FLAGS not defined - monkeypatch.delenv('NPY_DISTUTILS_APPEND_FLAGS', raising=False) - - for opt, envvar in customizable_flags: - new_flag = '-dummy-{}-flag'.format(opt) - with suppress_warnings() as sup: - sup.record() - prev_flags = getattr(fc.flag_vars, opt) - - monkeypatch.setenv(envvar, new_flag) - with suppress_warnings() as sup: - sup.record() - new_flags = getattr(fc.flag_vars, opt) - if prev_flags: - # Check that warning was issued - assert len(sup.log) == 1 - - monkeypatch.delenv(envvar) - assert_(new_flags == [new_flag]) - |