Merge pull request #9050 from juliantaylor/fortranobj-path

BUG: distutils, add compatiblity python parallelization

2 files changed, 50 insertions, 9 deletions

diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py
index 98c4cc022..e7557b3e6 100644
--- a/numpy/distutils/ccompiler.py
+++ b/numpy/distutils/ccompiler.py
@@ -5,6 +5,7 @@ import re
 import sys
 import types
 import shlex
+import time
 from copy import copy
 from distutils import ccompiler
 from distutils.ccompiler import *
@@ -20,6 +21,15 @@ from numpy.distutils.misc_util import cyg2win32, is_sequence, mingw32, \
                                       quote_args, get_num_build_jobs, \
                                       _commandline_dep_string
 
+# globals for parallel build management
+try:
+    import threading
+except ImportError:
+    import dummy_threading as threading
+_job_semaphore = None
+_global_lock = threading.Lock()
+_processing_files = set()
+
 
 def _needs_build(obj, cc_args, extra_postargs, pp_opts):
     """
@@ -211,6 +221,16 @@ def CCompiler_compile(self, sources, output_dir=None, macros=None,
     # This method is effective only with Python >=2.3 distutils.
     # Any changes here should be applied also to fcompiler.compile
     # method to support pre Python 2.3 distutils.
+    global _job_semaphore
+
+    jobs = get_num_build_jobs()
+
+    # setup semaphore to not exceed number of compile jobs when parallelized at
+    # extension level (python >= 3.5)
+    with _global_lock:
+        if _job_semaphore is None:
+            _job_semaphore = threading.Semaphore(jobs)
+
     if not sources:
         return []
     # FIXME:RELATIVE_IMPORT
@@ -242,8 +262,30 @@ def CCompiler_compile(self, sources, output_dir=None, macros=None,
 
     def single_compile(args):
         obj, (src, ext) = args
-        if _needs_build(obj, cc_args, extra_postargs, pp_opts):
-            self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+        if not _needs_build(obj, cc_args, extra_postargs, pp_opts):
+            return
+
+        # check if we are currently already processing the same object
+        # happens when using the same source in multiple extensions
+        while True:
+            # need explicit lock as there is no atomic check and add with GIL
+            with _global_lock:
+                # file not being worked on, start working
+                if obj not in _processing_files:
+                    _processing_files.add(obj)
+                    break
+            # wait for the processing to end
+            time.sleep(0.1)
+
+        try:
+            # retrieve slot from our #job semaphore and build
+            with _job_semaphore:
+                self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+        finally:
+            # register being done processing
+            with _global_lock:
+                _processing_files.remove(obj)
+
 
     if isinstance(self, FCompiler):
         objects_to_build = list(build.keys())
@@ -269,7 +311,6 @@ def CCompiler_compile(self, sources, output_dir=None, macros=None,
     else:
         build_items = build.items()
 
-    jobs = get_num_build_jobs()
     if len(build) > 1 and jobs > 1:
         # build parallel
         import multiprocessing.pool
diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py
index 18db1bb42..9def37822 100644
--- a/numpy/distutils/command/build_src.py
+++ b/numpy/distutils/command/build_src.py
@@ -569,14 +569,14 @@ class build_src(build_ext.build_ext):
         if not os.path.isfile(target_file):
             raise DistutilsError("f2py target file %r not generated" % (target_file,))
 
-        target_c = os.path.join(self.build_src, 'fortranobject.c')
-        target_h = os.path.join(self.build_src, 'fortranobject.h')
+        build_dir = os.path.join(self.build_src, target_dir)
+        target_c = os.path.join(build_dir, 'fortranobject.c')
+        target_h = os.path.join(build_dir, 'fortranobject.h')
         log.info("  adding '%s' to sources." % (target_c))
         new_sources.append(target_c)
-        if self.build_src not in extension.include_dirs:
-            log.info("  adding '%s' to include_dirs." \
-                     % (self.build_src))
-            extension.include_dirs.append(self.build_src)
+        if build_dir not in extension.include_dirs:
+            log.info("  adding '%s' to include_dirs." % (build_dir))
+            extension.include_dirs.append(build_dir)
 
         if not skip_f2py:
             import numpy.f2py