Enable Incremental build For Generated Files

3 files changed, 85 insertions, 22 deletions

diff --git a/build.cmd b/build.cmd
index c9a6d8b893..5f892afe55 100644
--- a/build.cmd
+++ b/build.cmd
@@ -67,7 +67,7 @@ set "__IntermediatesDir=%__RootBinDir%\obj\%__BuildOS%.%__BuildArch%.%__BuildTyp
 set "__PackagesBinDir=%__BinDir%\.nuget"
 set "__TestBinDir=%__RootBinDir%\tests\%__BuildOS%.%__BuildArch%.%__BuildType%"
 set "__TestIntermediatesDir=%__RootBinDir%\tests\obj\%__BuildOS%.%__BuildArch%.%__BuildType%"
-set "__GeneratedIntermediatesDir=%__IntermediatesDir%\Generated"
+set "__GeneratedIntermediatesDir=%__IntermediatesDir%\Generated_latest" ::use this variable to locate dynamically generated files, the actual location though will be different
 
 :: Generate path to be set for CMAKE_INSTALL_PREFIX to contain forward slash
 set "__CMakeBinDir=%__BinDir%"
@@ -81,7 +81,6 @@ echo.
 :: MSBuild projects would need a rebuild
 set __MSBCleanBuildArgs=/t:rebuild
 
-
 :: Cleanup the previous output for the selected configuration
 if exist "%__BinDir%" rd /s /q "%__BinDir%"
 if exist "%__IntermediatesDir%" rd /s /q "%__IntermediatesDir%"
@@ -131,7 +130,7 @@ exit /b 1
 :: Note: We've disabled node reuse because it causes file locking issues.
 ::       The issue is that we extend the build with our own targets which
 ::       means that that rebuilding cannot successfully delete the task
-::       assembly. 
+::       assembly.
 if /i "%__VSVersion%" =="vs2015" goto MSBuild14
 set _msbuildexe="%ProgramFiles(x86)%\MSBuild\12.0\Bin\MSBuild.exe"
 if not exist %_msbuildexe% set _msbuildexe="%ProgramFiles%\MSBuild\12.0\Bin\MSBuild.exe"
@@ -200,19 +199,24 @@ exit /b 1
 
 REM Build CoreCLR
 :BuildCoreCLR
-:: Run Steps to Generate ETW specific infrastructure the 
-if exist "%__GeneratedIntermediatesDir%" rd /s /q "%__GeneratedIntermediatesDir%" ::Ensure there are no stale files in the Generated Directory
-md "%__GeneratedIntermediatesDir%"
-md "%__GeneratedIntermediatesDir%\inc"
+:: Run Steps to Generate ETW specific infrastructure the
+if  exist "%__GeneratedIntermediatesDir%" rd /s /q "%__GeneratedIntermediatesDir%" ::Ensure there are no stale files in the Generated Directory
+md  "%__GeneratedIntermediatesDir%"
+md  "%__GeneratedIntermediatesDir%\inc"
 set "genetw=%__SourceDir%\scripts\genWinEtw.py"
 
 mc -h "%__GeneratedIntermediatesDir%\inc" -r "%__GeneratedIntermediatesDir%" -b -co -um -p FireEtw "%__SourceDir%\VM\ClrEtwAll.man"
-IF ERRORLEVEL 1 goto FailedToGenEtwMetadata 
+IF ERRORLEVEL 1 goto FailedToGenEtwMetadata
 
 echo generating clretwallmain.h and etmdummy.h
-"%PythonPath%"  "%genetw%" --man "%__SourceDir%\VM\ClrEtwAll.man" --exc "%__SourceDir%\VM\ClrEtwAllMeta.lst" --eventheader "%__GeneratedIntermediatesDir%\inc\ClrEtwAll.h" --macroheader "%__GeneratedIntermediatesDir%\inc\clretwallmain.h" --dummy "%__GeneratedIntermediatesDir%\inc\etmdummy.h" 
-IF  ERRORLEVEL 1 goto FailedToGenEtwMetadata 
+"%PythonPath%"  "%genetw%" --man "%__SourceDir%\VM\ClrEtwAll.man" --exc "%__SourceDir%\VM\ClrEtwAllMeta.lst" --eventheader "%__GeneratedIntermediatesDir%\inc\ClrEtwAll.h" --macroheader "%__GeneratedIntermediatesDir%\inc\clretwallmain.h" --dummy "%__GeneratedIntermediatesDir%\inc\etmdummy.h"
+IF  ERRORLEVEL 1 goto FailedToGenEtwMetadata
+
+set "__GeneratedIntermediatesDirPresent=%__IntermediatesDir%\Generated" ::do not use this variable, it is used below to support incremental build
+"%PythonPath%" -c "import sys;sys.path.insert(0,r\"%__SourceDir%\scripts\"); from Utilities import *;UpdateDirectory(r\"%__GeneratedIntermediatesDirPresent%\",r\"%__GeneratedIntermediatesDir%\")"
+IF  ERRORLEVEL 1 goto FailedToGenEtwMetadata
 
+set __GeneratedIntermediatesDir="%__GeneratedIntermediatesDirPresent%"
 goto BuildVM
 
 :FailedToGenEtwMetadata
@@ -222,7 +226,7 @@ exit /b %ERRORLEVEL%
 :BuildVM
 
 set "__CoreCLRBuildLog=%__LogsDir%\CoreCLR_%__BuildOS%__%__BuildArch%__%__BuildType%.log"
-if /i "%__BuildArch%" == "arm64" (  
+if /i "%__BuildArch%" == "arm64" (
   REM TODO, remove once we have msbuild support for this platform.
 
   %_msbuildexe% "%__IntermediatesDir%\install.vcxproj" %__MSBCleanBuildArgs% /nologo /maxcpucount /nodeReuse:false /p:Configuration=%__BuildType% /p:UseEnv=true /fileloggerparameters:Verbosity=normal;LogFile="%__CoreCLRBuildLog%"
@@ -333,4 +337,4 @@ echo osxmscorlib - Build mscorlib for OS X
 echo skiptestbuild - Skip building tests
 echo toolset_dir - toolset directory -- Arm64 use only.
 echo Priority (N) where N is a number that signifies the set of tests that will be built and consequently run.
-exit /b 1
\ No newline at end of file
+exit /b 1
diff --git a/build.sh b/build.sh
index 541e631088..4925f20e30 100755
--- a/build.sh
+++ b/build.sh
@@ -27,11 +27,9 @@ setup_dirs()
     mkdir -p "$__BinDir"
     mkdir -p "$__LogsDir"
     mkdir -p "$__IntermediatesDir"
-
     # Ensure there are no stale generated files
-    rm -rf "$__IntermediatesDir"
-    mkdir -p "$__GeneratedIntermediatesDir"
-    mkdir -p "$__GeneratedIntermediatesDir/inc"
+    rm -rf "$__GeneratedIntermediatesDir"
+
 }
 
 # Performs "clean build" type actions (deleting and remaking directories)
@@ -69,23 +67,36 @@ build_coreclr()
         return
     fi
 
-    # All set to commence the build
-
-    echo "Commencing build of native components for $__BuildOS.$__BuildArch.$__BuildType"
-    
     echo "Laying out dynamically generated files consumed by the build system "
     python "$__ProjectRoot/src/scripts/genXplatEventing.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --exc "$__ProjectRoot/src/vm/ClrEtwAllMeta.lst" --inc "$__GeneratedIntermediatesDir/inc" --dummy "$__GeneratedIntermediatesDir/inc/etmdummy.h" --testdir "$__GeneratedIntermediatesDir/eventprovider_tests"
-    
+    if  [[ $? != 0 ]]; then
+        exit
+    fi
+
     #determine the logging system
     case $__BuildOS in
         Linux)
             python "$__ProjectRoot/src/scripts/genXplatLttng.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediatesDir/"
+            if  [[ $? != 0 ]]; then
+                exit
+            fi
             ;;
         *)
             ;;
     esac
 
+    export __GeneratedIntermediatesDirPresent="$__IntermediatesDir/Generated" #do not use this variable, it is used below to support incremental build
+    python -c "import sys;sys.path.insert(0,\"$__ProjectRoot/src/scripts\"); from Utilities import *;UpdateDirectory(\"$__GeneratedIntermediatesDirPresent\",\"$__GeneratedIntermediatesDir\")"
+    if  [[ $? != 0 ]]; then
+        exit
+    fi
+
+    # Do not do any more processing from now on:
+    # All set to commence the build
+
+    export __GeneratedIntermediatesDir="$__GeneratedIntermediatesDirPresent"
     cd "$__IntermediatesDir"
+    echo "Commencing build of native components for $__BuildOS.$__BuildArch.$__BuildType"
 
     generator=""
     buildFile="Makefile"
@@ -370,7 +381,7 @@ __ToolsDir="$__RootBinDir/tools"
 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
-export __GeneratedIntermediatesDir="$__IntermediatesDir/Generated"
+export __GeneratedIntermediatesDir="$__IntermediatesDir/Generated_latest" # use this variable to locate dynamically generated files, the actual location though will be different
 
 # Specify path to be set for CMAKE_INSTALL_PREFIX.
 # This is where all built CoreClr libraries will copied to.
diff --git a/src/scripts/Utilities.py b/src/scripts/Utilities.py
new file mode 100644
index 0000000000..0de1cafdbd
--- /dev/null
+++ b/src/scripts/Utilities.py
@@ -0,0 +1,48 @@
+from filecmp import dircmp
+import shutil
+import os
+
+def walk_recursively_and_update(dcmp):
+    #for different Files Copy from right to left
+    for name in dcmp.diff_files:
+        srcpath = dcmp.right + "/" + name
+        destpath = dcmp.left + "/" + name
+        print "Updating %s" % (destpath)
+        if  os.path.isfile(srcpath):
+            shutil.copyfile(srcpath, destpath)
+        else :
+            raise Exception("path: " + srcpath + "is neither a file or folder")
+
+    #copy right only files
+    for name in dcmp.right_only:
+        srcpath = dcmp.right + "/" + name
+        destpath = dcmp.left + "/" + name
+        print "Updating %s" % (destpath)
+        if  os.path.isfile(srcpath):
+            shutil.copyfile(srcpath, destpath)
+        elif  os.path.isdir(srcpath):
+            shutil.copytree(srcpath, destpath)
+        else :
+            raise Exception("path: " + srcpath + "is neither a file or folder")
+
+    #delete left only files
+    for name in dcmp.left_only:
+        path = dcmp.left + "/" + name
+        print "Deleting " % (path)
+        if  os.path.isfile(path):
+            os.remove(path)
+        elif  os.path.isdir(path):
+            shutil.rmtree(path)
+        else :
+            raise Exception("path: " + path + "is neither a file or folder")
+
+    #call recursively
+    for sub_dcmp in dcmp.subdirs.values():
+        walk_recursively_and_update(sub_dcmp)
+
+def UpdateDirectory(destpath,srcpath):
+
+    if not os.path.exists(destpath):
+        os.makedirs(destpath)
+    dcmp = dircmp(destpath,srcpath)
+    walk_recursively_and_update(dcmp)