Fix incremental build by preventing nonchanging writes (#15817)

Fix incremental build by preventing non-changing writes in generated ETW source code

3 files changed, 86 insertions, 60 deletions

diff --git a/src/scripts/genEtwProvider.py b/src/scripts/genEtwProvider.py
index fc9d43800f..9f012bfaf0 100644
--- a/src/scripts/genEtwProvider.py
+++ b/src/scripts/genEtwProvider.py
@@ -14,7 +14,7 @@ import argparse
 import subprocess
 import xml.dom.minidom as DOM
 from genEventing import parseTemplateNodes
-from utilities import open_for_update
+from utilities import open_for_update, update_directory
 
 macroheader_filename = "etwmacros.h"
 mcheader_filename = "ClrEtwAll.h"
@@ -49,7 +49,7 @@ stdprolog_cmake="""
 """
 
 def genProviderInterface(manifest, intermediate):
-    provider_dirname = os.path.join(intermediate, etw_dirname)
+    provider_dirname = os.path.join(intermediate, etw_dirname + "_temp")
 
     if not os.path.exists(provider_dirname):
         os.makedirs(provider_dirname)
@@ -64,7 +64,7 @@ def genProviderInterface(manifest, intermediate):
     for pattern, replacement in replacements:
         header_text = re.sub(pattern, replacement, header_text)
 
-    with open_for_update(path.join(provider_dirname, mcheader_filename)) as mcheader_file:
+    with open(path.join(provider_dirname, mcheader_filename), 'w') as mcheader_file:
         mcheader_file.write(header_text)
 
 def genCmake(intermediate):
@@ -218,7 +218,7 @@ def checkConsistency(manifest, exclusion_filename):
                         raise Exception(sStackSpecificityError)
 
 def genEtwMacroHeader(manifest, exclusion_filename, intermediate):
-    provider_dirname = os.path.join(intermediate, etw_dirname)
+    provider_dirname = os.path.join(intermediate, etw_dirname + "_temp")
 
     if not os.path.exists(provider_dirname):
         os.makedirs(provider_dirname)
@@ -307,6 +307,15 @@ def main(argv):
     checkConsistency(manifest, exclusion_filename)
     genFiles(manifest, intermediate, exclusion_filename)
 
+    # Update the final directory from temp
+    provider_temp_dirname = os.path.join(intermediate, etw_dirname + "_temp")
+    provider_dirname = os.path.join(intermediate, etw_dirname)
+    if not os.path.exists(provider_dirname):
+        os.makedirs(provider_dirname)
+        
+    update_directory(provider_temp_dirname, provider_dirname)
+    shutil.rmtree(provider_temp_dirname)
+
 if __name__ == '__main__':
     return_code = main(sys.argv[1:])
     sys.exit(return_code)
\ No newline at end of file
diff --git a/src/scripts/genEventing.py b/src/scripts/genEventing.py
index d6d6afbb90..123ce81be4 100644
--- a/src/scripts/genEventing.py
+++ b/src/scripts/genEventing.py
@@ -721,7 +721,7 @@ def generateEtmDummyHeader(sClrEtwAllMan,clretwdummy):
             #pal: create etmdummy.h
             Clretwdummy.write(generateclrEtwDummy(eventNodes, allTemplates) + "\n")
 
-def generatePlatformIndependentFiles(sClrEtwAllMan, incDir, etmDummyFile, extern):
+def generatePlatformIndependentFiles(sClrEtwAllMan, incDir, etmDummyFile, extern, write_xplatheader):
 
     generateEtmDummyHeader(sClrEtwAllMan,etmDummyFile)
     tree           = DOM.parse(sClrEtwAllMan)
@@ -751,13 +751,23 @@ def generatePlatformIndependentFiles(sClrEtwAllMan, incDir, etmDummyFile, extern
             Clrallevents.write(generateClrallEvents(eventNodes, allTemplates) + "\n")
 
 
-    # Write secondary headers for FireEtXplat* and EventPipe* functions
-    clrxplatevents = os.path.join(incDir, "clrxplatevents.h")
     clreventpipewriteevents = os.path.join(incDir, "clreventpipewriteevents.h")
-    with open_for_update(clrxplatevents) as Clrxplatevents:
-        with open_for_update(clreventpipewriteevents) as Clreventpipewriteevents:
+    with open_for_update(clreventpipewriteevents) as Clreventpipewriteevents:
+        Clreventpipewriteevents.write(stdprolog + "\n")
+
+        for providerNode in tree.getElementsByTagName('provider'):
+            templateNodes = providerNode.getElementsByTagName('template')
+            allTemplates  = parseTemplateNodes(templateNodes)
+            eventNodes = providerNode.getElementsByTagName('event')
+
+            #eventpipe: create clreventpipewriteevents.h
+            Clreventpipewriteevents.write(generateClrEventPipeWriteEvents(eventNodes, allTemplates, extern) + "\n")
+                
+    # Write secondary headers for FireEtXplat* and EventPipe* functions
+    if write_xplatheader:
+        clrxplatevents = os.path.join(incDir, "clrxplatevents.h")
+        with open_for_update(clrxplatevents) as Clrxplatevents:
             Clrxplatevents.write(stdprolog + "\n")
-            Clreventpipewriteevents.write(stdprolog + "\n")
 
             for providerNode in tree.getElementsByTagName('provider'):
                 templateNodes = providerNode.getElementsByTagName('template')
@@ -767,9 +777,6 @@ def generatePlatformIndependentFiles(sClrEtwAllMan, incDir, etmDummyFile, extern
                 #pal: create clrallevents.h
                 Clrxplatevents.write(generateClrXplatEvents(eventNodes, allTemplates, extern) + "\n")
 
-                #eventpipe: create clreventpipewriteevents.h
-                Clreventpipewriteevents.write(generateClrEventPipeWriteEvents(eventNodes, allTemplates, extern) + "\n")
-
 import argparse
 import sys
 
@@ -789,6 +796,8 @@ def main(argv):
                                     help='full path to directory where the test assets will be deployed' )
     required.add_argument('--nonextern', action='store_true',
                                     help='if specified, will not generated extern function stub headers' )
+    required.add_argument('--noxplatheader', action='store_true',
+                                    help='if specified, will not write a generated cross-platform header' )
     args, unknown = parser.parse_known_args(argv)
     if unknown:
         print('Unknown argument(s): ', ', '.join(unknown))
@@ -799,8 +808,9 @@ def main(argv):
     etmDummyFile      = args.dummy
     testDir           = args.testdir
     extern            = not args.nonextern
+    write_xplatheader = not args.noxplatheader
 
-    generatePlatformIndependentFiles(sClrEtwAllMan, incdir, etmDummyFile, extern)
+    generatePlatformIndependentFiles(sClrEtwAllMan, incdir, etmDummyFile, extern, write_xplatheader)
     generateSanityTest(sClrEtwAllMan, testDir)
 
 if __name__ == '__main__':
diff --git a/src/scripts/utilities.py b/src/scripts/utilities.py
index 82e6c83346..7e69f462a9 100644
--- a/src/scripts/utilities.py
+++ b/src/scripts/utilities.py
@@ -5,9 +5,9 @@
 ##
 ##  This file provides utility functions to the adjacent python scripts
 
-from filecmp import dircmp
 from hashlib import sha256
 from io import StringIO
+import filecmp
 import shutil
 import sys
 import os
@@ -68,48 +68,55 @@ class UpdateFileWriter:
 def open_for_update(filename):
     return UpdateFileWriter(filename)
 
-def walk_recursively_and_update(dcmp):
-    #for different Files Copy from right to left
-    for name in dcmp.diff_files:
-        srcpath = dcmp.right + "/" + name
-        destpath = dcmp.left + "/" + name
-        print("Updating %s" % (destpath))
-        if  os.path.isfile(srcpath):
-            shutil.copyfile(srcpath, destpath)
-        else :
-            raise Exception("path: " + srcpath + "is neither a file or folder")
-
-    #copy right only files
-    for name in dcmp.right_only:
-        srcpath = dcmp.right + "/" + name
-        destpath = dcmp.left + "/" + name
-        print("Updating %s" % (destpath))
-        if  os.path.isfile(srcpath):
-            shutil.copyfile(srcpath, destpath)
-        elif  os.path.isdir(srcpath):
-            shutil.copytree(srcpath, destpath)
-        else :
-            raise Exception("path: " + srcpath + "is neither a file or folder")
-
-    #delete left only files
-    for name in dcmp.left_only:
-        path = dcmp.left + "/" + name
-        print("Deleting %s" % (path))
-        if  os.path.isfile(path):
-            os.remove(path)
-        elif  os.path.isdir(path):
-            shutil.rmtree(path)
-        else :
-            raise Exception("path: " + path + "is neither a file or folder")
-
-    #call recursively
-    for sub_dcmp in dcmp.subdirs.values():
-        walk_recursively_and_update(sub_dcmp)
-
-def UpdateDirectory(destpath,srcpath):
-
-    print("Updating %s with %s" % (destpath,srcpath))
-    if not os.path.exists(destpath):
-        os.makedirs(destpath)
-    dcmp = dircmp(destpath,srcpath)
-    walk_recursively_and_update(dcmp)
+def split_entries(entries, directory):
+    """Given a list of entries in a directory, listing return a set of file and a set of dirs"""
+    files = set([entry for entry in entries if os.path.isfile(os.path.join(directory, entry))])
+    dirs = set([entry for entry in entries if os.path.isdir(os.path.join(directory, entry))])
+
+    return files, dirs
+
+def update_directory(srcpath, dstpath, recursive=True, destructive=True, shallow=False):
+    """Updates dest directory with files from src directory
+
+    Args:
+        destpath (str): The destination path to sync with the source
+        srcpath (str): The source path to sync to the destination
+        recursive(boolean): If True, descend into and update subdirectories (default: True)
+        destructive(boolean): If True, delete files in the destination which do not exist in the source (default: True)
+        shallow(boolean): If True, only use os.stat to diff files. Do not examine contents (default: False)
+    """
+    srcfiles, srcdirs = split_entries(os.listdir(srcpath), srcpath)
+    dstfiles, dstdirs = split_entries(os.listdir(dstpath), dstpath)
+
+
+    # Update files in both src and destination which are different in destination
+    commonfiles = srcfiles.intersection(dstfiles)
+    _, mismatches, errors = filecmp.cmpfiles(srcpath, dstpath, commonfiles, shallow=shallow)
+
+    if errors:
+        raise RuntimeError("Comparison failed for the following files(s): {}".format(errors))
+
+    for mismatch in mismatches:
+        shutil.copyfile(os.path.join(srcpath, mismatch), os.path.join(dstpath, mismatch))
+
+    # Copy over files from source which do not exist in the destination
+    for missingfile in srcfiles.difference(dstfiles):
+        shutil.copyfile(os.path.join(srcpath, missingfile), os.path.join(dstpath, missingfile))
+
+    #If destructive, delete files in destination which do not exist in sourc
+    if destructive:
+        for deadfile in dstfiles.difference(srcfiles):
+            print(deadfile)
+            os.remove(os.path.join(dstpath, deadfile))
+
+        for deaddir in dstdirs.difference(srcdirs):
+            print(deaddir)
+            shutil.rmtree(os.path.join(dstpath, deaddir))
+
+    #If recursive, do this again for each source directory
+    if recursive:
+        for dirname in srcdirs:
+            dstdir, srcdir = os.path.join(dstpath, dirname), os.path.join(srcpath, dirname)
+            if not os.path.exists(dstdir):
+                os.makedirs(dstdir)
+            update_directory(srcdir, dstdir, recursive, destructive, shallow)