Group XUnitWrapper.dll into the same Payloads when submit to Helix (#23476)

* Add DisplayNameAttribute to each Fact

* Split files under bin/tests folder into groups and specify the corresponding *.XUnitWrapper.dll

* Group some of the *.XUnitWrapper.dll files into PayloadGroups and split others into different PayloadGroups if they have more than one TestGroup defined

* Add "TestGroup" trait to XUnitWrapper facts

* Add Microsoft.DotNet.XUnitConsoleRunner to eng\Versions.props

* Overwrite xunit.console.dll with the version coming from Microsoft.DotNet.XUnitConsoleRunner

1 files changed, 115 insertions, 25 deletions

diff --git a/tests/helixpublishwitharcade.proj b/tests/helixpublishwitharcade.proj
index d928269412..b421d56bbb 100644
--- a/tests/helixpublishwitharcade.proj
+++ b/tests/helixpublishwitharcade.proj
@@ -10,10 +10,6 @@
        that will run in parallel. -->
 
   <Target Name="RunInParallelForEachScenario">
-    <ItemGroup>
-      <_Scenarios Include="$(_Scenarios.Split(','))" />
-    </ItemGroup>
-
     <PropertyGroup>
       <!-- This specifies what properties are needed to be passed down as global properties to a child project. -->
 
@@ -34,9 +30,13 @@
       </_PropertiesToPass>
     </PropertyGroup>
 
+    <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" />
+    <MSBuild Projects="$(MSBuildProjectFile)" Targets="PreparePayloadsDirectories" Properties="Scenarios=$(_Scenarios)" StopOnFirstFailure="true" />
+
     <ItemGroup>
-      <!-- MSBuild creates a new instance of the project for each %(_Scenarios.Identity) and can build them in parallel. -->
+      <_Scenarios Include="$(_Scenarios.Split(','))" />
 
+      <!-- MSBuild creates a new instance of the project for each %(_Scenarios.Identity) and can build them in parallel. -->
       <_ProjectsToBuild Include="$(MSBuildProjectFile)">
         <AdditionalProperties>$(_PropertiesToPass);Scenario=%(_Scenarios.Identity)</AdditionalProperties>
       </_ProjectsToBuild>
@@ -44,41 +44,116 @@
 
     <PropertyGroup>
       <_BuildInParallel>false</_BuildInParallel>
-      <_BuildInParallel Condition=" '@(_ProjectsToBuild->Count())' > '1' ">true</_BuildInParallel>
+      <_BuildInParallel Condition=" '@(_ProjectsToBuild->Count())' &gt; '1' ">true</_BuildInParallel>
     </PropertyGroup>
 
-    <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFiles" StopOnFirstFailure="true" />
     <MSBuild Projects="@(_ProjectsToBuild)" Targets="Test" BuildInParallel="$(_BuildInParallel)" StopOnFirstFailure="false" Properties="UsesHelixSdk=true" />
   </Target>
 
-  <Import Project="..\dir.props" />
+  <Import Project="src\dir.props" />
 
   <PropertyGroup>
-    <CoreRootDir>$(TestWorkingDir)Tests\Core_Root</CoreRootDir>
+    <BinDir>$([MSBuild]::NormalizeDirectory($(BinDir)))</BinDir>
+    <CoreRootDirectory>$(BinDir)Tests\Core_Root\</CoreRootDirectory>
+    <PayloadsRootDirectory>$(BinDir)Payloads\</PayloadsRootDirectory>
     <TestEnvFileName Condition=" '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName>
     <TestEnvFileName Condition=" '$(TargetsWindows)' != 'true' ">SetStressModes_$(Scenario).sh</TestEnvFileName>
   </PropertyGroup>
 
-  <ItemGroup>
-    <XUnitWrapperDll Include="$(TestWorkingDir)\**\*.XUnitWrapper.dll">
-      <FileDirectory>%(RootDir)%(Directory)</FileDirectory>
-      <File>%(FileName)%(Extension)</File>
-    </XUnitWrapperDll>
-  </ItemGroup>
+  <Import Project="testgrouping.proj" />
 
-  <Target Name="CreateTestEnvFiles">
+  <Target Name="DiscoverAllXUnitWrappers">
+    <ItemGroup>
+      <_XUnitWrapperDll Include="%(TestGrouping.XUnitWrapperDll)" Condition="Exists('%(XUnitWrapperDll)')">
+        <PayloadGroup>%(TestGroup)</PayloadGroup>
+      </_XUnitWrapperDll>
+      <_XUnitWrapperDll Include="@(XUnitWrapperGrouping)" />
+
+      <!-- This adds the remaining *.XUnitWrapper.dll files in BinDir unless
+           1) they are in PayloadsRootDirectory or
+           2) they are grouped by XUnitWrapperGrouping. -->
+      <_XUnitWrapperDll Include="$(BinDir)**\*.XUnitWrapper.dll" Exclude="$(PayloadsRootDirectory)**\*.XUnitWrapper.dll;@(XUnitWrapperGrouping->Metadata('FullPath'))">
+         <!-- Set PayloadGroup to empty string, so we can update _XUnitWrapperDll items with no PayloadGroup to default value. Unfortunatelly, we can't do this right here. -->
+         <PayloadGroup></PayloadGroup>
+      </_XUnitWrapperDll>
+
+      <!-- Assign "default" value of PayloadGroup metadata. -->
+      <_XUnitWrapperDll Update="@(_XUnitWrapperDll)">
+         <PayloadGroup Condition="'%(PayloadGroup)' == ''">$([MSBuild]::ValueOrDefault(%(FileName),'').Replace('.XUnitWrapper',''))</PayloadGroup>
+      </_XUnitWrapperDll>
+    </ItemGroup>
+  </Target>
+
+  <Target Name="PreparePayloadDirectory" Outputs="%(_XUnitWrapperDll.FileName)%(PayloadGroup)" DependsOnTargets="DiscoverAllXUnitWrappers">
+    <PropertyGroup>
+      <_FileDirectory>%(_XUnitWrapperDll.RootDir)%(Directory)</_FileDirectory>
+      <_PayloadGroup>%(_XUnitWrapperDll.PayloadGroup)</_PayloadGroup>
+      <_XUnitWrapperDll>%(_XUnitWrapperDll.FullPath)</_XUnitWrapperDll>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <_TestGroupingRelevant Include="@(TestGrouping->WithMetadataValue('XUnitWrapperDll',$(_XUnitWrapperDll))->Metadata('FullPath'))" />
+    </ItemGroup>
+
+    <PropertyGroup>
+      <_TestGroupingExists>@(_TestGroupingRelevant->AnyHaveMetadataValue('TestGroup','$(_PayloadGroup)'))</_TestGroupingExists>
+    </PropertyGroup>
 
-    <!-- This target creates one __TestEnv file for each XUnitWrapper. -->
+    <ItemGroup>
+      <!-- If no TestGrouping is defined, all the files under $(_FileDirectory) and its subdirectories goes to the PayloadDirectory. -->
+      <_PayloadFiles Include="$(_FileDirectory)**" Exclude="@(_TestGroupingRelevant)" Condition="'$(_TestGroupingExists)' != 'true'" />
+      <!-- If there is a TestGrouping, then take only the files that belong to the TestGroup == $(_PayloadGroup). -->
+      <_PayloadFiles Include="@(_TestGroupingRelevant->WithMetadataValue('TestGroup','$(_PayloadGroup)')->DistinctWithCase())" Condition="'$(_TestGroupingExists)' == 'true'" />
+      <_PayloadFiles Include="$(_FileDirectory)*" Condition="'$(_TestGroupingExists)' == 'true'" />
+
+      <_PayloadFiles Update="@(_PayloadFiles)">
+        <!-- Never use [MSBuild]::MakeRelative here! We have some files containing Unicode characters in their %(FullPath) and
+             MakeRelative function calls Escape function internally that replaces all the Unicode characters with %<xx>. -->
+        <FileRelativeToPayloadsRootDirectory>$(_PayloadGroup)\$([System.IO.Path]::GetRelativePath($(BinDir), %(FullPath)))</FileRelativeToPayloadsRootDirectory>
+      </_PayloadFiles>
+    </ItemGroup>
+
+    <Copy SourceFiles="@(_PayloadFiles)" DestinationFiles="@(_PayloadFiles->'$(PayloadsRootDirectory)%(FileRelativeToPayloadsRootDirectory)')" />
+  </Target>
+
+  <Target Name="PrepareCorrelationPayloadDirectory">
+    <MSBuild Projects="xunitconsolerunner.depproj" Targets="Restore" />
+
+    <ItemGroup>
+      <_XUnitConsoleRunnerFiles Include="$(PackagesDir)$(MicrosoftDotNetXUnitConsoleRunnerPackage.ToLower())\$(MicrosoftDotNetXUnitConsoleRunnerVersion)\**\xunit.console.*" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(_XUnitConsoleRunnerFiles)" DestinationFolder="$(CoreRootDirectory)" />
+  </Target>
+
+  <Target Name="CreateTestEnvFiles">
+    <!-- This target creates one __TestEnv file for each combination of %(_PayloadGroups.Identity) and $(Scenario). -->
 
     <ItemGroup>
-      <_ProjectsToBuild Include=".\testenvironment.proj">
-        <Properties>Scenario=$(Scenario);TestEnvFileName=%(XUnitWrapperDll.FileDirectory)$(TestEnvFileName);TargetsWindows=$(TargetsWindows)</Properties>
+      <_PayloadGroups Include="$(PayloadGroups)" />
+      <_ProjectsToBuild Include="testenvironment.proj">
+        <Properties>Scenario=$(Scenario);TestEnvFileName=$(PayloadsRootDirectory)%(_PayloadGroups.Identity)\$(TestEnvFileName);TargetsWindows=$(TargetsWindows)</Properties>
       </_ProjectsToBuild>
     </ItemGroup>
 
     <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" />
   </Target>
 
+  <Target Name="PreparePayloadsDirectories" DependsOnTargets="PreparePayloadDirectory">
+    <PropertyGroup>
+      <_PayloadGroups>@(_XUnitWrapperDll->Metadata('PayloadGroup')->DistinctWithCase())</_PayloadGroups>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <_Scenario Include="$(_Scenarios.Split(','))" />
+      <_ProjectsToBuild Include="$(MSBuildProjectFile)">
+        <AdditionalProperties>Scenario=%(_Scenario.Identity);PayloadGroups=$(_PayloadGroups)</AdditionalProperties>
+      </_ProjectsToBuild>
+    </ItemGroup>
+
+    <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFiles" StopOnFirstFailure="true" />
+  </Target>
+
   <PropertyGroup>
     <EnableAzurePipelinesReporter>$(PublishTestResults)</EnableAzurePipelinesReporter>
     <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>
@@ -93,7 +168,7 @@
     <TestRunNamePrefix Condition=" '$(Scenario)' != 'normal' ">$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>
     <TimeoutPerTestInMilliseconds Condition=" '$(TimeoutPerTestInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestInMinutes)).TotalMilliseconds)</TimeoutPerTestInMilliseconds>
     <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
-    <XUnitRunnerArgs>-parallel collections -nocolor -noshadow -xml testResults.xml -notrait category=outerloop -notrait category=failing</XUnitRunnerArgs>
+    <XUnitRunnerArgs>-parallel collections -nocolor -noshadow -xml testResults.xml</XUnitRunnerArgs>
   </PropertyGroup>
 
   <!-- WARNING: HelixPreCommand ItemGroup is intentionally minimal and should be kept that way. -->
@@ -128,11 +203,26 @@
     <XUnitRunnerDll>$CORE_ROOT/xunit.console.dll</XUnitRunnerDll>
   </PropertyGroup>
 
-  <ItemGroup>
-    <HelixCorrelationPayload Include="$(CoreRootDir)" />
-    <HelixWorkItem Include="@(XUnitWrapperDll->'%(FileName)'->Replace('.XUnitWrapper', ''))">
-      <PayloadDirectory>%(FileDirectory)</PayloadDirectory>
-      <Command>$(CoreRun) $(XUnitRunnerDll) %(File) $(XUnitRunnerArgs)</Command>
+  <ItemGroup Condition=" '$(UsesHelixSdk)' == 'true' ">
+    <HelixCorrelationPayload Include="$(CoreRootDirectory)" />
+
+    <Payloads Include="$([System.IO.Directory]::GetDirectories($(PayloadsRootDirectory)))" Condition="Exists('$(PayloadsRootDirectory)')" />
+    <Payloads Update="@(Payloads)">
+      <PayloadGroup>$([MSBuild]::MakeRelative($(PayloadsRootDirectory), %(FullPath)))</PayloadGroup>
+      <PayloadDirectory>%(FullPath)</PayloadDirectory>
+      <XUnitWrapperDlls>$([System.String]::Join(' ', $([System.IO.Directory]::GetFiles(%(FullPath), '*.XUnitWrapper.dll', SearchOption.AllDirectories))).Replace($([MSBuild]::EnsureTrailingSlash(%(FullPath))),''))</XUnitWrapperDlls>
+    </Payloads>
+
+    <Payloads Update="@(Payloads)">
+      <TestGroup>%(PayloadGroup)</TestGroup>
+      <!-- When Payload contains more than one *.XUnitWrapper.dll TestGroup should not be specified. -->
+      <TestGroup Condition=" $([MSBuild]::ValueOrDefault(%(XUnitWrapperDlls), '').IndexOf('.XUnitWrapper.dll')) != $([MSBuild]::ValueOrDefault(%(XUnitWrapperDlls), '').LastIndexOf('.XUnitWrapper.dll')) "></TestGroup>
+    </Payloads>
+
+    <HelixWorkItem Include="@(Payloads->Metadata('PayloadGroup'))">
+      <PayloadDirectory>%(PayloadDirectory)</PayloadDirectory>
+      <Command>$(CoreRun) $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs)</Command>
+      <Command Condition=" '%(TestGroup)' != '' ">$(CoreRun) $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup)</Command>
       <Timeout Condition=" '$(TimeoutPerTestCollectionInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</Timeout>
     </HelixWorkItem>
   </ItemGroup>