Update CoreCLR to use xunit perf api

This updates the CoreCLR performance runs to use the new xunit
performance api.  This will unblock several people who were wanting to
move tests forward from netstandard1.4 which is were we have been locked
because of using the old desktop runner.

10 files changed, 74 insertions, 26 deletions

diff --git a/perf.groovy b/perf.groovy
index 2684b3b992..2a6e0e7dc5 100644
--- a/perf.groovy
+++ b/perf.groovy
@@ -89,8 +89,8 @@ def static getOSGroup(def os) {
 
 			// Save machinedata.json to /artifact/bin/ Jenkins dir
 			def archiveSettings = new ArchivalSettings()
-			archiveSettings.addFiles('perf-*.xml')
-			archiveSettings.addFiles('perf-*.etl')
+			archiveSettings.addFiles('Perf-*.xml')
+			archiveSettings.addFiles('Perf-*.etl')
 			Utilities.addArchival(newJob, archiveSettings)
 
 			Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
diff --git a/tests/scripts/run-xunit-perf.cmd b/tests/scripts/run-xunit-perf.cmd
index fef05e5527..bad1f93a40 100644
--- a/tests/scripts/run-xunit-perf.cmd
+++ b/tests/scripts/run-xunit-perf.cmd
@@ -44,9 +44,9 @@ pushd sandbox
 @rem stage stuff we need
 
 @rem xunit and perf
-xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.runner.Windows\1.0.0-alpha-build0040\tools\* . > %RUNLOG%
-xcopy /sy %CORECLR_REPO%\packages\Microsoft.DotNet.xunit.performance.analysis\1.0.0-alpha-build0040\tools\* . >> %RUNLOG%
-xcopy /sy %CORECLR_REPO%\packages\xunit.console.netcore\1.0.2-prerelease-00177\runtimes\any\native\* . >> %RUNLOG%
+"%CORECLR_REPO%\Tools\dotnetcli\dotnet.exe" restore "%CORECLR_REPO%\tests\src\Common\PerfHarness\project.json"
+"%CORECLR_REPO%\Tools\dotnetcli\dotnet.exe" publish "%CORECLR_REPO%\tests\src\Common\PerfHarness\project.json" -c Release -o %CORECLR_REPO%\sandbox
+xcopy /sy %CORECLR_REPO%\packages\Microsoft.Diagnostics.Tracing.TraceEvent\1.0.0-alpha-experimental\lib\native\* . >> %RUNLOG%
 xcopy /sy %CORECLR_REPO%\bin\tests\Windows_NT.%TEST_ARCH%.%TEST_CONFIG%\Tests\Core_Root\* . >> %RUNLOG%
 
 @rem find and stage the tests
@@ -94,16 +94,14 @@ if DEFINED TEST_ENV (
     )
 )
 
-xunit.performance.run.exe %BENCHNAME%.%TEST_FILE_EXT% -runner xunit.console.netcore.exe -runnerhost corerun.exe -verbose -runid %PERFOUT% > %BENCHNAME%.out
-
-xunit.performance.analysis.exe %PERFOUT%.xml -xml %XMLOUT% > %BENCHNAME%-analysis.out
+corerun.exe PerfHarness.dll %WORKSPACE%\sandbox\%BENCHNAME%.%TEST_FILE_EXT% --perf:runid Perf > %BENCHNAME%.out
 
 @rem optionally generate results for benchview
 if not [%BENCHVIEW_PATH%] == [] (
-  py "%BENCHVIEW_PATH%\measurement.py" xunit "perf-%BENCHNAME%.xml" --better desc --drop-first-value --append
+  py "%BENCHVIEW_PATH%\measurement.py" xunit "Perf-%BENCHNAME%.xml" --better desc --drop-first-value --append
   REM Save off the results to the root directory for recovery later in Jenkins
-  xcopy perf-%BENCHNAME%*.xml %CORECLR_REPO%\
-  xcopy perf-%BENCHNAME%*.etl %CORECLR_REPO%\
+  xcopy Perf-%BENCHNAME%*.xml %CORECLR_REPO%\
+  xcopy Perf-%BENCHNAME%*.etl %CORECLR_REPO%\
 ) else (
   type %XMLOUT% | findstr "test name"
   type %XMLOUT% | findstr Duration
diff --git a/tests/src/Common/PerfHarness/PerfHarness.cs b/tests/src/Common/PerfHarness/PerfHarness.cs
new file mode 100644
index 0000000000..b3a0ea48c5
--- /dev/null
+++ b/tests/src/Common/PerfHarness/PerfHarness.cs
@@ -0,0 +1,16 @@
+using System.IO;
+using System.Reflection;
+using System.Collections.Generic;
+using Microsoft.Xunit.Performance.Api;
+
+public class PerfHarness
+{
+    public static void Main(string[] args)
+    {
+        string assemblyName = args[0];
+        using (XunitPerformanceHarness harness = new XunitPerformanceHarness(args))
+        {
+            harness.RunBenchmarks(assemblyName);
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/src/Common/PerfHarness/project.json b/tests/src/Common/PerfHarness/project.json
new file mode 100644
index 0000000000..d920190d11
--- /dev/null
+++ b/tests/src/Common/PerfHarness/project.json
@@ -0,0 +1,19 @@
+{
+  "version": "1.0.0-*",
+  "buildOptions": {
+    "debugType": "portable",
+    "emitEntryPoint": true
+  },
+  "dependencies": {},
+  "frameworks": {
+    "netcoreapp1.1": {
+      "dependencies": {
+        "Microsoft.NETCore.App": {
+          "type": "platform",
+          "version": "1.1.0"
+        },
+        "xunit.performance.api": "1.0.0-alpha-build0049"
+      }
+    }
+  }
+}
diff --git a/tests/src/Common/external/external.depproj b/tests/src/Common/external/external.depproj
index 063c1f94f9..3f7890601d 100644
--- a/tests/src/Common/external/external.depproj
+++ b/tests/src/Common/external/external.depproj
@@ -27,6 +27,11 @@
     <PackageToInclude Include="Microsoft.DotNet.xunit.performance"/>
     <PackageToInclude Include="Microsoft.DotNet.xunit.performance.analysis"/>
     <PackageToInclude Include="Microsoft.DotNet.xunit.performance.runner.Windows"/>
+    <PackageToInclude Include="xunit.performance.api"/>
+    <PackageToInclude Include="xunit.performance.core"/>
+    <PackageToInclude Include="xunit.performance.execution"/>
+    <PackageToInclude Include="xunit.performance.metrics"/>
+    <PackageToInclude Include="Microsoft.Diagnostics.Tracing.TraceEvent"/>
     <PackageToInclude Include="Newtonsoft.Json"/>
     <PackageToInclude Include="Microsoft.CodeAnalysis.Analyzers"/>
     <PackageToInclude Include="Microsoft.CodeAnalysis.Common"/>
diff --git a/tests/src/Common/external/project.json b/tests/src/Common/external/project.json
index 5c0e636c04..63cdc04254 100644
--- a/tests/src/Common/external/project.json
+++ b/tests/src/Common/external/project.json
@@ -1,9 +1,11 @@
 {
   "dependencies": {
     "Microsoft.CodeAnalysis.Compilers": "1.1.1",
-    "Microsoft.DotNet.xunit.performance": "1.0.0-alpha-build0040",
-    "Microsoft.DotNet.xunit.performance.analysis": "1.0.0-alpha-build0040",
-    "Microsoft.DotNet.xunit.performance.runner.Windows": "1.0.0-alpha-build0040",
+    "xunit.performance.api": "1.0.0-alpha-build0049",
+    "xunit.performance.core": "1.0.0-alpha-build0049",
+    "xunit.performance.execution": "1.0.0-alpha-build0049",
+    "xunit.performance.metrics": "1.0.0-alpha-build0049",
+    "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Newtonsoft.Json": "9.0.1",
     "xunit": "2.2.0-beta2-build3300",
     "xunit.console.netcore": "1.0.2-prerelease-00177",
diff --git a/tests/src/JIT/config/benchmark+roslyn/project.json b/tests/src/JIT/config/benchmark+roslyn/project.json
index 0c3bbb94bd..d7569d366a 100644
--- a/tests/src/JIT/config/benchmark+roslyn/project.json
+++ b/tests/src/JIT/config/benchmark+roslyn/project.json
@@ -1,9 +1,11 @@
 {
   "dependencies": {
     "Microsoft.CodeAnalysis.Compilers": "1.1.1",
-    "Microsoft.DotNet.xunit.performance": "1.0.0-alpha-build0040",
-    "Microsoft.DotNet.xunit.performance.analysis": "1.0.0-alpha-build0040",
-    "Microsoft.DotNet.xunit.performance.runner.Windows": "1.0.0-alpha-build0040",
+    "xunit.performance.api": "1.0.0-alpha-build0049",
+    "xunit.performance.core": "1.0.0-alpha-build0049",
+    "xunit.performance.execution": "1.0.0-alpha-build0049",
+    "xunit.performance.metrics": "1.0.0-alpha-build0049",
+    "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-beta-25122-01",
     "System.Console": "4.4.0-beta-24913-02",
     "System.Dynamic.Runtime": "4.4.0-beta-24913-02",
diff --git a/tests/src/JIT/config/benchmark+serialize/project.json b/tests/src/JIT/config/benchmark+serialize/project.json
index 9857e5a58f..0700e10596 100644
--- a/tests/src/JIT/config/benchmark+serialize/project.json
+++ b/tests/src/JIT/config/benchmark+serialize/project.json
@@ -1,8 +1,10 @@
 {
   "dependencies": {
-    "Microsoft.DotNet.xunit.performance": "1.0.0-alpha-build0040",
-    "Microsoft.DotNet.xunit.performance.analysis": "1.0.0-alpha-build0040",
-    "Microsoft.DotNet.xunit.performance.runner.Windows": "1.0.0-alpha-build0040",
+    "xunit.performance.api": "1.0.0-alpha-build0049",
+    "xunit.performance.core": "1.0.0-alpha-build0049",
+    "xunit.performance.execution": "1.0.0-alpha-build0049",
+    "xunit.performance.metrics": "1.0.0-alpha-build0049",
+    "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-beta-25122-01",
     "Newtonsoft.Json": "7.0.1",
     "System.Console": "4.4.0-beta-24913-02",
diff --git a/tests/src/JIT/config/benchmark/project.json b/tests/src/JIT/config/benchmark/project.json
index f9f4e8e770..a44f585327 100644
--- a/tests/src/JIT/config/benchmark/project.json
+++ b/tests/src/JIT/config/benchmark/project.json
@@ -1,8 +1,10 @@
 {
   "dependencies": {
-    "Microsoft.DotNet.xunit.performance": "1.0.0-alpha-build0040",
-    "Microsoft.DotNet.xunit.performance.analysis": "1.0.0-alpha-build0040",
-    "Microsoft.DotNet.xunit.performance.runner.Windows": "1.0.0-alpha-build0040",
+    "xunit.performance.api": "1.0.0-alpha-build0049",
+    "xunit.performance.core": "1.0.0-alpha-build0049",
+    "xunit.performance.execution": "1.0.0-alpha-build0049",
+    "xunit.performance.metrics": "1.0.0-alpha-build0049",
+    "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-beta-25122-01",
     "System.Collections.NonGeneric": "4.4.0-beta-24913-02",
     "System.Console": "4.4.0-beta-24913-02",
diff --git a/tests/src/performance/project.json b/tests/src/performance/project.json
index eb62fa45ce..d52f479cc7 100644
--- a/tests/src/performance/project.json
+++ b/tests/src/performance/project.json
@@ -1,8 +1,10 @@
 {
   "dependencies": {
-    "Microsoft.DotNet.xunit.performance": "1.0.0-alpha-build0040",
-    "Microsoft.DotNet.xunit.performance.analysis": "1.0.0-alpha-build0040",
-    "Microsoft.DotNet.xunit.performance.runner.Windows": "1.0.0-alpha-build0040",
+    "xunit.performance.api": "1.0.0-alpha-build0049",
+    "xunit.performance.core": "1.0.0-alpha-build0049",
+    "xunit.performance.execution": "1.0.0-alpha-build0049",
+    "xunit.performance.metrics": "1.0.0-alpha-build0049",
+    "Microsoft.Diagnostics.Tracing.TraceEvent": "1.0.3-alpha-experimental",
     "Microsoft.NETCore.Platforms": "2.0.0-beta-25122-01",
     "System.Collections.NonGeneric": "4.4.0-beta-24913-02",
     "System.Console": "4.4.0-beta-24913-02",