summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--dependencies.props2
-rw-r--r--perf.groovy22
-rw-r--r--tests/scripts/run-xunit-perf.cmd68
-rwxr-xr-xtests/scripts/run-xunit-perf.sh41
-rw-r--r--tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj2
-rw-r--r--tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj2
-rw-r--r--tests/src/JIT/config/benchmark/benchmark.csproj10
-rw-r--r--tests/src/performance/Scenario/JitBench/IterationData.cs24
-rw-r--r--tests/src/performance/Scenario/JitBench/JitBench.csproj10
-rw-r--r--tests/src/performance/Scenario/JitBench/JitBenchHarness.cs597
-rw-r--r--tests/src/performance/Scenario/JitBench/JitBenchHarnessOptions.cs136
-rw-r--r--tests/src/performance/linkbench/linkbench.cs17
-rw-r--r--tests/src/performance/linkbench/linkbench.csproj16
-rw-r--r--tests/src/performance/performance.csproj15
14 files changed, 656 insertions, 306 deletions
diff --git a/dependencies.props b/dependencies.props
index 78ee86de3e..7cb0ebadc3 100644
--- a/dependencies.props
+++ b/dependencies.props
@@ -37,7 +37,7 @@
<MicrosoftNETCoreRuntimeCoreCLRPackageVersion>2.1.0-preview1-25829-02</MicrosoftNETCoreRuntimeCoreCLRPackageVersion>
<XunitPackageVersion>2.2.0-beta2-build3300</XunitPackageVersion>
<XunitConsoleNetcorePackageVersion>1.0.2-prerelease-00177</XunitConsoleNetcorePackageVersion>
- <XunitPerformanceApiPackageVersion>1.0.0-beta-build0007</XunitPerformanceApiPackageVersion>
+ <XunitPerformanceApiPackageVersion>1.0.0-beta-build0010</XunitPerformanceApiPackageVersion>
<MicrosoftDiagnosticsTracingTraceEventPackageVersion>1.0.3-alpha-experimental</MicrosoftDiagnosticsTracingTraceEventPackageVersion>
<VCRuntimeVersion>1.2.0</VCRuntimeVersion>
</PropertyGroup>
diff --git a/perf.groovy b/perf.groovy
index 6bbec2cb06..c1628b9c33 100644
--- a/perf.groovy
+++ b/perf.groovy
@@ -579,19 +579,11 @@ parallel(
['Windows_NT'].each { os ->
['x64', 'x86'].each { arch ->
['ryujit'].each { jit ->
-
- if (arch == 'x64' && jit == 'legacy_backend') {
- return
- }
-
- ['full_opt', 'min_opt'].each { opt_level ->
+ ['full_opt', 'min_opt', 'tiered'].each { opt_level ->
def architecture = arch
def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
def testEnv = ""
- if (jit == 'legacy_backend') {
- testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
- }
// Set the label.
label('windows_server_2016_clr_perf')
@@ -637,9 +629,15 @@ parallel(
def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
- // Scenario: JitBench
+ // Profile=Off
batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\")
+ batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\Off\\")
+
+ // Profile=On
+ if (opt_level != 'min_opt') {
+ batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
+ batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\On\\")
+ }
}
}
@@ -828,7 +826,7 @@ parallel(
// Scenario: ILLink
batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\")
+ batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\")
}
}
diff --git a/tests/scripts/run-xunit-perf.cmd b/tests/scripts/run-xunit-perf.cmd
index a024fbbe5c..fa95affc89 100644
--- a/tests/scripts/run-xunit-perf.cmd
+++ b/tests/scripts/run-xunit-perf.cmd
@@ -6,6 +6,7 @@
setlocal ENABLEDELAYEDEXPANSION
set ERRORLEVEL=
+ set DOTNET_MULTILEVEL_LOOKUP=0
set BENCHVIEW_RUN_TYPE=local
set CORECLR_REPO=%CD%
set LV_SANDBOX_DIR=%CORECLR_REPO%\bin\sandbox
@@ -24,10 +25,13 @@ setlocal ENABLEDELAYEDEXPANSION
set BENCHVIEW_GROUP=CoreCLR
set HAS_WARMUP_RUN=--drop-first-value
set BETTER=desc
+ set OPT_LEVEL=full_opt
+ set VALID_OPTLEVELS=min_opt full_opt tiered
call :parse_command_line_arguments %*
if defined USAGE_DISPLAYED exit /b %ERRORLEVEL%
+ call :is_valid_optlevel || exit /b 1
call :set_test_architecture || exit /b 1
call :set_collection_config || exit /b 1
call :verify_benchview_tools || exit /b 1
@@ -46,10 +50,6 @@ setlocal ENABLEDELAYEDEXPANSION
)
)
- if not defined OPT_LEVEL (
- set OPT_LEVEL=full_opt
- )
-
if not defined JIT_NAME (
set JIT_NAME=ryujit
)
@@ -88,12 +88,7 @@ setlocal
)
)
- rem setup optimisation level
- if DEFINED OPT_LEVEL (
- if /I "%OPT_LEVEL%" == "min_opt" (
- set COMPlus_JITMinOpts=1
- )
- )
+ call :setup_optimization_level
rem CORE_ROOT environment variable is used by some benchmarks such as Roslyn / CscBench.
set CORE_ROOT=%LV_SANDBOX_DIR%
@@ -108,7 +103,7 @@ setlocal
set LV_CMD=
if defined IS_SCENARIO_TEST (
- set "LV_CMD=corerun.exe "%LV_SANDBOX_DIR%\%BENCHNAME%.%TEST_FILE_EXT%" --perf:outputdir "%LV_BENCHMARKS_OUTPUT_DIR%" --perf:runid "%LV_RUNID%" --target-architecture "%TEST_ARCHITECTURE%""
+ set "LV_CMD=%STABILITY_PREFIX% corerun.exe "%LV_SANDBOX_DIR%\%BENCHNAME%.%TEST_FILE_EXT%" --perf:outputdir "%LV_BENCHMARKS_OUTPUT_DIR%" --perf:runid "%LV_RUNID%" --target-architecture "%TEST_ARCHITECTURE%" --perf:collect %COLLECTION_FLAGS%"
) else (
set "LV_CMD=%STABILITY_PREFIX% corerun.exe PerfHarness.dll "%LV_SANDBOX_DIR%\%BENCHNAME%.%TEST_FILE_EXT%" --perf:outputdir "%LV_BENCHMARKS_OUTPUT_DIR%" --perf:runid "%LV_RUNID%" --perf:collect %COLLECTION_FLAGS%"
)
@@ -365,7 +360,7 @@ rem ****************************************************************************
rem files on the current working directory.
set LV_PATTERN="%LV_BENCHMARKS_OUTPUT_DIR%\%LV_RUNID%-%BENCHNAME%.xml"
rem The first pattern is the general case, the second is used by IlLink
- if defined IS_SCENARIO_TEST set LV_PATTERN="%LV_RUNID%-%BENCHNAME%.xml" "%LV_RUNID%-*-%BENCHNAME%.xml"
+ if defined IS_SCENARIO_TEST set LV_PATTERN="%LV_BENCHMARKS_OUTPUT_DIR%\%LV_RUNID%-%BENCHNAME%.xml" "%LV_BENCHMARKS_OUTPUT_DIR%\%LV_RUNID%-*-%BENCHNAME%.xml"
for %%f in (%LV_PATTERN%) do (
if exist "%%~f" (
@@ -383,6 +378,11 @@ rem ****************************************************************************
rem Generates BenchView's submission data and upload it
rem ****************************************************************************
setlocal
+ if not exist measurement.json (
+ call :print_error measurement.json does not exist. There is no data to be uploaded.
+ exit /b 1
+ )
+
set LV_SUBMISSION_ARGS=
set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --build "%CORECLR_REPO%\build.json"
set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --machine-data "%CORECLR_REPO%\machinedata.json"
@@ -419,7 +419,7 @@ rem ****************************************************************************
rem Script's usage.
rem ****************************************************************************
set USAGE_DISPLAYED=1
- echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-generateBenchviewData] ^<path_to_benchview_tools^> [-warmup] [-better] ^<asc ^| desc^> [-group] ^<group^> [-runtype] ^<rolling^|private^> [-scenarioTest] [-collectionFlags] ^<default^+CacheMisses^+InstructionRetired^+BranchMispredictions^+gcapi^> [-outputdir] ^<outputdir^>
+ echo run-xunit-perf.cmd -testBinLoc ^<path_to_tests^> [-library] [-arch] ^<x86^|x64^> [-configuration] ^<Release^|Debug^> [-generateBenchviewData] ^<path_to_benchview_tools^> [-warmup] [-better] ^<asc ^| desc^> [-group] ^<group^> [-runtype] ^<rolling^|private^> [-scenarioTest] [-collectionFlags] ^<default^+CacheMisses^+InstructionRetired^+BranchMispredictions^+gcapi^> [-outputdir] ^<outputdir^> [-optLevel] ^<%VALID_OPTLEVELS: =^|%^>
echo/
echo For the path to the tests you can pass a parent directory and the script will grovel for
echo all tests in subdirectories and run them.
@@ -436,6 +436,7 @@ rem ****************************************************************************
echo -scenarioTest should be included if you are running a scenario benchmark.
echo -outputdir Specifies the directory where the generated performance output will be saved.
echo -collectionFlags This is used to specify what collectoin flags get passed to the performance
+ echo -optLevel Specifies the optimization level to be used by the jit.
echo harness that is doing the test running. If this is not specified we only use stopwatch.
echo Other flags are "default", which is the whatever the test being run specified, "CacheMisses",
echo "BranchMispredictions", and "InstructionsRetired".
@@ -459,6 +460,47 @@ rem ****************************************************************************
echo/[%DATE%][%TIME:~0,-3%] %*
exit /b %ERRORLEVEL%
+:is_valid_optlevel
+rem ****************************************************************************
+rem Validates the optlevel flag set by the user.
+rem ****************************************************************************
+setlocal
+ if not defined OPT_LEVEL (
+ call :print_error OPT_LEVEL is undefined.
+ exit /b 1
+ )
+
+ set "LV_IS_VALID_OPTLEVEL="
+ for %%i in (%VALID_OPTLEVELS%) do (
+ if /i "%%~i" == "%OPT_LEVEL%" (
+ set "LV_IS_VALID_OPTLEVEL=1"
+ )
+ )
+
+ if not defined LV_IS_VALID_OPTLEVEL (
+ call :print_error Unknown OPT_LEVEL=%OPT_LEVEL%
+ exit /b 1
+ )
+endlocal& exit /b 0
+
+:setup_optimization_level
+rem ****************************************************************************
+rem Setup the appropriate environment variables needed for the selected
+rem optlevel.
+rem ****************************************************************************
+ set "COMPlus_JITMinOpts="
+ set "COMPLUS_EXPERIMENTAL_TieredCompilation="
+
+ if /I "%OPT_LEVEL%" == "min_opt" (
+ set COMPlus_JITMinOpts=1
+ exit /b 0
+ )
+ if /I "%OPT_LEVEL%" == "tiered" (
+ set COMPLUS_EXPERIMENTAL_TieredCompilation=1
+ exit /b 0
+ )
+exit /b 0
+
:run_cmd
rem ****************************************************************************
rem Function wrapper used to send the command line being executed to the
diff --git a/tests/scripts/run-xunit-perf.sh b/tests/scripts/run-xunit-perf.sh
index c6700f4bba..677e60a39b 100755
--- a/tests/scripts/run-xunit-perf.sh
+++ b/tests/scripts/run-xunit-perf.sh
@@ -43,6 +43,8 @@ function print_usage {
echo ' --benchViewOS=<os> : Specify the os that will be used to insert data into Benchview.'
echo ' --runType=<local|private|rolling> : Specify the runType for Benchview. [Default: local]'
echo ' --outputdir : Specifies the directory where the generated performance output will be saved.'
+ echo ' --optLevel=<min_opt|full_opt|tiered>'
+ echo ' : Specifies the optimization level to be used by the jit.'
}
# libExtension determines extension for dynamic library files
@@ -96,6 +98,37 @@ function handle_ctrl_c {
exit_with_error "$errorSource" "Test run aborted by Ctrl+C."
}
+function is_valid_optlevel {
+ if [ -z "$optLevel" ]; then
+ echo "[ERROR] --optLevel is required."
+ return 1
+ fi
+
+ declare -A valid_optlevels=(
+ [min_opt]=1 [full_opt]=1 [tiered]=1
+ )
+ [[ -n "${valid_optlevels[$optLevel]}" ]] || {
+ echo "[ERROR] Specified an unknown optLevel=$optLevel";
+ return 1;
+ }
+ return 0
+}
+
+function setup_optimization_level {
+ unset COMPlus_JITMinOpts
+ unset COMPLUS_EXPERIMENTAL_TieredCompilation
+
+ if [ "$optLevel" == "min_opt" ]; then
+ export COMPlus_JITMinOpts=1
+ return 0
+ fi
+ if [ "$optLevel" == "tiered" ]; then
+ export COMPLUS_EXPERIMENTAL_TieredCompilation=1
+ return 0
+ fi
+ return 0
+}
+
# Register the Ctrl-C handler
trap handle_ctrl_c INT
@@ -204,6 +237,8 @@ function copy_test_native_bin_to_test_root {
done
}
+export DOTNET_MULTILEVEL_LOOKUP=0
+
# Exit code constants
readonly EXIT_CODE_SUCCESS=0 # Script ran normally.
readonly EXIT_CODE_EXCEPTION=1 # Script exited because something exceptional happened (e.g. bad arguments, Ctrl-C interrupt).
@@ -298,6 +333,7 @@ if [ ! -z "$BENCHVIEW_TOOLS_PATH" ] && { [ ! -d "$BENCHVIEW_TOOLS_PATH" ]; }; th
echo BenchView path: "$BENCHVIEW_TOOLS_PATH" was specified, but it does not exist.
exit $EXIT_CODE_EXCEPTION
fi
+is_valid_optlevel || exit $EXIT_CODE_EXCEPTION
if [ "$collectionflags" == "stopwatch" ]; then
perfCollection=Off
else
@@ -319,10 +355,7 @@ if [ ! -d "$benchmarksOutputDir" ]; then
mkdir -p "$benchmarksOutputDir" || { echo "Failed to delete $benchmarksOutputDir"; exit 1; }
fi
-# Set minopts
-if ["$optLevel" == "min_opt"]; then
- export COMPlus_JITMinOpts=1
-fi
+setup_optimization_level || exit $EXIT_CODE_EXCEPTION
cd $CORE_ROOT
diff --git a/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj b/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj
index c46e42ad1d..77cba12414 100644
--- a/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj
+++ b/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj
@@ -26,7 +26,7 @@
<Version>$(MicrosoftDiagnosticsTracingTraceEventPackageVersion)</Version>
</PackageReference>
<PackageReference Include="Microsoft.NETCore.Platforms">
- <Version>2.0.0-preview2-25302-03</Version>
+ <Version>$(MicrosoftNETCorePlatformsPackageVersion)</Version>
</PackageReference>
<PackageReference Include="System.Console">
<Version>4.4.0-beta-24913-02</Version>
diff --git a/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj b/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj
index 1b85f9b91a..d554c165f6 100644
--- a/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj
+++ b/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj
@@ -23,7 +23,7 @@
<Version>$(MicrosoftDiagnosticsTracingTraceEventPackageVersion)</Version>
</PackageReference>
<PackageReference Include="Microsoft.NETCore.Platforms">
- <Version>2.0.0-preview2-25302-03</Version>
+ <Version>$(MicrosoftNETCorePlatformsPackageVersion)</Version>
</PackageReference>
<PackageReference Include="Newtonsoft.Json">
<Version>7.0.1</Version>
diff --git a/tests/src/JIT/config/benchmark/benchmark.csproj b/tests/src/JIT/config/benchmark/benchmark.csproj
index 1c4f90faca..5ade686330 100644
--- a/tests/src/JIT/config/benchmark/benchmark.csproj
+++ b/tests/src/JIT/config/benchmark/benchmark.csproj
@@ -23,7 +23,7 @@
<Version>$(MicrosoftDiagnosticsTracingTraceEventPackageVersion)</Version>
</PackageReference>
<PackageReference Include="Microsoft.NETCore.Platforms">
- <Version>2.0.0-preview2-25302-03</Version>
+ <Version>$(MicrosoftNETCorePlatformsPackageVersion)</Version>
</PackageReference>
<PackageReference Include="System.Collections.NonGeneric">
<Version>4.4.0-beta-24913-02</Version>
@@ -44,10 +44,10 @@
<Version>4.4.0-beta-24913-02</Version>
</PackageReference>
<PackageReference Include="System.Memory">
- <Version>4.5.0-preview2-25504-02</Version>
+ <Version>$(MicrosoftPrivateCoreFxNETCoreAppPackageVersion)</Version>
</PackageReference>
<PackageReference Include="System.Numerics.Vectors">
- <Version>4.4.0-preview2-25302-03</Version>
+ <Version>$(MicrosoftPrivateCoreFxNETCoreAppPackageVersion)</Version>
</PackageReference>
<PackageReference Include="System.Reflection">
<Version>4.4.0-beta-24913-02</Version>
@@ -56,13 +56,13 @@
<Version>4.4.0-beta-24913-02</Version>
</PackageReference>
<PackageReference Include="System.Reflection.TypeExtensions">
- <Version>4.4.0-preview2-25302-03</Version>
+ <Version>$(MicrosoftPrivateCoreFxNETCoreAppPackageVersion)</Version>
</PackageReference>
<PackageReference Include="System.Runtime">
<Version>4.4.0-beta-24913-02</Version>
</PackageReference>
<PackageReference Include="System.Runtime.CompilerServices.Unsafe">
- <Version>4.4.0-preview2-25302-03</Version>
+ <Version>$(MicrosoftPrivateCoreFxNETCoreAppPackageVersion)</Version>
</PackageReference>
<PackageReference Include="System.Runtime.Extensions">
<Version>4.4.0-beta-24913-02</Version>
diff --git a/tests/src/performance/Scenario/JitBench/IterationData.cs b/tests/src/performance/Scenario/JitBench/IterationData.cs
new file mode 100644
index 0000000000..0c23f4fd63
--- /dev/null
+++ b/tests/src/performance/Scenario/JitBench/IterationData.cs
@@ -0,0 +1,24 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Xunit.Performance.Api;
+
+namespace JitBench
+{
+ /// <summary>
+ /// Interface used to buffer each scenario iteration/run for later post processing.
+ /// </summary>
+ internal class IterationData
+ {
+ public ScenarioExecutionResult ScenarioExecutionResult { get; set; }
+
+ public string StandardOutput { get; set; }
+
+ public double StartupTime { get; set; }
+
+ public double FirstRequestTime { get; set; }
+
+ public double SteadystateTime { get; set; }
+ }
+}
diff --git a/tests/src/performance/Scenario/JitBench/JitBench.csproj b/tests/src/performance/Scenario/JitBench/JitBench.csproj
index 02ce7700f4..04077f4900 100644
--- a/tests/src/performance/Scenario/JitBench/JitBench.csproj
+++ b/tests/src/performance/Scenario/JitBench/JitBench.csproj
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
<PropertyGroup>
@@ -12,8 +12,8 @@
<FileAlignment>512</FileAlignment>
<ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
- <NuGetTargetMoniker>.NETStandard,Version=v1.5</NuGetTargetMoniker>
- <NuGetTargetMonikerShort>netstandard1.5</NuGetTargetMonikerShort>
+ <NuGetTargetMoniker>.NETStandard,Version=v1.6</NuGetTargetMoniker>
+ <NuGetTargetMonikerShort>netstandard1.6</NuGetTargetMonikerShort>
</PropertyGroup>
<!-- Default configurations to help VS understand the configurations -->
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' " />
@@ -29,7 +29,9 @@
</CodeAnalysisDependentAssemblyPaths>
</ItemGroup>
<ItemGroup>
+ <Compile Include="IterationData.cs" />
<Compile Include="JitBenchHarness.cs" />
+ <Compile Include="JitBenchHarnessOptions.cs" />
</ItemGroup>
<ItemGroup>
<Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
@@ -38,4 +40,4 @@
<PropertyGroup>
<ProjectAssetsFile>..\..\obj\project.assets.json</ProjectAssetsFile>
</PropertyGroup>
-</Project>
+</Project> \ No newline at end of file
diff --git a/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs b/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs
index 035b74f56f..596052437d 100644
--- a/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs
+++ b/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs
@@ -2,9 +2,8 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
-using CommandLine;
-using CommandLine.Text;
using Microsoft.Xunit.Performance.Api;
+using Microsoft.Xunit.Performance.Api.Profilers.Etw;
using System;
using System.Collections.Generic;
using System.Diagnostics;
@@ -12,12 +11,12 @@ using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net.Http;
-using System.Reflection;
+using System.Text;
using System.Text.RegularExpressions;
namespace JitBench
{
- class Program
+ class JitBenchHarness
{
static void Main(string[] args)
{
@@ -30,6 +29,10 @@ namespace JitBench
ProcessStartInfo startInfo = options.UseExistingSetup ? UseExistingSetup() : CreateNewSetup();
string scenarioName = "MusicStore";
+ if (!startInfo.Environment.ContainsKey("DOTNET_MULTILEVEL_LOOKUP"))
+ throw new InvalidOperationException("DOTNET_MULTILEVEL_LOOKUP was not defined.");
+ if (startInfo.Environment["DOTNET_MULTILEVEL_LOOKUP"] != "0")
+ throw new InvalidOperationException("DOTNET_MULTILEVEL_LOOKUP was not set to 0.");
if (options.EnableTiering)
{
@@ -55,30 +58,60 @@ namespace JitBench
scenarioName += " NoNgen";
}
- var scenarioConfiguration = CreateScenarioConfiguration();
-
- h.RunScenario(startInfo, () => { PrintHeader(scenarioName); }, PostIteration, PostProcessing, scenarioConfiguration);
+ var program = new JitBenchHarness("JitBench");
+ try
+ {
+ var scenarioConfiguration = new ScenarioConfiguration(TimeSpan.FromMilliseconds(60000), startInfo) {
+ Iterations = (int)options.Iterations,
+ PreIterationDelegate = program.PreIteration,
+ PostIterationDelegate = program.PostIteration,
+ };
+ var processesOfInterest = new string[] {
+ "dotnet.exe",
+ };
+ var modulesOfInterest = new string[] {
+ "Anonymously Hosted DynamicMethods Assembly",
+ "clrjit.dll",
+ "coreclr.dll",
+ "dotnet.exe",
+ "MusicStore.dll",
+ "ntoskrnl.exe",
+ "System.Private.CoreLib.dll",
+ "Unknown",
+ };
+
+ if (!File.Exists(startInfo.FileName))
+ throw new FileNotFoundException(startInfo.FileName);
+ if (!Directory.Exists(startInfo.WorkingDirectory))
+ throw new DirectoryNotFoundException(startInfo.WorkingDirectory);
+
+ h.RunScenario(scenarioConfiguration, teardownDelegate: () => {
+ return program.PostRun("MusicStore", processesOfInterest, modulesOfInterest);
+ });
+ }
+ catch
+ {
+ Console.WriteLine(program.StandardOutput);
+ Console.WriteLine(program.StandardError);
+ throw;
+ }
}
}
- private static ScenarioConfiguration CreateScenarioConfiguration()
+ public JitBenchHarness(string scenarioBenchmarkName)
{
- var config = new ScenarioConfiguration(TimeSpan.FromMilliseconds(60000))
- {
- Iterations = (int)s_iterations
- };
- return config;
+ _scenarioBenchmarkName = scenarioBenchmarkName;
+ _stdout = new StringBuilder();
+ _stderr = new StringBuilder();
+ IterationsData = new List<IterationData>();
}
+ public string StandardOutput => _stdout.ToString();
+
+ public string StandardError => _stderr.ToString();
+
private static void SetupStatics(JitBenchHarnessOptions options)
{
- // Set variables we will need to store results.
- s_iterations = options.Iterations;
- s_iteration = 0;
- s_startupTimes = new double[s_iterations];
- s_requestTimes = new double[s_iterations];
- s_steadystateTimes = new double[s_iterations];
-
s_temporaryDirectory = options.IntermediateOutputDirectory;
s_targetArchitecture = options.TargetArchitecture;
if (string.IsNullOrWhiteSpace(s_targetArchitecture))
@@ -124,7 +157,7 @@ namespace JitBench
private static void InstallSharedRuntime()
{
- var psi = new ProcessStartInfo() {
+ var psi = new ProcessStartInfo {
WorkingDirectory = s_jitBenchDevDirectory,
FileName = @"powershell.exe",
Arguments = $".\\Dotnet-Install.ps1 -SharedRuntime -InstallDir .dotnet -Channel master -Architecture {s_targetArchitecture}"
@@ -134,7 +167,7 @@ namespace JitBench
private static IDictionary<string, string> InstallDotnet()
{
- var psi = new ProcessStartInfo() {
+ var psi = new ProcessStartInfo {
WorkingDirectory = s_jitBenchDevDirectory,
FileName = @"powershell.exe",
Arguments = $".\\Dotnet-Install.ps1 -InstallDir .dotnet -Channel master -Architecture {s_targetArchitecture}"
@@ -192,7 +225,7 @@ namespace JitBench
private static IDictionary<string, string> GenerateStore(IDictionary<string, string> environment)
{
// This step generates some environment variables needed later.
- var psi = new ProcessStartInfo() {
+ var psi = new ProcessStartInfo {
WorkingDirectory = s_jitBenchDevDirectory,
FileName = "powershell.exe",
Arguments = $"-Command \".\\AspNet-GenerateStore.ps1 -InstallDir .store -Architecture {s_targetArchitecture} -Runtime win7-{s_targetArchitecture}; gi env:JITBENCH_*, env:DOTNET_SHARED_STORE | %{{ \\\"$($_.Name)=$($_.Value)\\\" }} 1>>{EnvironmentFileName}\""
@@ -224,9 +257,20 @@ namespace JitBench
return environment;
}
+ private static void DotNetInfo(string workingDirectory, string dotnetFileName, IDictionary<string, string> environment)
+ {
+ var psi = new ProcessStartInfo {
+ WorkingDirectory = workingDirectory,
+ FileName = dotnetFileName,
+ Arguments = "--info"
+ };
+
+ LaunchProcess(psi, 60000, environment);
+ }
+
private static void RestoreMusicStore(string workingDirectory, string dotnetFileName, IDictionary<string, string> environment)
{
- var psi = new ProcessStartInfo() {
+ var psi = new ProcessStartInfo {
WorkingDirectory = workingDirectory,
FileName = dotnetFileName,
Arguments = "restore"
@@ -237,10 +281,14 @@ namespace JitBench
private static void PublishMusicStore(string workingDirectory, string dotnetFileName, IDictionary<string, string> environment)
{
- var psi = new ProcessStartInfo() {
+ var manifest = environment["JITBENCH_ASPNET_MANIFEST"];
+ if (!File.Exists(manifest))
+ throw new FileNotFoundException(manifest);
+
+ var psi = new ProcessStartInfo {
WorkingDirectory = workingDirectory,
- FileName = "cmd.exe",
- Arguments = $"/C \"{dotnetFileName} publish -c Release -f {JitBenchTargetFramework} --manifest %JITBENCH_ASPNET_MANIFEST% /p:MvcRazorCompileOnPublish=false\""
+ FileName = dotnetFileName,
+ Arguments = $"publish -c Release -f {JitBenchTargetFramework} --manifest \"{manifest}\" /p:MvcRazorCompileOnPublish=false -o \"{MusicStorePublishDirectory}\""
};
LaunchProcess(psi, 300000, environment);
@@ -250,24 +298,30 @@ namespace JitBench
private static IDictionary<string, string> GetInitialEnvironment()
{
// TODO: This is currently hardcoded, but we could probably pull it from the powershell cmdlet call.
- var environment = new Dictionary<string, string> { { "PATH", $"{Path.Combine(s_jitBenchDevDirectory, ".dotnet")};{Environment.GetEnvironmentVariable("PATH")}" } };
+ var dotnetPath = Path.Combine(s_jitBenchDevDirectory, ".dotnet");
+ var dotnetexe = Path.Combine(dotnetPath, "dotnet.exe");
+ if (!File.Exists(dotnetexe))
+ throw new FileNotFoundException(dotnetexe);
+
+ var environment = new Dictionary<string, string> {
+ { "DOTNET_MULTILEVEL_LOOKUP", "0" },
+ { "PATH", $"{dotnetPath};{Environment.GetEnvironmentVariable("PATH")}" }
+ };
return environment;
}
- private static ProcessStartInfo UseExistingSetup()
- {
- PrintHeader("Using existing SETUP");
-
- var environment = GetInitialEnvironment();
- environment = GetEnvironment(environment, Path.Combine(s_jitBenchDevDirectory, EnvironmentFileName));
- ValidateEnvironment(environment);
+ private static string MusicStorePublishDirectory =>
+ Path.Combine(s_musicStoreDirectory, "bin", s_targetArchitecture, "Release", JitBenchTargetFramework, "publish");
- var psi = new ProcessStartInfo()
- {
- FileName = "cmd.exe",
- Arguments = $"/C \"{s_dotnetProcessFileName} MusicStore.dll 1>{MusicStoreRedirectedStandardOutputFileName}\"",
- WorkingDirectory = Path.Combine(s_musicStoreDirectory, "bin", "Release", JitBenchTargetFramework, "publish")
+ private static ProcessStartInfo CreateJitBenchStartInfo(IDictionary<string, string> environment)
+ {
+ var psi = new ProcessStartInfo {
+ Arguments = "MusicStore.dll",
+ FileName = s_dotnetProcessFileName,
+ RedirectStandardError = true,
+ RedirectStandardOutput = true,
+ WorkingDirectory = MusicStorePublishDirectory,
};
foreach (KeyValuePair<string, string> pair in environment)
@@ -275,6 +329,18 @@ namespace JitBench
return psi;
}
+
+ private static ProcessStartInfo UseExistingSetup()
+ {
+ PrintHeader("Using existing SETUP");
+
+ IDictionary<string, string> environment = GetInitialEnvironment();
+ environment = GetEnvironment(environment, Path.Combine(s_jitBenchDevDirectory, EnvironmentFileName));
+ ValidateEnvironment(environment);
+
+ return CreateJitBenchStartInfo(environment);
+ }
+
private static ProcessStartInfo CreateNewSetup()
{
PrintHeader("Starting SETUP");
@@ -292,130 +358,289 @@ namespace JitBench
ModifySharedFramework();
+ DotNetInfo(s_musicStoreDirectory, s_dotnetProcessFileName, environment);
RestoreMusicStore(s_musicStoreDirectory, s_dotnetProcessFileName, environment);
PublishMusicStore(s_musicStoreDirectory, s_dotnetProcessFileName, environment);
- var psi = new ProcessStartInfo() {
- FileName = "cmd.exe",
- Arguments = $"/C \"{s_dotnetProcessFileName} MusicStore.dll 1>{MusicStoreRedirectedStandardOutputFileName}\"",
- WorkingDirectory = Path.Combine(s_musicStoreDirectory, "bin", "Release", JitBenchTargetFramework, "publish")
- };
-
- foreach (KeyValuePair<string, string> pair in environment)
- psi.Environment.Add(pair.Key, pair.Value);
-
- return psi;
+ return CreateJitBenchStartInfo(environment);
}
private static void ValidateEnvironment(IDictionary<string, string> environment)
{
var expectedVariables = new string[] {
+ "DOTNET_MULTILEVEL_LOOKUP",
"PATH",
"JITBENCH_ASPNET_MANIFEST",
"JITBENCH_FRAMEWORK_VERSION",
"JITBENCH_ASPNET_VERSION",
- "DOTNET_SHARED_STORE"
+ "DOTNET_SHARED_STORE",
};
if (expectedVariables.Except(environment.Keys, StringComparer.OrdinalIgnoreCase).Any())
throw new Exception("Missing expected environment variables.");
+
+ Console.WriteLine("**********************************************************************");
+ foreach (var env in expectedVariables)
+ Console.WriteLine($" {env}={environment[env]}");
+ Console.WriteLine("**********************************************************************");
}
- private const string MusicStoreRedirectedStandardOutputFileName = "measures.txt";
private const string JitBenchRepoUrl = "https://github.com/aspnet/JitBench";
private const string JitBenchCommitSha1Id = "b7e7b786c60daa255aacaea85006afe4d4ec8306";
private const string JitBenchTargetFramework = "netcoreapp2.1";
private const string EnvironmentFileName = "JitBenchEnvironment.txt";
- private static void PostIteration()
+ private void PreIteration(Scenario scenario)
{
- var path = Path.Combine(s_jitBenchDevDirectory, "src", "MusicStore", "bin", "Release", JitBenchTargetFramework, "publish");
- path = Path.Combine(path, MusicStoreRedirectedStandardOutputFileName);
+ PrintHeader("Setting up data standard output/error process handlers.");
+
+ _stderr.Clear();
+ _stdout.Clear();
+
+ if (scenario.Process.StartInfo.RedirectStandardError)
+ {
+ scenario.Process.ErrorDataReceived += (object sender, DataReceivedEventArgs errorLine) => {
+ if (!string.IsNullOrEmpty(errorLine.Data))
+ _stderr.AppendLine(errorLine.Data);
+ };
+ }
+
+ if (scenario.Process.StartInfo.RedirectStandardInput)
+ throw new NotImplementedException("RedirectStandardInput has not been implemented yet.");
+
+ if (scenario.Process.StartInfo.RedirectStandardOutput)
+ {
+ scenario.Process.OutputDataReceived += (object sender, DataReceivedEventArgs outputLine) => {
+ if (!string.IsNullOrEmpty(outputLine.Data))
+ _stdout.AppendLine(outputLine.Data);
+ };
+ }
+ }
+
+ private void PostIteration(ScenarioExecutionResult scenarioExecutionResult)
+ {
+ PrintHeader("Processing iteration results.");
double? startupTime = null;
- double? requestTime = null;
+ double? firstRequestTime = null;
double? steadyStateAverageTime = null;
- foreach (string line in File.ReadLines(path))
+
+ using (var reader = new StringReader(_stdout.ToString()))
{
- Match match = Regex.Match(line, @"^Server started in (\d+)ms$");
- if (match.Success && match.Groups.Count == 2)
+ string line;
+ while ((line = reader.ReadLine()) != null)
{
- startupTime = Convert.ToDouble(match.Groups[1].Value);
- continue;
- }
+ Match match = Regex.Match(line, @"^Server started in (\d+)ms$");
+ if (match.Success && match.Groups.Count == 2)
+ {
+ startupTime = Convert.ToDouble(match.Groups[1].Value);
+ continue;
+ }
- match = Regex.Match(line, @"^Request took (\d+)ms$");
- if (match.Success && match.Groups.Count == 2)
- {
- requestTime = Convert.ToDouble(match.Groups[1].Value);
- continue;
- }
+ match = Regex.Match(line, @"^Request took (\d+)ms$");
+ if (match.Success && match.Groups.Count == 2)
+ {
+ firstRequestTime = Convert.ToDouble(match.Groups[1].Value);
+ continue;
+ }
- match = Regex.Match(line, @"^Steadystate average response time: (\d+)ms$");
- if (match.Success && match.Groups.Count == 2)
- {
- steadyStateAverageTime = Convert.ToDouble(match.Groups[1].Value);
- break;
+ match = Regex.Match(line, @"^Steadystate average response time: (\d+)ms$");
+ if (match.Success && match.Groups.Count == 2)
+ {
+ steadyStateAverageTime = Convert.ToDouble(match.Groups[1].Value);
+ break;
+ }
}
}
if (!startupTime.HasValue)
throw new Exception("Startup time was not found.");
- if (!requestTime.HasValue)
+ if (!firstRequestTime.HasValue)
throw new Exception("First Request time was not found.");
if (!steadyStateAverageTime.HasValue)
throw new Exception("Steady state average response time not found.");
- s_startupTimes[s_iteration] = startupTime.Value;
- s_requestTimes[s_iteration] = requestTime.Value;
- s_steadystateTimes[s_iteration] = steadyStateAverageTime.Value;
+ IterationsData.Add(new IterationData {
+ ScenarioExecutionResult = scenarioExecutionResult,
+ StandardOutput = _stdout.ToString(),
+ StartupTime = startupTime.Value,
+ FirstRequestTime = firstRequestTime.Value,
+ SteadystateTime = steadyStateAverageTime.Value,
+ });
- PrintRunningStepInformation($"{s_iteration} Server started in {s_startupTimes[s_iteration]}ms");
- PrintRunningStepInformation($"{s_iteration} Request took {s_requestTimes[s_iteration]}ms");
- PrintRunningStepInformation($"{s_iteration} Cold start time (server start + first request time): {s_startupTimes[s_iteration] + s_requestTimes[s_iteration]}ms");
- PrintRunningStepInformation($"{s_iteration} Average steady state response {s_steadystateTimes[s_iteration]}ms");
+ PrintRunningStepInformation($"({IterationsData.Count}) Server started in {IterationsData.Last().StartupTime}ms");
+ PrintRunningStepInformation($"({IterationsData.Count}) Request took {IterationsData.Last().FirstRequestTime}ms");
+ PrintRunningStepInformation($"({IterationsData.Count}) Cold start time (server start + first request time): {IterationsData.Last().StartupTime + IterationsData.Last().FirstRequestTime}ms");
+ PrintRunningStepInformation($"({IterationsData.Count}) Average steady state response {IterationsData.Last().SteadystateTime}ms");
- ++s_iteration;
+ _stdout.Clear();
+ _stderr.Clear();
}
- private static ScenarioBenchmark PostProcessing()
+ private ScenarioBenchmark PostRun(
+ string scenarioTestModelName,
+ IReadOnlyCollection<string> processesOfInterest,
+ IReadOnlyCollection<string> modulesOfInterest)
{
- PrintHeader("Starting POST");
+ PrintHeader("Post-Processing scenario data.");
- var scenarioBenchmark = new ScenarioBenchmark("MusicStore") {
- Namespace = "JitBench"
- };
+ var scenarioBenchmark = new ScenarioBenchmark(_scenarioBenchmarkName);
- // Create (measured) test entries for this scenario.
- var startup = new ScenarioTestModel("Startup");
- scenarioBenchmark.Tests.Add(startup);
+ foreach (var iter in IterationsData)
+ {
+ var scenarioExecutionResult = iter.ScenarioExecutionResult;
+ var scenarioTestModel = scenarioBenchmark.Tests
+ .SingleOrDefault(t => t.Name == scenarioTestModelName);
- var request = new ScenarioTestModel("First Request");
- scenarioBenchmark.Tests.Add(request);
+ if (scenarioTestModel == null)
+ {
+ scenarioTestModel = new ScenarioTestModel(scenarioTestModelName);
+ scenarioBenchmark.Tests.Add(scenarioTestModel);
- // TODO: add response time once jit bench is updated to
- // report more reasonable numbers.
+ // Add measured metrics to each test.
+ scenarioTestModel.Performance.Metrics.Add(ElapsedTimeMilliseconds);
+ }
- // Add measured metrics to each test.
- startup.Performance.Metrics.Add(new MetricModel {
- Name = "Duration",
- DisplayName = "Duration",
- Unit = "ms"
- });
- request.Performance.Metrics.Add(new MetricModel {
- Name = "Duration",
- DisplayName = "Duration",
- Unit = "ms"
- });
+ scenarioTestModel.Performance.IterationModels.Add(new IterationModel {
+ Iteration = new Dictionary<string, double> {
+ { ElapsedTimeMilliseconds.Name, (scenarioExecutionResult.ProcessExitInfo.ExitTime - scenarioExecutionResult.ProcessExitInfo.StartTime).TotalMilliseconds},
+ }
+ });
- for (int i = 0; i < s_iterations; ++i)
+ // Create (measured) test entries for this scenario.
+ var startup = scenarioBenchmark.Tests
+ .SingleOrDefault(t => t.Name == "Startup" && t.Namespace == scenarioTestModel.Name);
+ if (startup == null)
+ {
+ startup = new ScenarioTestModel("Startup") {
+ Namespace = scenarioTestModel.Name,
+ };
+ scenarioBenchmark.Tests.Add(startup);
+
+ // Add measured metrics to each test.
+ startup.Performance.Metrics.Add(ElapsedTimeMilliseconds);
+ }
+
+ var firstRequest = scenarioBenchmark.Tests
+ .SingleOrDefault(t => t.Name == "First Request" && t.Namespace == scenarioTestModel.Name);
+ if (firstRequest == null)
+ {
+ firstRequest = new ScenarioTestModel("First Request") {
+ Namespace = scenarioTestModel.Name,
+ };
+ scenarioBenchmark.Tests.Add(firstRequest);
+
+ // Add measured metrics to each test.
+ firstRequest.Performance.Metrics.Add(ElapsedTimeMilliseconds);
+ }
+
+ startup.Performance.IterationModels.Add(new IterationModel {
+ Iteration = new Dictionary<string, double> {
+ { ElapsedTimeMilliseconds.Name, iter.StartupTime },
+ },
+ });
+
+ firstRequest.Performance.IterationModels.Add(new IterationModel {
+ Iteration = new Dictionary<string, double> {
+ { ElapsedTimeMilliseconds.Name, iter.FirstRequestTime },
+ },
+ });
+
+ if (!string.IsNullOrWhiteSpace(iter.ScenarioExecutionResult.EventLogFileName) &&
+ File.Exists(iter.ScenarioExecutionResult.EventLogFileName))
+ {
+ // Adding ETW data.
+ scenarioBenchmark = AddEtwData(
+ scenarioBenchmark, iter.ScenarioExecutionResult, processesOfInterest, modulesOfInterest);
+ }
+ }
+
+ for (int i = scenarioBenchmark.Tests.Count - 1; i >= 0; i--)
+ if (scenarioBenchmark.Tests[i].Performance.IterationModels.All(iter => iter.Iteration.Count == 0))
+ scenarioBenchmark.Tests.RemoveAt(i);
+
+ return scenarioBenchmark;
+ }
+
+ private static ScenarioBenchmark AddEtwData(
+ ScenarioBenchmark scenarioBenchmark,
+ ScenarioExecutionResult scenarioExecutionResult,
+ IReadOnlyCollection<string> processesOfInterest,
+ IReadOnlyCollection<string> modulesOfInterest)
+ {
+ var metricModels = scenarioExecutionResult.PerformanceMonitorCounters
+ .Select(pmc => new MetricModel {
+ DisplayName = pmc.DisplayName,
+ Name = pmc.Name,
+ Unit = pmc.Unit,
+ });
+
+ // Get the list of processes of interest.
+ Console.WriteLine($"Parsing: {scenarioExecutionResult.EventLogFileName}");
+ var processes = new SimpleTraceEventParser().GetProfileData(scenarioExecutionResult);
+
+ // Extract the Pmc data for each one of the processes.
+ foreach (var process in processes)
{
- var startupIteration = new IterationModel { Iteration = new Dictionary<string, double>() };
- startupIteration.Iteration.Add("Duration", s_startupTimes[i]);
- startup.Performance.IterationModels.Add(startupIteration);
+ if (!processesOfInterest.Any(p => p.Equals(process.Name, StringComparison.OrdinalIgnoreCase)))
+ continue;
+
+ var processTest = scenarioBenchmark.Tests
+ .SingleOrDefault(t => t.Name == process.Name && t.Namespace == "");
+ if (processTest == null)
+ {
+ processTest = new ScenarioTestModel(process.Name) {
+ Namespace = "",
+ };
+ scenarioBenchmark.Tests.Add(processTest);
+
+ // Add metrics definitions.
+ processTest.Performance.Metrics.Add(ElapsedTimeMilliseconds);
+ processTest.Performance.Metrics.AddRange(metricModels);
+ }
- var requestIteration = new IterationModel { Iteration = new Dictionary<string, double>() };
- requestIteration.Iteration.Add("Duration", s_requestTimes[i]);
- request.Performance.IterationModels.Add(requestIteration);
+ var processIterationModel = new IterationModel {
+ Iteration = new Dictionary<string, double>()
+ };
+ processTest.Performance.IterationModels.Add(processIterationModel);
+
+ processIterationModel.Iteration.Add(
+ ElapsedTimeMilliseconds.Name, process.LifeSpan.Duration.TotalMilliseconds);
+
+ // Add process metrics values.
+ foreach (var pmcData in process.PerformanceMonitorCounterData)
+ processIterationModel.Iteration.Add(pmcData.Key.Name, pmcData.Value);
+
+ foreach (var module in process.Modules)
+ {
+ var moduleName = Path.GetFileName(module.FullName);
+ if (modulesOfInterest.Any(m => m.Equals(moduleName, StringComparison.OrdinalIgnoreCase)))
+ {
+ var moduleTestName = $"{moduleName}";
+ var moduleTest = scenarioBenchmark.Tests
+ .SingleOrDefault(t => t.Name == moduleTestName && t.Namespace == process.Name);
+
+ if (moduleTest == null)
+ {
+ moduleTest = new ScenarioTestModel(moduleTestName) {
+ Namespace = process.Name,
+ Separator = "!",
+ };
+ scenarioBenchmark.Tests.Add(moduleTest);
+
+ // Add metrics definitions.
+ moduleTest.Performance.Metrics.AddRange(metricModels);
+ }
+
+ var moduleIterationModel = new IterationModel {
+ Iteration = new Dictionary<string, double>()
+ };
+ moduleTest.Performance.IterationModels.Add(moduleIterationModel);
+
+ // 5. Add module metrics values.
+ foreach (var pmcData in module.PerformanceMonitorCounterData)
+ moduleIterationModel.Iteration.Add(pmcData.Key.Name, pmcData.Value);
+ }
+ }
}
return scenarioBenchmark;
@@ -438,7 +663,7 @@ namespace JitBench
}
}
- using (var p = new Process() { StartInfo = processStartInfo })
+ using (var p = new System.Diagnostics.Process { StartInfo = processStartInfo })
{
p.Start();
if (p.WaitForExit(timeoutMilliseconds) == false)
@@ -457,7 +682,7 @@ namespace JitBench
{
Console.WriteLine();
Console.WriteLine("**********************************************************************");
- Console.WriteLine($"** {message}");
+ Console.WriteLine($"** [{DateTime.Now}] {message}");
Console.WriteLine("**********************************************************************");
}
@@ -466,137 +691,27 @@ namespace JitBench
Console.WriteLine($"-- {message}");
}
- private static uint s_iterations;
- private static uint s_iteration;
- private static double[] s_startupTimes;
- private static double[] s_requestTimes;
- private static double[] s_steadystateTimes;
+ private List<IterationData> IterationsData { get; }
+
+ private static MetricModel ElapsedTimeMilliseconds { get; } = new MetricModel {
+ DisplayName = "Duration",
+ Name = "Duration",
+ Unit = "ms",
+ };
+
+#if DEBUG
+ private const int NumberOfIterations = 2;
+#else
+ private const int NumberOfIterations = 11;
+#endif
+ private readonly string _scenarioBenchmarkName;
+ private readonly StringBuilder _stdout;
+ private readonly StringBuilder _stderr;
+
private static string s_temporaryDirectory;
private static string s_jitBenchDevDirectory;
private static string s_dotnetProcessFileName;
private static string s_musicStoreDirectory;
private static string s_targetArchitecture;
-
- /// <summary>
- /// Provides an interface to parse the command line arguments passed to the JitBench harness.
- /// </summary>
- private sealed class JitBenchHarnessOptions
- {
- public JitBenchHarnessOptions()
- {
- _tempDirectory = Directory.GetCurrentDirectory();
- _iterations = 11;
- }
-
- [Option("use-existing-setup", Required = false, HelpText = "Use existing setup.")]
- public Boolean UseExistingSetup { get; set; }
-
- [Option("tiering", Required = false, HelpText = "Enable tiered jit.")]
- public Boolean EnableTiering { get; set; }
-
- [Option("minopts", Required = false, HelpText = "Force jit to use minopt codegen.")]
- public Boolean Minopts { get; set; }
-
- [Option("disable-r2r", Required = false, HelpText = "Disable loading of R2R images.")]
- public Boolean DisableR2R { get; set; }
-
- [Option("disable-ngen", Required = false, HelpText = "Disable loading of ngen images.")]
- public Boolean DisableNgen { get; set; }
-
- [Option("iterations", Required = false, HelpText = "Number of iterations to run.")]
- public uint Iterations { get { return _iterations; } set { _iterations = value; } }
-
- [Option('o', Required = false, HelpText = "Specifies the intermediate output directory name.")]
- public string IntermediateOutputDirectory
- {
- get { return _tempDirectory; }
-
- set
- {
- if (string.IsNullOrWhiteSpace(value))
- throw new InvalidOperationException("The intermediate output directory name cannot be null, empty or white space.");
-
- if (value.Any(c => Path.GetInvalidPathChars().Contains(c)))
- throw new InvalidOperationException("Specified intermediate output directory name contains invalid path characters.");
-
- _tempDirectory = Path.IsPathRooted(value) ? value : Path.GetFullPath(value);
- Directory.CreateDirectory(_tempDirectory);
- }
- }
-
- [Option("target-architecture", Required = true, HelpText = "JitBench target architecture (It must match the built product that was copied into sandbox).")]
- public string TargetArchitecture { get; set; }
-
- public static JitBenchHarnessOptions Parse(string[] args)
- {
- using (var parser = new Parser((settings) => {
- settings.CaseInsensitiveEnumValues = true;
- settings.CaseSensitive = false;
- settings.HelpWriter = new StringWriter();
- settings.IgnoreUnknownArguments = true;
- }))
- {
- JitBenchHarnessOptions options = null;
- parser.ParseArguments<JitBenchHarnessOptions>(args)
- .WithParsed(parsed => options = parsed)
- .WithNotParsed(errors => {
- foreach (Error error in errors)
- {
- switch (error.Tag)
- {
- case ErrorType.MissingValueOptionError:
- throw new ArgumentException(
- $"Missing value option for command line argument '{(error as MissingValueOptionError).NameInfo.NameText}'");
- case ErrorType.HelpRequestedError:
- Console.WriteLine(Usage());
- Environment.Exit(0);
- break;
- case ErrorType.VersionRequestedError:
- Console.WriteLine(new AssemblyName(typeof(JitBenchHarnessOptions).GetTypeInfo().Assembly.FullName).Version);
- Environment.Exit(0);
- break;
- case ErrorType.BadFormatTokenError:
- case ErrorType.UnknownOptionError:
- case ErrorType.MissingRequiredOptionError:
- throw new ArgumentException(
- $"Missing required command line argument '{(error as MissingRequiredOptionError).NameInfo.NameText}'");
- case ErrorType.MutuallyExclusiveSetError:
- case ErrorType.BadFormatConversionError:
- case ErrorType.SequenceOutOfRangeError:
- case ErrorType.RepeatedOptionError:
- case ErrorType.NoVerbSelectedError:
- case ErrorType.BadVerbSelectedError:
- case ErrorType.HelpVerbRequestedError:
- break;
- }
- }
- });
- return options;
- }
- }
-
- public static string Usage()
- {
- var parser = new Parser((parserSettings) => {
- parserSettings.CaseInsensitiveEnumValues = true;
- parserSettings.CaseSensitive = false;
- parserSettings.EnableDashDash = true;
- parserSettings.HelpWriter = new StringWriter();
- parserSettings.IgnoreUnknownArguments = true;
- });
-
- var helpTextString = new HelpText {
- AddDashesToOption = true,
- AddEnumValuesToHelpText = true,
- AdditionalNewLineAfterOption = false,
- Heading = "JitBenchHarness",
- MaximumDisplayWidth = 80,
- }.AddOptions(parser.ParseArguments<JitBenchHarnessOptions>(new string[] { "--help" })).ToString();
- return helpTextString;
- }
-
- private string _tempDirectory;
- private uint _iterations;
- }
}
}
diff --git a/tests/src/performance/Scenario/JitBench/JitBenchHarnessOptions.cs b/tests/src/performance/Scenario/JitBench/JitBenchHarnessOptions.cs
new file mode 100644
index 0000000000..65a283ff6e
--- /dev/null
+++ b/tests/src/performance/Scenario/JitBench/JitBenchHarnessOptions.cs
@@ -0,0 +1,136 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using CommandLine;
+using CommandLine.Text;
+using Microsoft.Xunit.Performance.Api;
+using System;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+
+namespace JitBench
+{
+ /// <summary>
+ /// Provides an interface to parse the command line arguments passed to the JitBench harness.
+ /// </summary>
+ internal sealed class JitBenchHarnessOptions
+ {
+ public JitBenchHarnessOptions()
+ {
+ _tempDirectory = Directory.GetCurrentDirectory();
+ _iterations = 11;
+ }
+
+ [Option("use-existing-setup", Required = false, HelpText = "Use existing setup.")]
+ public Boolean UseExistingSetup { get; set; }
+
+ [Option("tiering", Required = false, HelpText = "Enable tiered jit.")]
+ public Boolean EnableTiering { get; set; }
+
+ [Option("minopts", Required = false, HelpText = "Force jit to use minopt codegen.")]
+ public Boolean Minopts { get; set; }
+
+ [Option("disable-r2r", Required = false, HelpText = "Disable loading of R2R images.")]
+ public Boolean DisableR2R { get; set; }
+
+ [Option("disable-ngen", Required = false, HelpText = "Disable loading of ngen images.")]
+ public Boolean DisableNgen { get; set; }
+
+ [Option("iterations", Required = false, HelpText = "Number of iterations to run.")]
+ public uint Iterations { get { return _iterations; } set { _iterations = value; } }
+
+ [Option('o', Required = false, HelpText = "Specifies the intermediate output directory name.")]
+ public string IntermediateOutputDirectory
+ {
+ get { return _tempDirectory; }
+
+ set
+ {
+ if (string.IsNullOrWhiteSpace(value))
+ throw new InvalidOperationException("The intermediate output directory name cannot be null, empty or white space.");
+
+ if (value.Any(c => Path.GetInvalidPathChars().Contains(c)))
+ throw new InvalidOperationException("Specified intermediate output directory name contains invalid path characters.");
+
+ _tempDirectory = Path.IsPathRooted(value) ? value : Path.GetFullPath(value);
+ Directory.CreateDirectory(_tempDirectory);
+ }
+ }
+
+ [Option("target-architecture", Required = true, HelpText = "JitBench target architecture (It must match the built product that was copied into sandbox).")]
+ public string TargetArchitecture { get; set; }
+
+ public static JitBenchHarnessOptions Parse(string[] args)
+ {
+ using (var parser = new Parser((settings) => {
+ settings.CaseInsensitiveEnumValues = true;
+ settings.CaseSensitive = false;
+ settings.HelpWriter = new StringWriter();
+ settings.IgnoreUnknownArguments = true;
+ }))
+ {
+ JitBenchHarnessOptions options = null;
+ parser.ParseArguments<JitBenchHarnessOptions>(args)
+ .WithParsed(parsed => options = parsed)
+ .WithNotParsed(errors => {
+ foreach (Error error in errors)
+ {
+ switch (error.Tag)
+ {
+ case ErrorType.MissingValueOptionError:
+ throw new ArgumentException(
+ $"Missing value option for command line argument '{(error as MissingValueOptionError).NameInfo.NameText}'");
+ case ErrorType.HelpRequestedError:
+ Console.WriteLine(Usage());
+ Environment.Exit(0);
+ break;
+ case ErrorType.VersionRequestedError:
+ Console.WriteLine(new AssemblyName(typeof(JitBenchHarnessOptions).GetTypeInfo().Assembly.FullName).Version);
+ Environment.Exit(0);
+ break;
+ case ErrorType.BadFormatTokenError:
+ case ErrorType.UnknownOptionError:
+ case ErrorType.MissingRequiredOptionError:
+ throw new ArgumentException(
+ $"Missing required command line argument '{(error as MissingRequiredOptionError).NameInfo.NameText}'");
+ case ErrorType.MutuallyExclusiveSetError:
+ case ErrorType.BadFormatConversionError:
+ case ErrorType.SequenceOutOfRangeError:
+ case ErrorType.RepeatedOptionError:
+ case ErrorType.NoVerbSelectedError:
+ case ErrorType.BadVerbSelectedError:
+ case ErrorType.HelpVerbRequestedError:
+ break;
+ }
+ }
+ });
+ return options;
+ }
+ }
+
+ public static string Usage()
+ {
+ var parser = new Parser((parserSettings) => {
+ parserSettings.CaseInsensitiveEnumValues = true;
+ parserSettings.CaseSensitive = false;
+ parserSettings.EnableDashDash = true;
+ parserSettings.HelpWriter = new StringWriter();
+ parserSettings.IgnoreUnknownArguments = true;
+ });
+
+ var helpTextString = new HelpText {
+ AddDashesToOption = true,
+ AddEnumValuesToHelpText = true,
+ AdditionalNewLineAfterOption = false,
+ Heading = "JitBenchHarness",
+ MaximumDisplayWidth = 80,
+ }.AddOptions(parser.ParseArguments<JitBenchHarnessOptions>(new string[] { "--help" })).ToString();
+ return helpTextString;
+ }
+
+ private string _tempDirectory;
+ private uint _iterations;
+ }
+}
diff --git a/tests/src/performance/linkbench/linkbench.cs b/tests/src/performance/linkbench/linkbench.cs
index 7e710a775e..06b951c307 100644
--- a/tests/src/performance/linkbench/linkbench.cs
+++ b/tests/src/performance/linkbench/linkbench.cs
@@ -1,17 +1,12 @@
-using Microsoft.Xunit.Performance;
-using Microsoft.Xunit.Performance.Api;
using System;
-using System.Diagnostics;
using System.Collections.Generic;
-using System.IO;
-using System.Reflection;
-using System.Runtime.Loader;
-using System.Text;
+using System.Diagnostics;
using System.Globalization;
+using System.IO;
using System.Linq;
+using System.Runtime.Loader;
using System.Xml.Linq;
-using Xunit;
-using Xunit.Abstractions;
+using Microsoft.Xunit.Performance.Api;
namespace LinkBench
{
@@ -234,7 +229,6 @@ namespace LinkBench
public class LinkBench
{
- private static ScenarioConfiguration scenarioConfiguration = new ScenarioConfiguration(new TimeSpan(2000000));
private static MetricModel SizeMetric = new MetricModel { Name = "Size", DisplayName = "File Size", Unit = "MB" };
private static MetricModel PercMetric = new MetricModel { Name = "Ratio", DisplayName = "Reduction", Unit = "Linked/Unlinked" };
public static string Workspace;
@@ -472,7 +466,8 @@ namespace LinkBench
};
using (var h = new XunitPerformanceHarness(scriptArgs))
{
- h.RunScenario(emptyCmd, null, null, PostRun, scenarioConfiguration);
+ var configuration = new ScenarioConfiguration(new TimeSpan(2000000), emptyCmd);
+ h.RunScenario(configuration, PostRun);
}
}
diff --git a/tests/src/performance/linkbench/linkbench.csproj b/tests/src/performance/linkbench/linkbench.csproj
index f36537f197..8605f1a442 100644
--- a/tests/src/performance/linkbench/linkbench.csproj
+++ b/tests/src/performance/linkbench/linkbench.csproj
@@ -11,18 +11,17 @@
<AppDesignerFolder>Properties</AppDesignerFolder>
<FileAlignment>512</FileAlignment>
<ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
- <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
- <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
- <DefineConstants>$(DefineConstants);STATIC</DefineConstants>
- <NuGetTargetMoniker>.NETStandard,Version=v1.4</NuGetTargetMoniker>
- <NuGetTargetMonikerShort>netstandard1.4</NuGetTargetMonikerShort>
+ <NuGetTargetMoniker>.NETStandard,Version=v1.6</NuGetTargetMoniker>
+ <NuGetTargetMonikerShort>netstandard1.6</NuGetTargetMonikerShort>
</PropertyGroup>
<!-- Default configurations to help VS understand the configurations -->
- <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"></PropertyGroup>
- <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'"></PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' " />
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' " />
<PropertyGroup>
<RestoreOutputPath>..\obj</RestoreOutputPath>
+ <DebugType>pdbonly</DebugType>
+ <Optimize>true</Optimize>
</PropertyGroup>
<ItemGroup>
<CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
@@ -36,4 +35,7 @@
<Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
</ItemGroup>
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), performance.targets))\performance.targets" />
+ <PropertyGroup>
+ <ProjectAssetsFile>$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), performance.targets))\obj\project.assets.json</ProjectAssetsFile>
+ </PropertyGroup>
</Project>
diff --git a/tests/src/performance/performance.csproj b/tests/src/performance/performance.csproj
index 4a68139f43..8c2cd3c7de 100644
--- a/tests/src/performance/performance.csproj
+++ b/tests/src/performance/performance.csproj
@@ -2,8 +2,8 @@
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
<PropertyGroup>
- <NugetTargetMoniker>.NETStandard,Version=v1.5</NugetTargetMoniker>
- <NugetTargetMonikerShort>netstandard1.5</NugetTargetMonikerShort>
+ <NugetTargetMoniker>.NETStandard,Version=v1.6</NugetTargetMoniker>
+ <NugetTargetMonikerShort>netstandard1.6</NugetTargetMonikerShort>
<IsTestProject>false</IsTestProject>
</PropertyGroup>
<ItemGroup>
@@ -23,7 +23,7 @@
<Version>$(MicrosoftDiagnosticsTracingTraceEventPackageVersion)</Version>
</PackageReference>
<PackageReference Include="Microsoft.NETCore.Platforms">
- <Version>2.0.0-preview2-25302-03</Version>
+ <Version>$(MicrosoftNETCorePlatformsPackageVersion)</Version>
</PackageReference>
<PackageReference Include="System.Collections.NonGeneric">
<Version>4.4.0-beta-24913-02</Version>
@@ -41,7 +41,7 @@
<Version>4.4.0-beta-24913-02</Version>
</PackageReference>
<PackageReference Include="System.Numerics.Vectors">
- <Version>4.4.0-preview2-25302-03</Version>
+ <Version>$(MicrosoftPrivateCoreFxNETCoreAppPackageVersion)</Version>
</PackageReference>
<PackageReference Include="System.Reflection">
<Version>4.4.0-beta-24913-02</Version>
@@ -50,7 +50,7 @@
<Version>4.4.0-beta-24913-02</Version>
</PackageReference>
<PackageReference Include="System.Reflection.TypeExtensions">
- <Version>4.4.0-preview2-25302-03</Version>
+ <Version>$(MicrosoftPrivateCoreFxNETCoreAppPackageVersion)</Version>
</PackageReference>
<PackageReference Include="System.Runtime">
<Version>4.4.0-beta-24913-02</Version>
@@ -97,9 +97,12 @@
<PackageReference Include="xunit.runner.utility">
<Version>$(XunitPackageVersion)</Version>
</PackageReference>
+ <PackageReference Include="System.Security.Principal.Windows">
+ <Version>4.4.0</Version>
+ </PackageReference>
</ItemGroup>
<PropertyGroup>
- <TargetFramework>netstandard1.5</TargetFramework>
+ <TargetFramework>netstandard1.6</TargetFramework>
<TargetFrameworkIdentifier>.NETStandard</TargetFrameworkIdentifier>
<PackageTargetFallback>$(PackageTargetFallback);dnxcore50;portable-net45+win8</PackageTargetFallback>
<ContainsPackageReferences>true</ContainsPackageReferences>
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-27 21:59:47 +0000