summaryrefslogtreecommitdiff
path: root/tests/src/performance/linkbench/linkbench.cs
diff options
context:
space:
mode:
Diffstat (limited to 'tests/src/performance/linkbench/linkbench.cs')
-rw-r--r--tests/src/performance/linkbench/linkbench.cs387
1 files changed, 249 insertions, 138 deletions
diff --git a/tests/src/performance/linkbench/linkbench.cs b/tests/src/performance/linkbench/linkbench.cs
index 3d29957f90..ad0f83ef0e 100644
--- a/tests/src/performance/linkbench/linkbench.cs
+++ b/tests/src/performance/linkbench/linkbench.cs
@@ -1,3 +1,5 @@
+using Microsoft.Xunit.Performance;
+using Microsoft.Xunit.Performance.Api;
using System;
using System.Diagnostics;
using System.Collections.Generic;
@@ -8,8 +10,6 @@ using System.Text;
using System.Globalization;
using System.Linq;
using System.Xml.Linq;
-using Microsoft.Xunit.Performance;
-using Microsoft.Xunit.Performance.Api;
using Xunit;
using Xunit.Abstractions;
@@ -18,7 +18,8 @@ namespace LinkBench
public class Benchmark
{
public string Name;
-
+ public bool ShouldRun;
+ public string ProjFile;
public string UnlinkedDir;
public string LinkedDir;
public double UnlinkedMsilSize;
@@ -28,18 +29,29 @@ namespace LinkBench
public double MsilSizeReduction;
public double DirSizeReduction;
+ public delegate void SetupDelegate();
+ public SetupDelegate Setup;
+
private DirectoryInfo unlinkedDirInfo;
private DirectoryInfo linkedDirInfo;
private double certDiff;
const double MB = 1024 * 1024;
- public Benchmark(string _Name, string _UnlinkedDir, string _LinkedDir)
+ public Benchmark(string _Name, string _UnlinkedDir, string _LinkedDir, SetupDelegate _setup = null, bool _shouldRun = false)
{
Name = _Name;
UnlinkedDir = _UnlinkedDir;
LinkedDir = _LinkedDir;
unlinkedDirInfo = new DirectoryInfo(UnlinkedDir);
linkedDirInfo = new DirectoryInfo(LinkedDir);
+ ShouldRun = _shouldRun;
+ Setup = _setup;
+ }
+
+ public void SetToRun()
+ {
+ ShouldRun = true;
+ Environment.SetEnvironmentVariable("__test_" + Name, "true");
}
public void Compute()
@@ -50,8 +62,8 @@ namespace LinkBench
UnlinkedDirSize = GetDirSize(unlinkedDirInfo);
LinkedDirSize = GetDirSize(linkedDirInfo);
- MsilSizeReduction = (UnlinkedMsilSize - LinkedMsilSize) / UnlinkedMsilSize * 100;
- DirSizeReduction = (UnlinkedDirSize - LinkedDirSize) / UnlinkedDirSize * 100;
+ MsilSizeReduction = LinkedMsilSize / UnlinkedMsilSize;
+ DirSizeReduction = LinkedDirSize / UnlinkedDirSize;
}
// Compute total size of a directory, in MegaBytes
@@ -82,20 +94,10 @@ namespace LinkBench
foreach (string file in files)
{
- if (file.EndsWith(".ni.dll") || file.EndsWith(".ni.exe"))
+ if (IsMSIL(file))
{
- continue;
+ msilSize += new FileInfo(file).Length;
}
- try
- {
- AssemblyLoadContext.GetAssemblyName(file);
- }
- catch (BadImageFormatException)
- {
- continue;
- }
-
- msilSize += new FileInfo(file).Length;
}
return msilSize / MB;
@@ -126,22 +128,104 @@ namespace LinkBench
foreach (string file in files)
{
- try
+ if (IsMSIL(file))
{
- AssemblyLoadContext.GetAssemblyName(file);
+ FileInfo fileInfo = new FileInfo(file);
+ long linkedCert = GetCertSize(file);
+ long unlinkedCert = GetCertSize(UnlinkedDir + "\\" + fileInfo.Name);
+ totalDiff += (unlinkedCert - linkedCert);
}
- catch (BadImageFormatException)
+ }
+
+ return totalDiff / MB;
+ }
+
+ //Use AssemblyLoadContext.GetAssemblyName(file);
+ private bool IsMSIL(string file)
+ {
+ if (file.EndsWith(".ni.dll") || file.EndsWith(".ni.exe"))
+ {
+ // Likely Native Image.
+ return false;
+ }
+
+ try
+ {
+ AssemblyLoadContext.GetAssemblyName(file);
+ }
+ catch (Exception)
+ {
+ // We should check only for BadImageFormatException.
+ // But Checking for any exception until the following
+ // issue is fixed:
+ // https://github.com/dotnet/coreclr/issues/11499
+
+ return false;
+ }
+
+ return true;
+ }
+
+ public static void AddLinkerReference(string csproj)
+ {
+ var xdoc = XDocument.Load(csproj);
+ var ns = xdoc.Root.GetDefaultNamespace();
+ bool added = false;
+ foreach (var el in xdoc.Root.Elements(ns + "ItemGroup"))
+ {
+ if (el.Elements(ns + "PackageReference").Any())
{
- continue;
+ el.Add(new XElement(ns + "PackageReference",
+ new XAttribute("Include", "ILLink.Tasks"),
+ new XAttribute("Version", "0.1.4-preview")));
+ added = true;
+ break;
}
+ }
+ if (!added)
+ {
+ xdoc.Root.Add(new XElement(ns + "ItemGroup",
+ new XElement(ns + "PackageReference",
+ new XAttribute("Include", "ILLink.Tasks"),
+ new XAttribute("Version", "0.1.4-preview"))));
+ added = true;
+ }
+ using (var fs = new FileStream(csproj, FileMode.Create))
+ {
+ xdoc.Save(fs);
+ }
+ }
- FileInfo fileInfo = new FileInfo(file);
- long linkedCert = GetCertSize(file);
- long unlinkedCert = GetCertSize(UnlinkedDir + "\\" + fileInfo.Name);
- totalDiff += (unlinkedCert - linkedCert);
+ // TODO: remove this once the linker is able to handle
+ // ready-to-run assembies
+ public static void SetRuntimeFrameworkVersion(string csproj)
+ {
+ var xdoc = XDocument.Load(csproj);
+ var ns = xdoc.Root.GetDefaultNamespace();
+ var versionElement = xdoc.Root.Descendants(ns + "RuntimeFrameworkVersion").First();
+ string runtimeFrameworkVersion = "2.0.0-preview2-002093-00";
+ versionElement.Value = runtimeFrameworkVersion;
+ using (var fs = new FileStream(csproj, FileMode.Create))
+ {
+ xdoc.Save(fs);
}
+ }
- return totalDiff / MB;
+ // TODO: Remove this once we figure out what to do about apps
+ // that have the publish output filtered by a manifest
+ // file. It looks like aspnet has made this the default. See
+ // the bug at https://github.com/dotnet/sdk/issues/1160.
+ public static void PreventPublishFiltering(string csproj)
+ {
+ var xdoc = XDocument.Load(csproj);
+ var ns = xdoc.Root.GetDefaultNamespace();
+ var propertygroup = xdoc.Root.Element(ns + "PropertyGroup");
+ propertygroup.Add(new XElement(ns + "PublishWithAspNetCoreTargetManifest",
+ "false"));
+ using (var fs = new FileStream(csproj, FileMode.Create))
+ {
+ xdoc.Save(fs);
+ }
}
}
@@ -149,42 +233,132 @@ namespace LinkBench
{
private static ScenarioConfiguration scenarioConfiguration = new ScenarioConfiguration(new TimeSpan(2000000));
private static MetricModel SizeMetric = new MetricModel { Name = "Size", DisplayName = "File Size", Unit = "MB" };
- private static MetricModel PercMetric = new MetricModel { Name = "Perc", DisplayName = "% Reduction", Unit = "%" };
+ private static MetricModel PercMetric = new MetricModel { Name = "Ratio", DisplayName = "Reduction", Unit = "Linked/Unlinked" };
public static string Workspace;
+ public static string LinkBenchRoot;
public static string ScriptDir;
public static string AssetsDir;
private static Benchmark CurrentBenchmark;
+ private static Benchmark[] Benchmarks =
+ {
+ new Benchmark("HelloWorld",
+ "HelloWorld\\bin\\release\\netcoreapp2.0\\win10-x64\\unlinked",
+ "HelloWorld\\bin\\release\\netcoreapp2.0\\win10-x64\\linked",
+ () => Benchmark.AddLinkerReference("HelloWorld\\HelloWorld.csproj")),
+ new Benchmark("WebAPI",
+ "WebAPI\\bin\\release\\netcoreapp2.0\\win10-x64\\unlinked",
+ "WebAPI\\bin\\release\\netcoreapp2.0\\win10-x64\\linked",
+ () => { Benchmark.AddLinkerReference("WebAPI\\WebAPI.csproj");
+ Benchmark.PreventPublishFiltering("WebAPI\\WebAPI.csproj"); }),
+ //Remove the MusicStore from the run as the scenario is currently flaky and breaking performance runs
+ /*new Benchmark("MusicStore",
+ "JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\unlinked",
+ "JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\linked",
+ () => { Benchmark.AddLinkerReference("JitBench\\src\\MusicStore\\MusicStore.csproj");
+ Benchmark.SetRuntimeFrameworkVersion("JitBench\\src\\MusicStore\\MusicStore.csproj"); }),
+ new Benchmark("MusicStore_R2R",
+ "JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\R2R\\unlinked",
+ "JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\R2R\\linked"),*/
+ new Benchmark("Corefx",
+ "corefx\\bin\\ILLinkTrimAssembly\\netcoreapp-Windows_NT-Release-x64\\pretrimmed",
+ "corefx\\bin\\ILLinkTrimAssembly\\netcoreapp-Windows_NT-Release-x64\\trimmed"),
+ /*new Benchmark("Roslyn",
+ "roslyn\\Binaries\\Release\\Exes\\CscCore\\win7-x64\\publish",
+ "roslyn\\Binaries\\Release\\Exes\\CscCore\\win7-x64\\Linked") */
+ };
+
+ static int UsageError()
+ {
+ Console.WriteLine("Usage: LinkBench [--nosetup] [--nobuild] [--perf:runid <id>] [<benchmarks>]");
+ Console.WriteLine(" --nosetup: Don't clone and fixup benchmark repositories");
+ Console.WriteLine(" --nosetup: Don't build and link benchmarks");
+ Console.WriteLine(" --perf:runid: Specify the ID to append to benchmark result files");
+ Console.WriteLine(" Benchmarks: HelloWorld, WebAPI, MusicStore, MusicStore_R2R, CoreFX, Roslyn");
+ Console.WriteLine(" Default is to run all the above benchmarks.");
+ return -4;
+ }
+
public static int Main(String [] args)
{
- // Workspace is the ROOT of the coreclr tree.
- // If CORECLR_REPO is not set, the script assumes that the location of sandbox
- // is <path>\coreclr\sandbox.
- bool doClone = true;
+ bool doSetup = true;
bool doBuild = true;
+ string runId = "";
+ string runOne = null;
+ bool benchmarkSpecified = false;
- for(int i=0; i < args.Length; i++)
+ for (int i = 0; i < args.Length; i++)
{
- if (String.Compare(args[i], "noclone", true) == 0)
+ if (String.Compare(args[i], "--nosetup", true) == 0)
{
- doClone = false;
+ doSetup = false;
}
- else if (String.Compare(args[i], "nobuild", true) == 0)
+ else if (String.Compare(args[i], "--nobuild", true) == 0)
{
- doClone = false;
+ doSetup = false;
doBuild = false;
}
+ else if (String.Compare(args[i], "--perf:runid", true) == 0)
+ {
+ if (i + 1 < args.Length)
+ {
+ runId = args[++i] + "-";
+ }
+ else
+ {
+ Console.WriteLine("Missing runID ");
+ return UsageError();
+ }
+ }
+ else if (args[i][0] == '-')
+ {
+ Console.WriteLine("Unknown Option {0}", args[i]);
+ return UsageError();
+ }
else
{
- Console.WriteLine("Unknown argument");
- return -4;
+ foreach (Benchmark benchmark in Benchmarks)
+ {
+ if (String.Compare(args[i], benchmark.Name, true) == 0)
+ {
+ benchmark.SetToRun();
+ benchmarkSpecified = true;
+ break;
+ }
+ }
+
+ if (!benchmarkSpecified)
+ {
+ Console.WriteLine("Unknown Benchmark {0}", args[i]);
+ }
}
}
+ // If benchmarks are not explicitly specified, run all benchmarks
+ if (!benchmarkSpecified)
+ {
+ foreach (Benchmark benchmark in Benchmarks)
+ {
+ if (String.Compare(benchmark.Name, "CoreFX", true) == 0)
+ {
+ // CoreFX is not enabled by default, because the lab cannot run it yet.
+ // Jenkins runs on an older OS with path-length limit, which causes
+ // CoreFX build to fail.
+ continue;
+ }
+
+ benchmark.SetToRun();
+ }
+ }
+
+ // Workspace is the ROOT of the coreclr tree.
+ // If CORECLR_REPO is not set, the script assumes that the location of sandbox
+ // is <path>\coreclr\sandbox.
+ LinkBenchRoot = Directory.GetCurrentDirectory();
Workspace = Environment.GetEnvironmentVariable("CORECLR_REPO");
if (Workspace == null)
{
- Workspace = Directory.GetParent(Directory.GetCurrentDirectory()).FullName;
+ Workspace = Directory.GetParent(LinkBenchRoot).FullName;
}
if (Workspace == null)
{
@@ -192,38 +366,38 @@ namespace LinkBench
return -1;
}
- string LinkBenchDir = Workspace + "\\tests\\src\\performance\\linkbench\\";
- ScriptDir = LinkBenchDir + "scripts\\";
- AssetsDir = LinkBenchDir + "assets\\";
+ string linkBenchSrcDir = Workspace + "\\tests\\src\\performance\\linkbench\\";
+ ScriptDir = linkBenchSrcDir + "scripts\\";
+ AssetsDir = linkBenchSrcDir + "assets\\";
- Benchmark[] Benchmarks =
- {
- new Benchmark("HelloWorld",
- "LinkBench\\HelloWorld\\bin\\release\\netcoreapp2.0\\win10-x64\\publish",
- "LinkBench\\HelloWorld\\bin\\release\\netcoreapp2.0\\win10-x64\\linked"),
- new Benchmark("WebAPI",
- "LinkBench\\WebAPI\\bin\\release\\netcoreapp2.0\\win10-x64\\publish",
- "LinkBench\\WebAPI\\bin\\release\\netcoreapp2.0\\win10-x64\\linked"),
- new Benchmark("MusicStore",
- "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\publish",
- "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\linked"),
- new Benchmark("MusicStore_R2R",
- "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\publish_r2r",
- "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\linked_r2r"),
- new Benchmark("Corefx",
- "LinkBench\\corefx\\bin\\ILLinkTrimAssembly\\netcoreapp-Windows_NT-Release-x64\\pretrimmed",
- "LinkBench\\corefx\\bin\\ILLinkTrimAssembly\\netcoreapp-Windows_NT-Release-x64\\trimmed"),
- new Benchmark("Roslyn",
- "LinkBench\\roslyn\\Binaries\\Release\\Exes\\CscCore",
- "LinkBench\\roslyn\\Binaries\\Release\\Exes\\Linked"),
- };
+ Environment.SetEnvironmentVariable("LinkBenchRoot", LinkBenchRoot);
+ Environment.SetEnvironmentVariable("__dotnet1", LinkBenchRoot + "\\.Net1\\dotnet.exe");
+ Environment.SetEnvironmentVariable("__dotnet2", LinkBenchRoot + "\\.Net2\\dotnet.exe");
// Update the build files to facilitate the link step
- if(doClone)
+ if (doSetup)
{
- if(!Setup())
+ // Clone the benchmarks
+ using (var setup = new Process())
{
- return -2;
+ setup.StartInfo.FileName = ScriptDir + "clone.cmd";
+ setup.Start();
+ setup.WaitForExit();
+ if (setup.ExitCode != 0)
+ {
+ Console.WriteLine("Benchmark Setup failed");
+ return -2;
+ }
+ }
+
+ // Setup the benchmarks
+
+ foreach (Benchmark benchmark in Benchmarks)
+ {
+ if (benchmark.ShouldRun && benchmark.Setup != null)
+ {
+ benchmark.Setup();
+ }
}
}
@@ -238,7 +412,7 @@ namespace LinkBench
setup.WaitForExit();
if (setup.ExitCode != 0)
{
- Console.WriteLine("Setup failed");
+ Console.WriteLine("Benchmark build failed");
return -3;
}
}
@@ -255,8 +429,12 @@ namespace LinkBench
for (int i = 0; i < Benchmarks.Length; i++)
{
CurrentBenchmark = Benchmarks[i];
- string[] scriptArgs = { "--perf:runid", CurrentBenchmark.Name };
+ if (!CurrentBenchmark.ShouldRun)
+ {
+ continue;
+ }
+ string[] scriptArgs = { "--perf:runid", runId + CurrentBenchmark.Name };
using (var h = new XunitPerformanceHarness(scriptArgs))
{
h.RunScenario(emptyCmd, null, null, PostRun, scenarioConfiguration);
@@ -280,81 +458,14 @@ namespace LinkBench
addMeasurement(ref scenario, "MSIL Unlinked", SizeMetric, CurrentBenchmark.UnlinkedMsilSize);
addMeasurement(ref scenario, "MSIL Linked", SizeMetric, CurrentBenchmark.LinkedMsilSize);
- addMeasurement(ref scenario, "MSIL %Reduction", PercMetric, CurrentBenchmark.MsilSizeReduction);
+ addMeasurement(ref scenario, "MSIL Reduction", PercMetric, CurrentBenchmark.MsilSizeReduction);
addMeasurement(ref scenario, "Total Uninked", SizeMetric, CurrentBenchmark.UnlinkedDirSize);
addMeasurement(ref scenario, "Total Linked", SizeMetric, CurrentBenchmark.LinkedDirSize);
- addMeasurement(ref scenario, "Total %Reduction", PercMetric, CurrentBenchmark.DirSizeReduction);
+ addMeasurement(ref scenario, "Total Reduction", PercMetric, CurrentBenchmark.DirSizeReduction);
return scenario;
}
- private static bool Setup()
- {
- // Clone the benchmarks
- using (var setup = new Process())
- {
- setup.StartInfo.FileName = ScriptDir + "clone.cmd";
- Console.WriteLine("Run {0}", setup.StartInfo.FileName);
- setup.Start();
- setup.WaitForExit();
- if (setup.ExitCode != 0)
- {
- Console.WriteLine("clone failed");
- return false;
- }
- }
-
- //Update the project files
- AddLinkerReference("LinkBench\\HelloWorld\\HelloWorld.csproj");
- AddLinkerReference("LinkBench\\WebAPI\\WebAPI.csproj");
- AddLinkerReference("LinkBench\\JitBench\\src\\MusicStore\\MusicStore.csproj");
- RemoveCrossgenTarget("LinkBench\\JitBench\\src\\MusicStore\\MusicStore.csproj");
-
- return true;
- }
-
- private static void AddLinkerReference(string csproj)
- {
- var xdoc = XDocument.Load(csproj);
- var ns = xdoc.Root.GetDefaultNamespace();
- bool added = false;
- foreach (var el in xdoc.Root.Elements(ns + "ItemGroup"))
- {
- if (el.Elements(ns + "PackageReference").Any())
- {
- el.Add(new XElement(ns + "PackageReference",
- new XAttribute("Include", "ILLink.Tasks"),
- new XAttribute("Version", "0.1.0-preview")));
- added = true;
- break;
- }
- }
- if (!added)
- {
- xdoc.Root.Add(new XElement(ns + "ItemGroup",
- new XElement(ns + "PackageReference",
- new XAttribute("Include", "ILLink.Tasks"),
- new XAttribute("Version", "0.1.0-preview"))));
- added = true;
- }
- using (var fs = new FileStream(csproj, FileMode.Create))
- {
- xdoc.Save(fs);
- }
- }
-
- private static void RemoveCrossgenTarget(string csproj)
- {
- var xdoc = XDocument.Load(csproj);
- var ns = xdoc.Root.GetDefaultNamespace();
- var target = xdoc.Root.Element(ns + "Target");
- target.Remove();
- using (var fs = new FileStream(csproj, FileMode.Create))
- {
- xdoc.Save(fs);
- }
- }
-
private static void addMeasurement(ref ScenarioBenchmark scenario, string name, MetricModel metric, double value)
{
var iteration = new IterationModel