diff options
Diffstat (limited to 'tests/src/performance/linkbench/linkbench.cs')
-rw-r--r-- | tests/src/performance/linkbench/linkbench.cs | 387 |
1 files changed, 249 insertions, 138 deletions
diff --git a/tests/src/performance/linkbench/linkbench.cs b/tests/src/performance/linkbench/linkbench.cs index 3d29957f90..ad0f83ef0e 100644 --- a/tests/src/performance/linkbench/linkbench.cs +++ b/tests/src/performance/linkbench/linkbench.cs @@ -1,3 +1,5 @@ +using Microsoft.Xunit.Performance; +using Microsoft.Xunit.Performance.Api; using System; using System.Diagnostics; using System.Collections.Generic; @@ -8,8 +10,6 @@ using System.Text; using System.Globalization; using System.Linq; using System.Xml.Linq; -using Microsoft.Xunit.Performance; -using Microsoft.Xunit.Performance.Api; using Xunit; using Xunit.Abstractions; @@ -18,7 +18,8 @@ namespace LinkBench public class Benchmark { public string Name; - + public bool ShouldRun; + public string ProjFile; public string UnlinkedDir; public string LinkedDir; public double UnlinkedMsilSize; @@ -28,18 +29,29 @@ namespace LinkBench public double MsilSizeReduction; public double DirSizeReduction; + public delegate void SetupDelegate(); + public SetupDelegate Setup; + private DirectoryInfo unlinkedDirInfo; private DirectoryInfo linkedDirInfo; private double certDiff; const double MB = 1024 * 1024; - public Benchmark(string _Name, string _UnlinkedDir, string _LinkedDir) + public Benchmark(string _Name, string _UnlinkedDir, string _LinkedDir, SetupDelegate _setup = null, bool _shouldRun = false) { Name = _Name; UnlinkedDir = _UnlinkedDir; LinkedDir = _LinkedDir; unlinkedDirInfo = new DirectoryInfo(UnlinkedDir); linkedDirInfo = new DirectoryInfo(LinkedDir); + ShouldRun = _shouldRun; + Setup = _setup; + } + + public void SetToRun() + { + ShouldRun = true; + Environment.SetEnvironmentVariable("__test_" + Name, "true"); } public void Compute() @@ -50,8 +62,8 @@ namespace LinkBench UnlinkedDirSize = GetDirSize(unlinkedDirInfo); LinkedDirSize = GetDirSize(linkedDirInfo); - MsilSizeReduction = (UnlinkedMsilSize - LinkedMsilSize) / UnlinkedMsilSize * 100; - DirSizeReduction = (UnlinkedDirSize - LinkedDirSize) / UnlinkedDirSize * 100; + MsilSizeReduction = LinkedMsilSize / UnlinkedMsilSize; + DirSizeReduction = LinkedDirSize / UnlinkedDirSize; } // Compute total size of a directory, in MegaBytes @@ -82,20 +94,10 @@ namespace LinkBench foreach (string file in files) { - if (file.EndsWith(".ni.dll") || file.EndsWith(".ni.exe")) + if (IsMSIL(file)) { - continue; + msilSize += new FileInfo(file).Length; } - try - { - AssemblyLoadContext.GetAssemblyName(file); - } - catch (BadImageFormatException) - { - continue; - } - - msilSize += new FileInfo(file).Length; } return msilSize / MB; @@ -126,22 +128,104 @@ namespace LinkBench foreach (string file in files) { - try + if (IsMSIL(file)) { - AssemblyLoadContext.GetAssemblyName(file); + FileInfo fileInfo = new FileInfo(file); + long linkedCert = GetCertSize(file); + long unlinkedCert = GetCertSize(UnlinkedDir + "\\" + fileInfo.Name); + totalDiff += (unlinkedCert - linkedCert); } - catch (BadImageFormatException) + } + + return totalDiff / MB; + } + + //Use AssemblyLoadContext.GetAssemblyName(file); + private bool IsMSIL(string file) + { + if (file.EndsWith(".ni.dll") || file.EndsWith(".ni.exe")) + { + // Likely Native Image. + return false; + } + + try + { + AssemblyLoadContext.GetAssemblyName(file); + } + catch (Exception) + { + // We should check only for BadImageFormatException. + // But Checking for any exception until the following + // issue is fixed: + // https://github.com/dotnet/coreclr/issues/11499 + + return false; + } + + return true; + } + + public static void AddLinkerReference(string csproj) + { + var xdoc = XDocument.Load(csproj); + var ns = xdoc.Root.GetDefaultNamespace(); + bool added = false; + foreach (var el in xdoc.Root.Elements(ns + "ItemGroup")) + { + if (el.Elements(ns + "PackageReference").Any()) { - continue; + el.Add(new XElement(ns + "PackageReference", + new XAttribute("Include", "ILLink.Tasks"), + new XAttribute("Version", "0.1.4-preview"))); + added = true; + break; } + } + if (!added) + { + xdoc.Root.Add(new XElement(ns + "ItemGroup", + new XElement(ns + "PackageReference", + new XAttribute("Include", "ILLink.Tasks"), + new XAttribute("Version", "0.1.4-preview")))); + added = true; + } + using (var fs = new FileStream(csproj, FileMode.Create)) + { + xdoc.Save(fs); + } + } - FileInfo fileInfo = new FileInfo(file); - long linkedCert = GetCertSize(file); - long unlinkedCert = GetCertSize(UnlinkedDir + "\\" + fileInfo.Name); - totalDiff += (unlinkedCert - linkedCert); + // TODO: remove this once the linker is able to handle + // ready-to-run assembies + public static void SetRuntimeFrameworkVersion(string csproj) + { + var xdoc = XDocument.Load(csproj); + var ns = xdoc.Root.GetDefaultNamespace(); + var versionElement = xdoc.Root.Descendants(ns + "RuntimeFrameworkVersion").First(); + string runtimeFrameworkVersion = "2.0.0-preview2-002093-00"; + versionElement.Value = runtimeFrameworkVersion; + using (var fs = new FileStream(csproj, FileMode.Create)) + { + xdoc.Save(fs); } + } - return totalDiff / MB; + // TODO: Remove this once we figure out what to do about apps + // that have the publish output filtered by a manifest + // file. It looks like aspnet has made this the default. See + // the bug at https://github.com/dotnet/sdk/issues/1160. + public static void PreventPublishFiltering(string csproj) + { + var xdoc = XDocument.Load(csproj); + var ns = xdoc.Root.GetDefaultNamespace(); + var propertygroup = xdoc.Root.Element(ns + "PropertyGroup"); + propertygroup.Add(new XElement(ns + "PublishWithAspNetCoreTargetManifest", + "false")); + using (var fs = new FileStream(csproj, FileMode.Create)) + { + xdoc.Save(fs); + } } } @@ -149,42 +233,132 @@ namespace LinkBench { private static ScenarioConfiguration scenarioConfiguration = new ScenarioConfiguration(new TimeSpan(2000000)); private static MetricModel SizeMetric = new MetricModel { Name = "Size", DisplayName = "File Size", Unit = "MB" }; - private static MetricModel PercMetric = new MetricModel { Name = "Perc", DisplayName = "% Reduction", Unit = "%" }; + private static MetricModel PercMetric = new MetricModel { Name = "Ratio", DisplayName = "Reduction", Unit = "Linked/Unlinked" }; public static string Workspace; + public static string LinkBenchRoot; public static string ScriptDir; public static string AssetsDir; private static Benchmark CurrentBenchmark; + private static Benchmark[] Benchmarks = + { + new Benchmark("HelloWorld", + "HelloWorld\\bin\\release\\netcoreapp2.0\\win10-x64\\unlinked", + "HelloWorld\\bin\\release\\netcoreapp2.0\\win10-x64\\linked", + () => Benchmark.AddLinkerReference("HelloWorld\\HelloWorld.csproj")), + new Benchmark("WebAPI", + "WebAPI\\bin\\release\\netcoreapp2.0\\win10-x64\\unlinked", + "WebAPI\\bin\\release\\netcoreapp2.0\\win10-x64\\linked", + () => { Benchmark.AddLinkerReference("WebAPI\\WebAPI.csproj"); + Benchmark.PreventPublishFiltering("WebAPI\\WebAPI.csproj"); }), + //Remove the MusicStore from the run as the scenario is currently flaky and breaking performance runs + /*new Benchmark("MusicStore", + "JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\unlinked", + "JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\linked", + () => { Benchmark.AddLinkerReference("JitBench\\src\\MusicStore\\MusicStore.csproj"); + Benchmark.SetRuntimeFrameworkVersion("JitBench\\src\\MusicStore\\MusicStore.csproj"); }), + new Benchmark("MusicStore_R2R", + "JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\R2R\\unlinked", + "JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\R2R\\linked"),*/ + new Benchmark("Corefx", + "corefx\\bin\\ILLinkTrimAssembly\\netcoreapp-Windows_NT-Release-x64\\pretrimmed", + "corefx\\bin\\ILLinkTrimAssembly\\netcoreapp-Windows_NT-Release-x64\\trimmed"), + /*new Benchmark("Roslyn", + "roslyn\\Binaries\\Release\\Exes\\CscCore\\win7-x64\\publish", + "roslyn\\Binaries\\Release\\Exes\\CscCore\\win7-x64\\Linked") */ + }; + + static int UsageError() + { + Console.WriteLine("Usage: LinkBench [--nosetup] [--nobuild] [--perf:runid <id>] [<benchmarks>]"); + Console.WriteLine(" --nosetup: Don't clone and fixup benchmark repositories"); + Console.WriteLine(" --nosetup: Don't build and link benchmarks"); + Console.WriteLine(" --perf:runid: Specify the ID to append to benchmark result files"); + Console.WriteLine(" Benchmarks: HelloWorld, WebAPI, MusicStore, MusicStore_R2R, CoreFX, Roslyn"); + Console.WriteLine(" Default is to run all the above benchmarks."); + return -4; + } + public static int Main(String [] args) { - // Workspace is the ROOT of the coreclr tree. - // If CORECLR_REPO is not set, the script assumes that the location of sandbox - // is <path>\coreclr\sandbox. - bool doClone = true; + bool doSetup = true; bool doBuild = true; + string runId = ""; + string runOne = null; + bool benchmarkSpecified = false; - for(int i=0; i < args.Length; i++) + for (int i = 0; i < args.Length; i++) { - if (String.Compare(args[i], "noclone", true) == 0) + if (String.Compare(args[i], "--nosetup", true) == 0) { - doClone = false; + doSetup = false; } - else if (String.Compare(args[i], "nobuild", true) == 0) + else if (String.Compare(args[i], "--nobuild", true) == 0) { - doClone = false; + doSetup = false; doBuild = false; } + else if (String.Compare(args[i], "--perf:runid", true) == 0) + { + if (i + 1 < args.Length) + { + runId = args[++i] + "-"; + } + else + { + Console.WriteLine("Missing runID "); + return UsageError(); + } + } + else if (args[i][0] == '-') + { + Console.WriteLine("Unknown Option {0}", args[i]); + return UsageError(); + } else { - Console.WriteLine("Unknown argument"); - return -4; + foreach (Benchmark benchmark in Benchmarks) + { + if (String.Compare(args[i], benchmark.Name, true) == 0) + { + benchmark.SetToRun(); + benchmarkSpecified = true; + break; + } + } + + if (!benchmarkSpecified) + { + Console.WriteLine("Unknown Benchmark {0}", args[i]); + } } } + // If benchmarks are not explicitly specified, run all benchmarks + if (!benchmarkSpecified) + { + foreach (Benchmark benchmark in Benchmarks) + { + if (String.Compare(benchmark.Name, "CoreFX", true) == 0) + { + // CoreFX is not enabled by default, because the lab cannot run it yet. + // Jenkins runs on an older OS with path-length limit, which causes + // CoreFX build to fail. + continue; + } + + benchmark.SetToRun(); + } + } + + // Workspace is the ROOT of the coreclr tree. + // If CORECLR_REPO is not set, the script assumes that the location of sandbox + // is <path>\coreclr\sandbox. + LinkBenchRoot = Directory.GetCurrentDirectory(); Workspace = Environment.GetEnvironmentVariable("CORECLR_REPO"); if (Workspace == null) { - Workspace = Directory.GetParent(Directory.GetCurrentDirectory()).FullName; + Workspace = Directory.GetParent(LinkBenchRoot).FullName; } if (Workspace == null) { @@ -192,38 +366,38 @@ namespace LinkBench return -1; } - string LinkBenchDir = Workspace + "\\tests\\src\\performance\\linkbench\\"; - ScriptDir = LinkBenchDir + "scripts\\"; - AssetsDir = LinkBenchDir + "assets\\"; + string linkBenchSrcDir = Workspace + "\\tests\\src\\performance\\linkbench\\"; + ScriptDir = linkBenchSrcDir + "scripts\\"; + AssetsDir = linkBenchSrcDir + "assets\\"; - Benchmark[] Benchmarks = - { - new Benchmark("HelloWorld", - "LinkBench\\HelloWorld\\bin\\release\\netcoreapp2.0\\win10-x64\\publish", - "LinkBench\\HelloWorld\\bin\\release\\netcoreapp2.0\\win10-x64\\linked"), - new Benchmark("WebAPI", - "LinkBench\\WebAPI\\bin\\release\\netcoreapp2.0\\win10-x64\\publish", - "LinkBench\\WebAPI\\bin\\release\\netcoreapp2.0\\win10-x64\\linked"), - new Benchmark("MusicStore", - "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\publish", - "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\linked"), - new Benchmark("MusicStore_R2R", - "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\publish_r2r", - "LinkBench\\JitBench\\src\\MusicStore\\bin\\release\\netcoreapp2.0\\win10-x64\\linked_r2r"), - new Benchmark("Corefx", - "LinkBench\\corefx\\bin\\ILLinkTrimAssembly\\netcoreapp-Windows_NT-Release-x64\\pretrimmed", - "LinkBench\\corefx\\bin\\ILLinkTrimAssembly\\netcoreapp-Windows_NT-Release-x64\\trimmed"), - new Benchmark("Roslyn", - "LinkBench\\roslyn\\Binaries\\Release\\Exes\\CscCore", - "LinkBench\\roslyn\\Binaries\\Release\\Exes\\Linked"), - }; + Environment.SetEnvironmentVariable("LinkBenchRoot", LinkBenchRoot); + Environment.SetEnvironmentVariable("__dotnet1", LinkBenchRoot + "\\.Net1\\dotnet.exe"); + Environment.SetEnvironmentVariable("__dotnet2", LinkBenchRoot + "\\.Net2\\dotnet.exe"); // Update the build files to facilitate the link step - if(doClone) + if (doSetup) { - if(!Setup()) + // Clone the benchmarks + using (var setup = new Process()) { - return -2; + setup.StartInfo.FileName = ScriptDir + "clone.cmd"; + setup.Start(); + setup.WaitForExit(); + if (setup.ExitCode != 0) + { + Console.WriteLine("Benchmark Setup failed"); + return -2; + } + } + + // Setup the benchmarks + + foreach (Benchmark benchmark in Benchmarks) + { + if (benchmark.ShouldRun && benchmark.Setup != null) + { + benchmark.Setup(); + } } } @@ -238,7 +412,7 @@ namespace LinkBench setup.WaitForExit(); if (setup.ExitCode != 0) { - Console.WriteLine("Setup failed"); + Console.WriteLine("Benchmark build failed"); return -3; } } @@ -255,8 +429,12 @@ namespace LinkBench for (int i = 0; i < Benchmarks.Length; i++) { CurrentBenchmark = Benchmarks[i]; - string[] scriptArgs = { "--perf:runid", CurrentBenchmark.Name }; + if (!CurrentBenchmark.ShouldRun) + { + continue; + } + string[] scriptArgs = { "--perf:runid", runId + CurrentBenchmark.Name }; using (var h = new XunitPerformanceHarness(scriptArgs)) { h.RunScenario(emptyCmd, null, null, PostRun, scenarioConfiguration); @@ -280,81 +458,14 @@ namespace LinkBench addMeasurement(ref scenario, "MSIL Unlinked", SizeMetric, CurrentBenchmark.UnlinkedMsilSize); addMeasurement(ref scenario, "MSIL Linked", SizeMetric, CurrentBenchmark.LinkedMsilSize); - addMeasurement(ref scenario, "MSIL %Reduction", PercMetric, CurrentBenchmark.MsilSizeReduction); + addMeasurement(ref scenario, "MSIL Reduction", PercMetric, CurrentBenchmark.MsilSizeReduction); addMeasurement(ref scenario, "Total Uninked", SizeMetric, CurrentBenchmark.UnlinkedDirSize); addMeasurement(ref scenario, "Total Linked", SizeMetric, CurrentBenchmark.LinkedDirSize); - addMeasurement(ref scenario, "Total %Reduction", PercMetric, CurrentBenchmark.DirSizeReduction); + addMeasurement(ref scenario, "Total Reduction", PercMetric, CurrentBenchmark.DirSizeReduction); return scenario; } - private static bool Setup() - { - // Clone the benchmarks - using (var setup = new Process()) - { - setup.StartInfo.FileName = ScriptDir + "clone.cmd"; - Console.WriteLine("Run {0}", setup.StartInfo.FileName); - setup.Start(); - setup.WaitForExit(); - if (setup.ExitCode != 0) - { - Console.WriteLine("clone failed"); - return false; - } - } - - //Update the project files - AddLinkerReference("LinkBench\\HelloWorld\\HelloWorld.csproj"); - AddLinkerReference("LinkBench\\WebAPI\\WebAPI.csproj"); - AddLinkerReference("LinkBench\\JitBench\\src\\MusicStore\\MusicStore.csproj"); - RemoveCrossgenTarget("LinkBench\\JitBench\\src\\MusicStore\\MusicStore.csproj"); - - return true; - } - - private static void AddLinkerReference(string csproj) - { - var xdoc = XDocument.Load(csproj); - var ns = xdoc.Root.GetDefaultNamespace(); - bool added = false; - foreach (var el in xdoc.Root.Elements(ns + "ItemGroup")) - { - if (el.Elements(ns + "PackageReference").Any()) - { - el.Add(new XElement(ns + "PackageReference", - new XAttribute("Include", "ILLink.Tasks"), - new XAttribute("Version", "0.1.0-preview"))); - added = true; - break; - } - } - if (!added) - { - xdoc.Root.Add(new XElement(ns + "ItemGroup", - new XElement(ns + "PackageReference", - new XAttribute("Include", "ILLink.Tasks"), - new XAttribute("Version", "0.1.0-preview")))); - added = true; - } - using (var fs = new FileStream(csproj, FileMode.Create)) - { - xdoc.Save(fs); - } - } - - private static void RemoveCrossgenTarget(string csproj) - { - var xdoc = XDocument.Load(csproj); - var ns = xdoc.Root.GetDefaultNamespace(); - var target = xdoc.Root.Element(ns + "Target"); - target.Remove(); - using (var fs = new FileStream(csproj, FileMode.Create)) - { - xdoc.Save(fs); - } - } - private static void addMeasurement(ref ScenarioBenchmark scenario, string name, MetricModel metric, double value) { var iteration = new IterationModel |