From 43aee529c3fa4eb44a120edddca50216bf752af5 Mon Sep 17 00:00:00 2001 From: Michelle McDaniel Date: Wed, 2 Aug 2017 16:29:37 -0700 Subject: Add JitMinOpts throughput jobs This change adds an opt_level throughput option so that we can run throughput jobs in minopts mode. This also adds jobs to the lab for min opts runs --- perf.groovy | 230 +++++++++++++++++++---------------- tests/scripts/run-throughput-perf.py | 23 +++- 2 files changed, 143 insertions(+), 110 deletions(-) diff --git a/perf.groovy b/perf.groovy index e3f42c7f6d..609fccaabf 100644 --- a/perf.groovy +++ b/perf.groovy @@ -154,68 +154,75 @@ def static getOSGroup(def os) { [true, false].each { isPR -> ['Windows_NT'].each { os -> ['x64', 'x86'].each { arch -> - def architecture = arch + ['full_opt', 'min_opt'].each { opt_level -> + def architecture = arch - def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) { - // Set the label. - label('windows_clr_perf') - wrappers { - credentialsBinding { - string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') + def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}", isPR)) { + // Set the label. + label('windows_clr_perf') + wrappers { + credentialsBinding { + string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') + } } - } - if (isPR) - { - parameters - { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.') - } - } - def configuration = 'Release' - def runType = isPR ? 'private' : 'rolling' - def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' + if (isPR) + { + parameters + { + stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.') + } + } + def configuration = 'Release' + def runType = isPR ? 'private' : 'rolling' + def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' - steps { - // Batch + steps { + // Batch - batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"") - batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"") - batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") - batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") - //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView - //we have to do it all as one statement because cmd is called each time and we lose the set environment variable - batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" + - "set \"BENCHVIEWNAME=${benchViewName}\"\n" + - "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" + - "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" + - "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}") - batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"") - batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests") - batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") - batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}") + batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"") + batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"") + batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") + batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") + //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView + //we have to do it all as one statement because cmd is called each time and we lose the set environment variable + batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" + + "set \"BENCHVIEWNAME=${benchViewName}\"\n" + + "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" + + "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" + + "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}") + batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"") + batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests") + batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") + batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}") + } } - } - // Save machinedata.json to /artifact/bin/ Jenkins dir - def archiveSettings = new ArchivalSettings() - archiveSettings.addFiles('throughput-*.csv') - Utilities.addArchival(newJob, archiveSettings) + // Save machinedata.json to /artifact/bin/ Jenkins dir + def archiveSettings = new ArchivalSettings() + archiveSettings.addFiles('throughput-*.csv') + Utilities.addArchival(newJob, archiveSettings) - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") + Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - if (isPR) { - TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() - builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests") - builder.triggerOnlyOnComment() - builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*") - builder.triggerForBranch(branch) - builder.emitTrigger(newJob) - } - else { - // Set a push trigger - TriggerBuilder builder = TriggerBuilder.triggerOnCommit() - builder.emitTrigger(newJob) + if (isPR) { + def opts = "" + if (opt_level == 'min_opts') + { + opts = '\\W+min_opts' + } + TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() + builder.setGithubContext("${os} ${arch} ${opt_level} CoreCLR Throughput Perf Tests") + builder.triggerOnlyOnComment() + builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}\\W+throughput.*") + builder.triggerForBranch(branch) + builder.emitTrigger(newJob) + } + else { + // Set a push trigger + TriggerBuilder builder = TriggerBuilder.triggerOnCommit() + builder.emitTrigger(newJob) + } } } } @@ -384,76 +391,89 @@ def static getFullThroughputJobName(def project, def os, def isPR) { // Actual perf testing on the following OSes def throughputOSList = ['Ubuntu14.04'] + def throughputOptLevelList = ['full_opt', 'min_opt'] + + def throughputOSOptLevelList = [] + throughputOSList.each { os -> - def newJob = job(getFullThroughputJobName(project, os, isPR)) { + throughputOptLevelList.each { opt_level -> + throughputOSOptLevelList.add("${os}_${opt_level}") + } + } - label('linux_clr_perf') - wrappers { - credentialsBinding { - string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') + throughputOSList.each { os -> + throughputOptLevelList.each { opt_level -> + def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) { + + label('linux_clr_perf') + wrappers { + credentialsBinding { + string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') + } } - } - if (isPR) - { - parameters + if (isPR) { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.') + parameters + { + stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.') + } } - } - parameters { - stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux') - } + parameters { + stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux') + } - def osGroup = getOSGroup(os) - def runType = isPR ? 'private' : 'rolling' - def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT' + def osGroup = getOSGroup(os) + def runType = isPR ? 'private' : 'rolling' + def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT' - steps { - shell("bash ./tests/scripts/perf-prep.sh --throughput") - shell("./init-tools.sh") - copyArtifacts(fullBuildJobName) { - includePatterns("bin/Product/**") - buildSelector { - buildNumber('\${PRODUCT_BUILD}') + steps { + shell("bash ./tests/scripts/perf-prep.sh --throughput") + shell("./init-tools.sh") + copyArtifacts(fullBuildJobName) { + includePatterns("bin/Product/**") + buildSelector { + buildNumber('\${PRODUCT_BUILD}') + } } + shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" + + "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" + + "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}") + shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\ + -arch \"${architecture}\" \\ + -os \"${os}\" \\ + -configuration \"${configuration}\" \\ + -opt_level \"${opt_level}\" \\ + -clr_root \"\${WORKSPACE}\" \\ + -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\ + -run_type \"${runType}\" \\ + -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""") } - shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" + - "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" + - "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}") - shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\ - -arch \"${architecture}\" \\ - -os \"${os}\" \\ - -configuration \"${configuration}\" \\ - -clr_root \"\${WORKSPACE}\" \\ - -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\ - -run_type \"${runType}\" \\ - -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""") } - } - // Save machinedata.json to /artifact/bin/ Jenkins dir - def archiveSettings = new ArchivalSettings() - archiveSettings.addFiles('throughput-*.csv') - archiveSettings.addFiles('machinedata.json') - Utilities.addArchival(newJob, archiveSettings) + // Save machinedata.json to /artifact/bin/ Jenkins dir + def archiveSettings = new ArchivalSettings() + archiveSettings.addFiles('throughput-*.csv') + archiveSettings.addFiles('machinedata.json') + Utilities.addArchival(newJob, archiveSettings) - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") + Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - // For perf, we need to keep the run results longer - newJob.with { - // Enable the log rotator - logRotator { - artifactDaysToKeep(7) - daysToKeep(300) - artifactNumToKeep(25) - numToKeep(1000) + // For perf, we need to keep the run results longer + newJob.with { + // Enable the log rotator + logRotator { + artifactDaysToKeep(7) + daysToKeep(300) + artifactNumToKeep(25) + numToKeep(1000) + } } - } + } // opt_level } // os - def flowJobTPRunList = throughputOSList.collect { os -> + def flowJobTPRunList = throughputOSOptLevelList.collect { os -> "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }" } def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) { diff --git a/tests/scripts/run-throughput-perf.py b/tests/scripts/run-throughput-perf.py index 4fa43e9b7d..c0ed43a459 100644 --- a/tests/scripts/run-throughput-perf.py +++ b/tests/scripts/run-throughput-perf.py @@ -116,6 +116,7 @@ parser.add_argument('-clr_root', dest='clr_root', default=None) parser.add_argument('-assembly_root', dest='assembly_root', default=None) parser.add_argument('-benchview_path', dest='benchview_path', default=None) parser.add_argument('-iterations', dest='iterations', default=5, type=int) +parser.add_argument('-opt_level', dest='opt_level', default='full_opt') ########################################################################## # Helper Functions @@ -141,6 +142,7 @@ def validate_args(args): assembly_root = args.assembly_root benchview_path = args.benchview_path iterations = args.iterations + opt_level = args.opt_level.lower() def validate_arg(arg, check): """ Validate an individual arg @@ -162,6 +164,7 @@ def validate_args(args): valid_build_types = ['Release'] valid_run_types = ['rolling', 'private'] valid_os = ['Windows_NT', 'Ubuntu14.04'] + valid_opt_levels = ['full_opt', 'min_opt'] arch = next((a for a in valid_archs if a.lower() == arch.lower()), arch) build_type = next((b for b in valid_build_types if b.lower() == build_type.lower()), build_type) @@ -174,6 +177,7 @@ def validate_args(args): validate_arg(build_type, lambda item: item in valid_build_types) validate_arg(run_type, lambda item: item in valid_run_types) validate_arg(iterations, lambda item: item > 0) + validate_arg(opt_level, lambda item: item in valid_opt_levels) if clr_root is None: raise Exception('--clr_root must be set') @@ -191,7 +195,7 @@ def validate_args(args): benchview_path = os.path.normpath(benchview_path) validate_arg(benchview_path, lambda item: os.path.isdir(benchview_path)) - args = (arch, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations) + args = (arch, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations, opt_level) # Log configuration log('Configuration:') @@ -199,6 +203,7 @@ def validate_args(args): log(' os: %s' % operating_system) log(' os_group: %s' % os_group) log(' build_type: %s' % build_type) + log(' opt_level: %s' % opt_level) log(' run_type: %s' % run_type) log(' iterations: %d' % iterations) log(' clr_root: %s' % clr_root) @@ -256,7 +261,7 @@ def generateCSV(dll_name, dll_runtimes): return csv_file_name -def runIterations(dll_name, dll_path, iterations, crossgen_path, jit_path, assemblies_path): +def runIterations(dll_name, dll_path, iterations, crossgen_path, jit_path, assemblies_path, opt_level): """ Run throughput testing for a given dll Args: dll_name: the name of the dll @@ -280,13 +285,18 @@ def runIterations(dll_name, dll_path, iterations, crossgen_path, jit_path, assem dll_path ] + my_env = os.environ + + if opt_level == 'min_opt': + my_env['COMPlus_JITMinOpts'] = '1' + log(" ".join(run_args)) # Time.clock() returns seconds, with a resolution of 0.4 microseconds, so multiply by the multiplier to get milliseconds multiplier = 1000 for iteration in range(0,iterations + 1): - proc = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=my_env) start_time = timeit.default_timer() (out, err) = proc.communicate() @@ -312,7 +322,7 @@ def main(args): global os_group_list global python_exe_list - architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations = validate_args(args) + architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations, opt_level = validate_args(args) arch = architecture current_dir = os.getcwd() @@ -351,7 +361,7 @@ def main(args): (not dll_file_name in dll_exclude_list["All"])): dll_name = dll_file_name.replace(".dll", "") dll_path = os.path.join(assembly_root, dll_file_name) - dll_elapsed_times = runIterations(dll_file_name, dll_path, iterations, crossgen_path, jit_path, assembly_root) + dll_elapsed_times = runIterations(dll_file_name, dll_path, iterations, crossgen_path, jit_path, assembly_root, opt_level) if len(dll_elapsed_times) != 0: if not benchview_path is None: @@ -404,6 +414,9 @@ def main(args): "--config", "OS", operating_system, + "--config", + "OptLevel", + opt_level, "--arch", architecture, "--machinepool", -- cgit v1.2.3