diff options
| -rw-r--r-- | perf.groovy | 35 | ||||
| -rwxr-xr-x[-rw-r--r--] | tests/scripts/perf-prep.sh | 34 | ||||
| -rwxr-xr-x[-rw-r--r--] | tests/scripts/run-xunit-perf.sh | 32 |
3 files changed, 68 insertions, 33 deletions
diff --git a/perf.groovy b/perf.groovy index 9ac98d9665..e33d950420 100644 --- a/perf.groovy +++ b/perf.groovy @@ -8,7 +8,7 @@ def projectName = Utilities.getFolderName(project) def projectFolder = projectName + '/' + Utilities.getFolderName(branch) def static getOSGroup(def os) { - def osGroupMap = ['Ubuntu':'Linux', + def osGroupMap = ['Ubuntu14.04':'Linux', 'RHEL7.2': 'Linux', 'Ubuntu16.04': 'Linux', 'Debian8.4':'Linux', @@ -85,26 +85,41 @@ def static getOSGroup(def os) { // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario [true, false].each { isPR -> - ['Ubuntu'].each { os -> + ['Ubuntu14.04'].each { os -> def osGroup = getOSGroup(os) def architecture = 'x64' def configuration = 'Release' + def runType = isPR ? 'private' : 'rolling' + def benchViewName = isPR ? 'coreclr private \$ghprbPullTitle' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT' def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) { + + label('linux_clr_perf') + wrappers { + credentialsBinding { + string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') + } + } + steps { - shell("sudo bash ./tests/scripts/perf-prep.sh --branch=${projectName}") - shell("sudo ./init-tools.sh") - shell("""sudo bash ./tests/scripts/run-xunit-perf.sh \\ + shell("bash ./tests/scripts/perf-prep.sh") + shell("./init-tools.sh") + shell("./build.sh ${architecture} ${configuration}") + shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" + + "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name " + "\"" + benchViewName + "\"" + " --user " + "\"dotnet-bot@microsoft.com\"\n" + + "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type " + runType) + shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\ --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\ --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\ --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\ --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\ - --coreFxBinDir=\"\${WORKSPACE}/bin/${osGroup}.AnyCPU.${configuration};\${WORKSPACE}/bin/Unix.AnyCPU.${configuration};\${WORKSPACE}/bin/AnyOS.AnyCPU.${configuration}\" \\ - --coreFxNativeBinDir=\"\${WORKSPACE}/bin/${osGroup}.${architecture}.${configuration}\"""") + --coreFxBinDir=\"\${WORKSPACE}/corefx/bin/${osGroup}.AnyCPU.${configuration};\${WORKSPACE}/corefx/bin/Unix.AnyCPU.${configuration};\${WORKSPACE}/corefx/bin/AnyOS.AnyCPU.${configuration}\" \\ + --coreFxNativeBinDir=\"\${WORKSPACE}/corefx/bin/${osGroup}.${architecture}.${configuration}\" \\ + --runType=\"${runType}\" \\ + --benchViewOS=\"${os}\" \\ + --uploadToBenchview""") } } - Utilities.setMachineAffinity(newJob, os, 'latest-or-auto') // Just run against Linux VM's for now. - // Save machinedata.json to /artifact/bin/ Jenkins dir def archiveSettings = new ArchivalSettings() archiveSettings.addFiles('sandbox/perf-*.xml') @@ -137,4 +152,4 @@ def static getOSGroup(def os) { builder.emitTrigger(newJob) } } // os -} // isPR
\ No newline at end of file +} // isPR diff --git a/tests/scripts/perf-prep.sh b/tests/scripts/perf-prep.sh index 698886e2c1..eca03c5187 100644..100755 --- a/tests/scripts/perf-prep.sh +++ b/tests/scripts/perf-prep.sh @@ -44,39 +44,35 @@ echo "branch = $perfBranch" echo "architecture = $perfArch" echo "configuration = $perfConfig" -# Install nuget to download benchview package, which includes the script machinedata.py for machine data collection -wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -chmod u+x ./nuget.exe -./nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory ./tests/scripts -Prerelease -ExcludeVersion +# Since not all perf machines have Mono we cannot run nuget locally to get the Benchview tools +# Instead we curl the package feed and use grep and sed to find the newest package. +# We grep for content type and that returns us strings that contain the path to the nupkg +# Then we match only the last line using '$' and use the s command to replace the entire line +# with what we find inside of the quotes after src=. We then jump to label x on a match and if +# we don't match we delete the line. This returns just the address of the last nupkg to curl. +curl "http://benchviewtestfeed.azurewebsites.net/nuget/FindPackagesById()?id='Microsoft.BenchView.JSONFormat'" | grep "content type" | sed "$ s/.*src=\"\([^\"]*\)\".*/\1/;tx;d;:x" | xargs curl -o benchview.zip +unzip -q -o benchview.zip -d ./tests/scripts/Microsoft.BenchView.JSONFormat # Install python 3.5.2 to run machinedata.py for machine data collection -sudo add-apt-repository ppa:fkrull/deadsnakes -sudo apt-get update -sudo apt-get --assume-yes install python3.5 python3.5 --version python3.5 ./tests/scripts/Microsoft.BenchView.JSONFormat/tools/machinedata.py # Set up the copies # Coreclr build containing the tests and mscorlib -curl http://dotnet-ci.cloudapp.net/job/$perfBranch/job/master/job/release_windows_nt/lastSuccessfulBuild/artifact/bin/tests/tests.zip -o tests.zip +curl http://ci.dot.net/job/$perfBranch/job/master/job/release_windows_nt/lastSuccessfulBuild/artifact/bin/tests/tests.zip -o tests.zip -# Coreclr build we are trying to test -curl http://dotnet-ci.cloudapp.net/job/$perfBranch/job/master/job/release_ubuntu/lastSuccessfulBuild/artifact/*zip*/archive.zip -o bin.zip # Corefx components. We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs. -curl http://dotnet-ci.cloudapp.net/job/dotnet_corefx/job/master/job/ubuntu14.04_release/lastSuccessfulBuild/artifact/bin/build.tar.gz -o build.tar.gz +mkdir corefx +curl http://ci.dot.net/job/dotnet_corefx/job/master/job/ubuntu14.04_release/lastSuccessfulBuild/artifact/bin/build.tar.gz -o ./corefx/build.tar.gz # Unpack the corefx binaries +pushd corefx > /dev/null tar -xf build.tar.gz - -# Unzip the coreclr binaries -unzip -q -o bin.zip - -# Copy coreclr binaries to the right dir -cp -R ./archive/bin/obj ./bin -cp -R ./archive/bin/Product ./bin +popd > /dev/null # Unzip the tests first. Exit with 0 -mkdir ./bin/tests +mkdir bin +mkdir bin/tests unzip -q -o tests.zip -d ./bin/tests/Windows_NT.$perfArch.$perfConfig || exit 0 echo "unzip tests to ./bin/tests/Windows_NT.$perfArch.$perfConfig" diff --git a/tests/scripts/run-xunit-perf.sh b/tests/scripts/run-xunit-perf.sh index f8ba514713..85ac43321b 100644..100755 --- a/tests/scripts/run-xunit-perf.sh +++ b/tests/scripts/run-xunit-perf.sh @@ -30,6 +30,11 @@ function print_usage { echo ' (e.g. "corefx/bin/Linux.AnyCPU.Debug;corefx/bin/Unix.AnyCPU.Debug;corefx/bin/AnyOS.AnyCPU.Debug").' echo ' If files with the same name are present in multiple directories, the first one wins.' echo ' --coreFxNativeBinDir=<path> : Directory of the CoreFX native build (e.g. corefx/bin/Linux.x64.Debug).' + echo ' --uploadToBenchview : Specify this flag in order to have the results of the run uploaded to Benchview.' + echo ' This also requires that the os flag and runtype flag to be set. Lastly you must' + echo ' also have the BV_UPLOAD_SAS_TOKEN set to a SAS token for the Benchview upload container' + echo ' --benchViewOS=<os> : Specify the os that will be used to insert data into Benchview.' + echo ' --runType=<private|rolling> : Specify the runType for Benchview.' } # Variables for xUnit-style XML output. XML format: https://xunit.github.io/docs/format-xml-v2.html @@ -311,6 +316,9 @@ coreClrBinDir= mscorlibDir= coreFxBinDir= coreFxNativeBinDir= +uploadToBenchview= +benchViewOS= +runType= for i in "$@" do @@ -340,6 +348,15 @@ do --coreFxNativeBinDir=*) coreFxNativeBinDir=${i#*=} ;; + --benchViewOS=*) + benchViewOS=${i#*=} + ;; + --runType=*) + runType=${i#*=} + ;; + --uploadToBenchview) + uploadToBenchview=TRUE + ;; *) echo "Unknown switch: $i" print_usage @@ -394,16 +411,15 @@ fi # Run coreclr performance tests echo "Test root dir is: $testRootDir" -tests=($(find $testRootDir/JIT/Performance/CodeQuality -name '*.exe')) +tests=($(find $testRootDir/JIT/Performance/CodeQuality -name '*.exe') $(find $testRootDir/performance/perflab/PerfLab -name '*.dll')) echo "current dir is $PWD" - +rm measurement.json for testcase in ${tests[@]}; do test=$(basename $testcase) testname=$(basename $testcase .exe) echo "....Running $testname" - cp $testcase . chmod u+x ./corerun @@ -411,7 +427,15 @@ echo "./corerun Microsoft.DotNet.xunit.performance.runner.cli.dll $test -runner ./corerun Microsoft.DotNet.xunit.performance.runner.cli.dll $test -runner xunit.console.netcore.exe -runnerhost ./corerun -verbose -runid perf-$testname echo "./corerun Microsoft.DotNet.xunit.performance.analysis.cli.dll perf-$testname.xml -xml perf-$testname-summary.xml" ./corerun Microsoft.DotNet.xunit.performance.analysis.cli.dll perf-$testname.xml -xml perf-$testname-summary.xml +if [ "$uploadToBenchview" == "TRUE" ] + then + python3.5 ../../../../../tests/scripts/Microsoft.BenchView.JSONFormat/tools/measurement.py xunit perf-$testname.xml --better desc --drop-first-value --append +fi done - +if [ "$uploadToBenchview" == "TRUE" ] + then + python3.5 ../../../../../tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission.py measurement.json --build ../../../../../build.json --machine-data ../../../../../machinedata.json --metadata ../../../../../submission-metadata.json --group "CoreCLR" --type "$runType" --config-name "Release" --config Configuration "Release" --config OS "$benchViewOS" --arch "x64" --machinepool "Perfsnake" + python3.5 ../../../../../tests/scripts/Microsoft.BenchView.JSONFormat/tools/upload.py submission.json --container coreclr +fi mkdir ../../../../../sandbox cp *.xml ../../../../../sandbox |