1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
// Import the utility functionality.
import jobs.generation.*;
def project = GithubProject
def branch = GithubBranchName
def projectName = Utilities.getFolderName(project)
def projectFolder = projectName + '/' + Utilities.getFolderName(branch)
def static getOSGroup(def os) {
def osGroupMap = ['Ubuntu':'Linux',
'RHEL7.2': 'Linux',
'Ubuntu16.04': 'Linux',
'Debian8.4':'Linux',
'Fedora23':'Linux',
'OSX':'OSX',
'Windows_NT':'Windows_NT',
'FreeBSD':'FreeBSD',
'CentOS7.1': 'Linux',
'OpenSUSE13.2': 'Linux',
'OpenSUSE42.1': 'Linux',
'LinuxARMEmulator': 'Linux']
def osGroup = osGroupMap.get(os, null)
assert osGroup != null : "Could not find os group for ${os}"
return osGroupMap[os]
}
// Setup perflab tests runs
[true, false].each { isPR ->
['Windows_NT'].each { os ->
['x64'].each { architecture ->
def configuration = 'Release'
def runType = isPR ? 'private' : 'rolling'
def benchViewName = isPR ? 'coreclr private %ghprbPullTitle%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}", isPR)) {
// Set the label.
label('windows_clr_perf')
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
}
}
steps {
// Batch
batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease -ExcludeVersion")
//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
"py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py --name " + "\"" + benchViewName + "\"" + " --user " + "\"dotnet-bot@microsoft.com\"\n" +
"py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\build.py git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type " + runType)
batchFile("py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py")
batchFile("set __TestIntermediateDir=int&&build.cmd release ${architecture}")
batchFile("tests\\runtest.cmd release ${architecture} GenerateLayoutOnly")
batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.${architecture}.Release\\performance\\perflab\\Perflab -library -uploadToBenchview C:\\Tools\\Microsoft.Benchview.JSONFormat\\tools -runtype " + runType)
batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.${architecture}.Release\\Jit\\Performance\\CodeQuality -uploadToBenchview C:\\Tools\\Microsoft.Benchview.JSONFormat\\tools -runtype " + runType)
}
}
// Save machinedata.json to /artifact/bin/ Jenkins dir
def archiveSettings = new ArchivalSettings()
archiveSettings.addFiles('perf-*.xml')
archiveSettings.addFiles('perf-*.etl')
Utilities.addArchival(newJob, archiveSettings)
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
if (isPR) {
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
builder.setGithubContext("${os} CoreCLR Perf Tests")
builder.triggerOnlyOnComment()
builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
builder.triggerForBranch(branch)
builder.emitTrigger(newJob)
}
else {
// Set a push trigger
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
builder.emitTrigger(newJob)
}
}
}
}
// Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
[true, false].each { isPR ->
['Ubuntu'].each { os ->
def osGroup = getOSGroup(os)
def architecture = 'x64'
def configuration = 'Release'
def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
steps {
shell("sudo bash ./tests/scripts/perf-prep.sh --branch=${projectName}")
shell("sudo ./init-tools.sh")
shell("""sudo bash ./tests/scripts/run-xunit-perf.sh \\
--testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
--testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
--coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
--mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
--coreFxBinDir=\"\${WORKSPACE}/bin/${osGroup}.AnyCPU.${configuration};\${WORKSPACE}/bin/Unix.AnyCPU.${configuration};\${WORKSPACE}/bin/AnyOS.AnyCPU.${configuration}\" \\
--coreFxNativeBinDir=\"\${WORKSPACE}/bin/${osGroup}.${architecture}.${configuration}\"""")
}
}
Utilities.setMachineAffinity(newJob, os, 'latest-or-auto') // Just run against Linux VM's for now.
// Save machinedata.json to /artifact/bin/ Jenkins dir
def archiveSettings = new ArchivalSettings()
archiveSettings.addFiles('sandbox/perf-*.xml')
archiveSettings.addFiles('machinedata.json')
Utilities.addArchival(newJob, archiveSettings)
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
// For perf, we need to keep the run results longer
newJob.with {
// Enable the log rotator
logRotator {
artifactDaysToKeep(7)
daysToKeep(300)
artifactNumToKeep(25)
numToKeep(1000)
}
}
if (isPR) {
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
builder.setGithubContext("${os} Perf Tests")
builder.triggerOnlyOnComment()
builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
builder.triggerForBranch(branch)
builder.emitTrigger(newJob)
}
else {
// Set a push trigger
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
builder.emitTrigger(newJob)
}
} // os
} // isPR
|
