diff options
author | Jarret Shook <jashoo@microsoft.com> | 2018-12-03 12:46:35 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-12-03 12:46:35 -0800 |
commit | 5632c9e3ac7f9dd6b858f175b2cf97963be996c4 (patch) | |
tree | c25d68fe779bd5f4160cdef213c2b7ddf441de75 /eng | |
parent | f8b4745ae754774ebf2a802ef1fd1f2bf6dbd024 (diff) | |
download | coreclr-5632c9e3ac7f9dd6b858f175b2cf97963be996c4.tar.gz coreclr-5632c9e3ac7f9dd6b858f175b2cf97963be996c4.tar.bz2 coreclr-5632c9e3ac7f9dd6b858f175b2cf97963be996c4.zip |
Correct platorm reach for the Official build (#21310)
* Correct platorm reach for the Official build
Does the following:
1. Sets up a ci rule for master and a pr rule for master
2. Corrects the OSX queues
3. Corrects ubuntu internal queue
4. Converts all linux jobs to build using containers
5. Only runs official builds on: Pri1, release, bringing us to parity with old process
6. Fixes centos build-test
7. Adds a super-annoying groupname tag that avoids name mangling.
* make sure we only add triggers for internal builds
* Fix syntax
* Missing :
* Remove pr/ci
* Fix internal queue name
* Correct Public/internal choices.
* Add crossgen as well
* Correct indentation
* Correct checked pri1 tests
* Address pr feedback
* Add todo comments
Diffstat (limited to 'eng')
-rw-r--r-- | eng/build-job.yml | 33 | ||||
-rw-r--r-- | eng/platform-matrix.yml | 67 | ||||
-rw-r--r-- | eng/test-job.yml | 26 | ||||
-rw-r--r-- | eng/xplat-job.yml | 22 |
4 files changed, 118 insertions, 30 deletions
diff --git a/eng/build-job.yml b/eng/build-job.yml index 08f8ac0b95..e44df9c05f 100644 --- a/eng/build-job.yml +++ b/eng/build-job.yml @@ -2,6 +2,8 @@ parameters: buildConfig: '' archType: '' osGroup: '' + osGroupName: '' + containerName: '' ### Product build jobs: @@ -10,15 +12,22 @@ jobs: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} + osGroupName: ${{ parameters.osGroupName }} # Compute job name from template parameters - name: ${{ format('build_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Build {0} {1} {2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }} + name: ${{ format('build_{0}_{1}_{2}', parameters.osGroupName, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Build {0} {1} {2}', parameters.osGroupName, parameters.archType, parameters.buildConfig) }} + + # Run all steps in the container. + # Note that the containers are resources defined in azure-pipelines.yml + containerName: ${{ parameters.containerName }} steps: # Install native dependencies - - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}: + # + # This is only required for non-docker builds. + - ${{ if eq(parameters.osGroup, 'OSX') }}: - script: sh eng/install-native-dependencies.sh $(osGroup) displayName: Install native dependencies - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: @@ -26,19 +35,17 @@ jobs: - script: eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force displayName: Install native dependencies - # Run init-tools (pre-arcade dependency bootstrapping) # TODO: replace this with an arcade equivalent - - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}: + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - script: ./init-tools.sh displayName: Init tools - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - script: .\init-tools.cmd displayName: Init tools - # Sync - - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}: + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - script: ./Tools/dotnetcli/dotnet msbuild build.proj /p:RestoreDuringBuild=true /t:Sync displayName: Sync - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: @@ -47,9 +54,15 @@ jobs: # Build - - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}: + - ${{ if and(and(ne(parameters.archType, 'arm'), ne(parameters.archType, 'arm64')), ne(parameters.osGroup, 'Windows_NT')) }}: - script: ./build.sh $(buildConfig) $(archType) -skipnuget -skiprestore displayName: Build product + - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm')) }}: + - script: ROOTFS_DIR=$(rootfsDir) CAC_ROOTFS_DIR=$(cacRootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skipnuget -skiprestore + displayName: Build product + - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm64')) }}: + - script: ROOTFS_DIR=$(rootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skipnuget -skiprestore + displayName: Build product - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: # TODO: IBCOptimize? EnforcePGO? pass an OfficialBuildId? SignType? file logging parameters? - script: set __TestIntermediateDir=int&&build.cmd $(buildConfig) $(archType) -skiptests -skipbuildpackages -skiprestore @@ -57,11 +70,11 @@ jobs: # Upload build as pipeline artifact - - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}: + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - task: PublishPipelineArtifact@0 displayName: Save product build as pipeline artifact inputs: - artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroup, parameters.archType, parameters.buildConfig) }} + artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroupName, parameters.archType, parameters.buildConfig) }} targetPath: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper) - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - task: PublishPipelineArtifact@0 diff --git a/eng/platform-matrix.yml b/eng/platform-matrix.yml index 7f1a463639..fb1a3c66ed 100644 --- a/eng/platform-matrix.yml +++ b/eng/platform-matrix.yml @@ -5,6 +5,59 @@ parameters: jobs: +# TODO: simplify osGroupName by adding osGroup and osSubGroup. See +# https://github.com/Microsoft/azure-pipelines-yaml/pull/46 for more information + +# Linux arm +- template: ${{ parameters.jobTemplate }} + parameters: + buildConfig: ${{ parameters.buildConfig }} + archType: arm + osGroup: Linux + osGroupName: Linux + containerName: ubuntu_1404_arm_cross_build_image + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux arm64 +- template: ${{ parameters.jobTemplate }} + parameters: + buildConfig: ${{ parameters.buildConfig }} + archType: arm64 + osGroup: Linux + osGroupName: Linux + containerName: ubuntu_1604_arm64_cross_build_image + ${{ insert }}: ${{ parameters.jobParameters }} + +# Linux musl +- template: ${{ parameters.jobTemplate }} + parameters: + buildConfig: ${{ parameters.buildConfig }} + archType: x64 + osGroup: Linux + osGroupName: Linux_musl + containerName: musl_x64_build_image + ${{ insert }}: ${{ parameters.jobParameters }} + +# RHEL 6 +- template: ${{ parameters.jobTemplate }} + parameters: + buildConfig: ${{ parameters.buildConfig }} + archType: x64 + osGroup: Linux + osGroupName: Linux_rhel6 + containerName: centos6_x64_build_image + ${{ insert }}: ${{ parameters.jobParameters }} + +# RHEL 7 +- template: ${{ parameters.jobTemplate }} + parameters: + buildConfig: ${{ parameters.buildConfig }} + archType: x64 + osGroup: Linux + osGroupName: Linux_rhel7 + containerName: centos7_x64_build_image + ${{ insert }}: ${{ parameters.jobParameters }} + # Linux x64 - template: ${{ parameters.jobTemplate }} @@ -12,6 +65,8 @@ jobs: buildConfig: ${{ parameters.buildConfig }} archType: x64 osGroup: Linux + osGroupName: Linux + containerName: ubuntu_1604_x64_build_image ${{ insert }}: ${{ parameters.jobParameters }} # macOS x64 @@ -21,6 +76,7 @@ jobs: buildConfig: ${{ parameters.buildConfig }} archType: x64 osGroup: OSX + osGroupName: OSX ${{ insert }}: ${{ parameters.jobParameters }} # Windows x64/x86/arm/arm64 @@ -30,6 +86,7 @@ jobs: buildConfig: ${{ parameters.buildConfig }} archType: x64 osGroup: Windows_NT + osGroupName: Windows_NT ${{ insert }}: ${{ parameters.jobParameters }} - template: ${{ parameters.jobTemplate }} @@ -37,6 +94,7 @@ jobs: buildConfig: ${{ parameters.buildConfig }} archType: x86 osGroup: Windows_NT + osGroupName: Windows_NT ${{ insert }}: ${{ parameters.jobParameters }} - template: ${{ parameters.jobTemplate }} @@ -44,6 +102,7 @@ jobs: buildConfig: ${{ parameters.buildConfig }} archType: arm osGroup: Windows_NT + osGroupName: Windows_NT ${{ insert }}: ${{ parameters.jobParameters }} - template: ${{ parameters.jobTemplate }} @@ -51,11 +110,5 @@ jobs: buildConfig: ${{ parameters.buildConfig }} archType: arm64 osGroup: Windows_NT + osGroupName: Windows_NT ${{ insert }}: ${{ parameters.jobParameters }} - -# TODO for official build: -# RedHat x64 -# Linux crossbuild arm -# Linux crossbuild arm64 -# Linux musl x64 - diff --git a/eng/test-job.yml b/eng/test-job.yml index 0ec46d1387..00982f762b 100644 --- a/eng/test-job.yml +++ b/eng/test-job.yml @@ -2,6 +2,7 @@ parameters: buildConfig: '' archType: '' osGroup: '' + osGroupName: '' priority: 0 crossgen: false scenarios: '' @@ -17,14 +18,15 @@ jobs: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} + osGroupName: ${{ parameters.osGroupName }} # Compute job name from template parameters ${{ if eq(parameters.crossgen, 'false') }}: - name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osGroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Test pri{0} {1} {2} {3}', parameters.priority, parameters.osGroup, parameters.archType, parameters.buildConfig) }} + name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test pri{0} {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }} ${{ if eq(parameters.crossgen, 'true') }}: - name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osGroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osGroup, parameters.archType, parameters.buildConfig) }} + name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }} variables: # Map template parameters to command line arguments @@ -52,12 +54,16 @@ jobs: condition: false # Test job depends on the corresponding build job - dependsOn: ${{ format('build_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }} + dependsOn: ${{ format('build_{0}_{1}_{2}', parameters.osGroupName, parameters.archType, parameters.buildConfig) }} + + # Run all steps in the container. + # Note that the containers are resources defined in azure-pipelines.yml + containerName: ${{ parameters.containerName }} steps: # Install test build dependencies - - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}: + - ${{ if eq(parameters.osGroup, 'OSX') }}: - script: sh eng/install-native-dependencies.sh $(osGroup) displayName: Install native dependencies @@ -67,13 +73,13 @@ jobs: - task: DownloadPipelineArtifact@0 displayName: Download product build pipeline artifact inputs: - artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroup, parameters.archType, parameters.buildConfig) }} + artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroupName, parameters.archType, parameters.buildConfig) }} targetPath: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper) - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - task: DownloadPipelineArtifact@0 displayName: Download product build pipeline artifact inputs: - artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroup, parameters.archType, parameters.buildConfig) }} + artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroupName, parameters.archType, parameters.buildConfig) }} targetPath: $(Build.SourcesDirectory)\bin\Product\Windows_NT.$(archType).$(buildConfigUpper) @@ -105,7 +111,7 @@ jobs: env: ${{ if eq(variables['System.TeamProject'], 'internal') }}: # Access token variable for internal project - HelixAccessToken: $(HelixApiAccessToken) + HelixAccessToken: $(DotNet-HelixApi-Access) ${{ if eq(variables['System.TeamProject'], 'public') }}: # Access token variable for public project HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token) @@ -115,7 +121,7 @@ jobs: env: ${{ if eq(variables['System.TeamProject'], 'internal') }}: # Access token variable for internal project - HelixAccessToken: $(HelixApiAccessToken) + HelixAccessToken: $(DotNet-HelixApi-Access) ${{ if eq(variables['System.TeamProject'], 'public') }}: # Access token variable for public project HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token) diff --git a/eng/xplat-job.yml b/eng/xplat-job.yml index b2d04df537..9677da4bd4 100644 --- a/eng/xplat-job.yml +++ b/eng/xplat-job.yml @@ -2,10 +2,12 @@ parameters: buildConfig: '' archType: '' osGroup: '' + osGroupName: '' name: '' displayName: '' condition: '' dependsOn: '' + containerName: '' variables: {} ## any extra variables to add to the defaults defined below jobs: @@ -20,11 +22,13 @@ jobs: dependsOn: ${{ parameters.dependsOn }} queue: - ${{ if eq(parameters.osGroup, 'Linux') }}: + ${{ if and(eq(parameters.osGroup, 'Linux'), eq(variables['System.TeamProject'], 'public')) }}: name: Hosted Ubuntu 1604 - ${{ if and(eq(parameters.osGroup, 'Windows_NT'), ne(variables['System.TeamProject'], 'public')) }}: + ${{ if and(eq(parameters.osGroup, 'Linux'), ne(variables['System.TeamProject'], 'public')) }}: + name: dnceng-linux-internal-temp + ${{ if and(eq(parameters.osGroup, 'OSX'), ne(variables['System.TeamProject'], 'public')) }}: name: Hosted Mac Internal - ${{ if and(eq(parameters.osGroup, 'Windows_NT'), eq(variables['System.TeamProject'], 'public')) }}: + ${{ if and(eq(parameters.osGroup, 'OSX'), eq(variables['System.TeamProject'], 'public')) }}: name: Hosted MacOS ${{ if and(eq(parameters.osGroup, 'Windows_NT'), ne(variables['System.TeamProject'], 'public')) }}: name: dotnet-internal-temp @@ -32,6 +36,9 @@ jobs: name: dotnet-external-temp timeoutInMinutes: 240 + ${{ if ne(parameters.containerName, '') }}: + container: ${{ parameters.containerName }} + ${{ if eq(parameters.osGroup, 'Linux') }}: agentOs: Ubuntu ${{ if eq(parameters.osGroup, 'OSX') }}: @@ -49,6 +56,15 @@ jobs: buildConfigUpper: 'Release' archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} + osGroupName: ${{ parameters.osGroupName }} + + # Crossbuild specific variables. + ${{ if eq(parameters.archType, 'arm') }}: + rootfsDir: /crossrootfs/arm + cacRootfsDir: /crossrootfs/x86 + ${{ if eq(parameters.archType, 'arm64') }}: + rootfsDir: /crossrootfs/arm64 + ${{insert}}: ${{ parameters.variables }} steps: ${{ parameters.steps }} |