path: root/eng/test-job.yml

parameters:
  buildConfig: ''
  archType: ''
  osGroup: ''
  osIdentifier: ''
  container: ''
  testGroup: ''
  readyToRun: false
  helixQueues: ''
  crossrootfsDir: ''
  # If true, run the corefx tests instead of the coreclr ones
  corefxTests: false
  displayNameArgs: ''
  runInUnloadableContext: false
  ignoreDependencyOnBuildJobs: false

### Test job

### Each test job depends on a corresponding build job with the same
### buildConfig and archType.

jobs:
- template: xplat-job.yml
  parameters:
    buildConfig: ${{ parameters.buildConfig }}
    archType: ${{ parameters.archType }}
    osGroup: ${{ parameters.osGroup }}
    osIdentifier: ${{ parameters.osIdentifier }}
    helixType: 'build/tests/'

    # Test jobs should continue on error for internal builds
    ${{ if eq(variables['System.TeamProject'], 'internal') }}:
      continueOnError: true

    # Compute job name from template parameters
    ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}:
      name: ${{ format('test_{0}_{1}_{2}_{3}', 'p0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
      displayName: ${{ format('Test {0} {1} {2} {3}', 'Pri0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}

    ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}:
      name: ${{ format('test_{0}_{1}_{2}_{3}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
      displayName: ${{ format('Test {0} {1} {2} {3}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}

    ${{ if and(eq(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}:
      name: ${{ format('test_{0}_{1}_{2}_{3}_{4}', 'p0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
      displayName: ${{ format('Test {0} {1} {2} {3} {4}', 'Pri0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}

    ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}:
      name: ${{ format('test_{0}_{1}_{2}_{3}_{4}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
      displayName: ${{ format('Test {0} {1} {2} {3} {4}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}

    crossrootfsDir: ${{ parameters.crossrootfsDir }}

    variables:
    - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
      - group: DotNet-HelixApi-Access

    # Map template parameters to command line arguments
    - name: priorityArg
      value: ''
    - ${{ if ne(parameters.testGroup, 'innerloop') }}:
      - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
        - name: priorityArg
          value: 'priority1'
      - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
        - name: priorityArg
          value: '-priority=1'

    - name: crossgenArg
      value: ''
    - ${{ if eq(parameters.readyToRun, true) }}:
      - name: crossgenArg
        value: 'crossgen'

    - name: clangArg
      value: ''
    # Our FreeBSD doesn't yet detect available clang versions, so pass it explicitly.
    - ${{ if eq(parameters.osGroup, 'FreeBSD') }}:
      - name: clangArg
        value: '-clang6.0'
    - ${{ if eq(parameters.archType, 'arm64') }}:
      - name: clangArg
        value: '-clang5.0'

    - name: testhostArg
      value: ''
    - ${{ if eq(parameters.corefxTests, true) }}:
      - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
        - name: testhostArg
          value: 'generatetesthostonly'
      - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
        - name: testhostArg
          value: 'buildtesthostonly'

    # FreeBSD test jobs are disabled since we don't have any FreeBSD helix queues.
    ${{ if eq(parameters.osGroup, 'FreeBSD') }}:
      condition: false

    # Test job depends on the corresponding build job
    ${{ if ne(parameters.ignoreDependencyOnBuildJobs, true) }}:
      dependsOn: ${{ format('build_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}

    # Run all steps in the container.
    # Note that the containers are defined in platform-matrix.yml
    container: ${{ parameters.container }}

    # "Total" means building tests, waiting for a queue in Helix to become available, and running the tests.
    # In case test-job.yml gets split into two separate jobs (e.g. build-test-job.yml and run-test.yml)
    # this number should be adjusted accordingly.
    ${{ if eq(parameters.testGroup, 'innerloop') }}:
      timeoutInMinutes: 240
    ${{ if in(parameters.testGroup, 'outerloop') }}:
      timeoutInMinutes: 360
    ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'gcstress0x3-gcstress0xc') }}:
      timeoutInMinutes: 480
    ${{ if in(parameters.testGroup, 'jitstress-isas-x86', 'gcstress-extra', 'r2r-extra') }}:
      timeoutInMinutes: 600

    steps:

    # Install test build dependencies
    - ${{ if eq(parameters.osGroup, 'OSX') }}:
      - script: sh eng/install-native-dependencies.sh $(osGroup)
        displayName: Install native dependencies


    # Download product binaries directory
    - task: DownloadBuildArtifacts@0
      displayName: Download product build
      inputs:
        buildType: current
        downloadType: single
        artifactName: ${{ format('BinDir_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
        downloadPath: $(System.ArtifactsDirectory)


    # Populate Product directory
    - task: CopyFiles@2
      displayName: Populate Product directory
      inputs:
        sourceFolder: $(System.ArtifactsDirectory)/${{ format('BinDir_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
        contents: '**'
        targetFolder: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper)


    # Build tests
    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
      - script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(crossgenArg) $(clangArg) $(testhostArg)
        displayName: Build tests
    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
      - script: build-test.cmd $(buildConfig) $(archType) $(priorityArg) $(crossgenArg) $(testhostArg)
        displayName: Build tests


    # Send tests to Helix
    - template: /eng/send-to-helix-step.yml
      parameters:
        displayName: Send tests to Helix
        buildConfig: $(buildConfigUpper)
        archType: ${{ parameters.archType }}
        osGroup: ${{ parameters.osGroup }}

        ${{ if eq(variables['System.TeamProject'], 'public') }}:
          creator: $(Build.DefinitionName)

        helixBuild: $(Build.BuildNumber)
        helixSource: $(_HelixSource)

        # REVIEW: not sure why "cli" is part of the names here. Leave it for the ones that already had it,
        # but don't add it to new ones.
        ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
          helixType: 'test/functional/r2r_corefx/'
        ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
          helixType: 'test/functional/corefx/'
        ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
          helixType: 'test/functional/r2r/cli/'
        ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
          helixType: 'test/functional/cli/'

        helixQueues: ${{ parameters.helixQueues }}

        # This tests whether an array is empty
        ${{ if eq(join('', parameters.helixQueues), '') }}:
          condition: false

        publishTestResults: true

        # Set job timeouts
        #
        # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete.
        # In case xUnit test wrappers get refactored this number should also be adjusted.
        #
        # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting
        # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the
        # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing.

        ${{ if and(eq(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}:
          timeoutPerTestCollectionInMinutes: 120
        ${{ if and(ne(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}:
          timeoutPerTestCollectionInMinutes: 30
          timeoutPerTestInMinutes: 10
        ${{ if in(parameters.testGroup, 'outerloop') }}:
          timeoutPerTestCollectionInMinutes: 120
          timeoutPerTestInMinutes: 10
        ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs' ) }}:
          timeoutPerTestCollectionInMinutes: 120
          timeoutPerTestInMinutes: 30
        ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}:
          timeoutPerTestCollectionInMinutes: 240
          timeoutPerTestInMinutes: 60
        ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra') }}:
          timeoutPerTestCollectionInMinutes: 300
          timeoutPerTestInMinutes: 90

        runCrossGen: ${{ parameters.readyToRun }}
        runInUnloadableContext: ${{ parameters.runInUnloadableContext }}

        ${{ if eq(variables['System.TeamProject'], 'internal') }}:
          # Access token variable for internal project from the
          # DotNet-HelixApi-Access variable group
          helixAccessToken: $(HelixApiAccessToken)

        # Choose which tests to send to Helix: CoreFX or CoreCLR.
        ${{ if eq(parameters.corefxTests, true) }}:
          helixProjectArguments: 'eng/helixcorefxtests.proj'
        ${{ if ne(parameters.corefxTests, true) }}:
          helixProjectArguments: 'tests/helixpublishwitharcade.proj'

        ${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}:
          scenarios:
          - normal
          - no_tiered_compilation
        ${{ if in(parameters.testGroup, 'jitstress') }}:
          scenarios:
          - jitminopts
          - jitstress1
          - jitstress1_tiered
          - jitstress2
          - jitstress2_tiered
          - zapdisable
          - tailcallstress
        ${{ if in(parameters.testGroup, 'jitstress-isas-arm') }}:
          scenarios:
          - jitstress_isas_incompletehwintrinsic
          - jitstress_isas_nohwintrinsic
          - jitstress_isas_nohwintrinsic_nosimd
          - jitstress_isas_nosimd
        ${{ if in(parameters.testGroup, 'jitstress-isas-x86') }}:
          scenarios:
          - jitstress_isas_incompletehwintrinsic
          - jitstress_isas_nohwintrinsic
          - jitstress_isas_nohwintrinsic_nosimd
          - jitstress_isas_nosimd
          - jitstress_isas_x86_noaes
          - jitstress_isas_x86_noavx
          - jitstress_isas_x86_noavx2
          - jitstress_isas_x86_nobmi1
          - jitstress_isas_x86_nobmi2
          - jitstress_isas_x86_nofma
          - jitstress_isas_x86_nohwintrinsic
          - jitstress_isas_x86_nolzcnt
          - jitstress_isas_x86_nopclmulqdq
          - jitstress_isas_x86_nopopcnt
          - jitstress_isas_x86_nosse
          - jitstress_isas_x86_nosse2
          - jitstress_isas_x86_nosse3
          - jitstress_isas_x86_nosse3_4
          - jitstress_isas_x86_nosse41
          - jitstress_isas_x86_nosse42
          - jitstress_isas_x86_nossse3
        ${{ if in(parameters.testGroup, 'jitstressregs-x86') }}:
          scenarios:
          - jitstressregs1_x86_noavx
          - jitstressregs2_x86_noavx
          - jitstressregs3_x86_noavx
          - jitstressregs4_x86_noavx
          - jitstressregs8_x86_noavx
          - jitstressregs0x10_x86_noavx
          - jitstressregs0x80_x86_noavx
          - jitstressregs0x1000_x86_noavx
        ${{ if in(parameters.testGroup, 'jitstressregs' ) }}:
          scenarios:
          - jitstressregs1
          - jitstressregs2
          - jitstressregs3
          - jitstressregs4
          - jitstressregs8
          - jitstressregs0x10
          - jitstressregs0x80
          - jitstressregs0x1000
        ${{ if in(parameters.testGroup, 'jitstress2-jitstressregs') }}:
          scenarios:
          - jitstress2_jitstressregs1
          - jitstress2_jitstressregs2
          - jitstress2_jitstressregs3
          - jitstress2_jitstressregs4
          - jitstress2_jitstressregs8
          - jitstress2_jitstressregs0x10
          - jitstress2_jitstressregs0x80
          - jitstress2_jitstressregs0x1000
        ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}:
          scenarios:
          - gcstress0x3
          - gcstress0xc
        ${{ if in(parameters.testGroup, 'gcstress-extra') }}:
          scenarios:
          - heapverify1
          - gcstress0xc_zapdisable
          - gcstress0xc_zapdisable_jitstress2
          - gcstress0xc_zapdisable_heapverify1
          - gcstress0xc_jitstress1
          - gcstress0xc_jitstress2
          - gcstress0xc_jitminopts_heapverify1
        ${{ if in(parameters.testGroup, 'r2r-extra') }}:
          scenarios:
          - jitstress1
          - jitstress2
          - jitstress1_tiered
          - jitstress2_tiered
          - jitstressregs1
          - jitstressregs2
          - jitstressregs3
          - jitstressregs4
          - jitstressregs8
          - jitstressregs0x10
          - jitstressregs0x80
          - jitstressregs0x1000
          - jitminopts
          - forcerelocs
          - gcstress15

    # Publish Logs
    - task: PublishBuildArtifacts@1
      displayName: Publish Logs
      inputs:
        pathtoPublish: $(Build.SourcesDirectory)/bin/Logs
        ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
          artifactName: ${{ format('TestLogs_r2r_corefx_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
        ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
          artifactName: ${{ format('TestLogs_corefx_{0}_{1}_{2}_{3}',     parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
        ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}:
          artifactName: ${{ format('TestLogs_r2r_{0}_{1}_{2}_{3}',        parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
        ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}:
          artifactName: ${{ format('TestLogs_{0}_{1}_{2}_{3}',            parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }}
      continueOnError: true
      condition: always()