Refactor CI pipeline to separate build and test stages

azure-pipelines.yml

@@ -42,31 +42,24 @@ variables:
 
 stages:
 
-- stage: build
-  displayName: Build
+- stage: BuildAndPack
+  displayName: Build and Pack
   jobs:
-
   - template: /eng/common/templates/jobs/jobs.yml
     parameters:
       enableMicrobuild: true
       enablePublishBuildArtifacts: true
-      enablePublishTestResults: true
-      testResultsFormat: 'vstest'
+      enablePublishTestResults: false
       enablePublishBuildAssets: true
       enablePublishUsingPipelines: true
       enableTelemetry: true
       jobs:
-      - job: Windows
+      - job: Windows_BuildAndPack
+        displayName: Windows Build and Pack
         timeoutInMinutes: 90
         pool:
           name: NetCore-Public
           demands: ImageOverride -equals windows.vs2022preview.amd64.open
-        strategy:
-          matrix:
-            Release:
-              _BuildConfig: Release
-            Debug:
-              _BuildConfig: Debug
         steps:
         - task: PowerShell@2
           displayName: 'Install Windows SDK'
@@ -83,15 +76,7 @@ stages:
             filePath: ./eng/install-procdump.ps1
             failOnStderr: true
             showWarnings: true
-        - task: PowerShell@2
-          displayName: 'Enable local dumps'
-          inputs:
-            targetType: 'inline'
-            script: |
-              New-Item -Path $(Build.SourcesDirectory)\artifacts\CrashDumps -ItemType Directory -Force
-              Get-ChildItem "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps"
-              New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpFolder" -Value "$(Build.SourcesDirectory)\artifacts\CrashDumps" -PropertyType ExpandString -Force
-              New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpCount" -Value 10 -PropertyType DWord -Force
+
         - task: PowerShell@2
           displayName: 'Install Access Database Engine'
           inputs:
@@ -100,196 +85,42 @@ stages:
             failOnStderr: true
             showWarnings: true
 
+        # Build only production projects, no tests
         - script: eng\common\CIBuild.cmd
-            -configuration $(_BuildConfig)
+            -configuration Release
             -prepareMachine
             /p:Publish=false
+            /p:ProductsToBuild=all
             /p:Test=false
-            /p:FastAcceptanceTest=true
-          name: Build
-          displayName: Build
+          name: BuildAndPack
+          displayName: Build and Pack
 
         - task: PublishBuildArtifacts@1
-          displayName: 'Publish NuGet packages'
+          displayName: 'Publish NuGet packages for testing'
           inputs:
-            PathtoPublish: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping'
-            ArtifactName: '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)'
+            PathtoPublish: '$(Build.SourcesDirectory)/artifacts/packages/Release'
+            ArtifactName: 'NuGetPackages'
           condition: always()
 
-        - ${{ if eq(parameters.SkipTests, False) }}:
-
-          # Because the build step is using -ci flag, restore is done in a local .packages directory.
-          # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them.
-          # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set.
-          - script: dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig)\TestStep.binlog --no-progress
-            name: Test
-            displayName: Test
-            env:
-              DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet
-              NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages
-              DOTNET_CLI_CONTEXT_VERBOSE: 1
-
-          - task: PublishBuildArtifacts@1
-            displayName: 'Publish Test Results folders'
-            inputs:
-              PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
-              ArtifactName: TestResults_Windows_$(_BuildConfig)_Attempt$(System.JobAttempt)
-            condition: always()
-
-          - task: CopyFiles@2
-            displayName: 'Copy binlogs'
-            inputs:
-              SourceFolder: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite'
-              Contents: |
-                **/*.binlog
-              TargetFolder: '$(Build.ArtifactStagingDirectory)/binlogs'
-            condition: always()
-
-          - task: PublishBuildArtifacts@1
-            displayName: 'Publish integration tests binlogs'
-            inputs:
-              PathtoPublish: '$(Build.ArtifactStagingDirectory)/binlogs'
-              ArtifactName: Integration_Tests_Windows_Binlogs_$(_BuildConfig)
-            condition: always()
-
-          # Upload code coverage to codecov.io
-          - script: dotnet msbuild -restore
-              eng/CodeCoverage.proj
-              /p:Configuration=$(_BuildConfig)
-              /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\CodeCoverage.binlog
-            displayName: Upload coverage to codecov.io
-            condition: and(succeeded(), eq(variables._BuildConfig, 'Debug'))
-
-          - task: PublishBuildArtifacts@1
-            displayName: 'Publish local dumps'
-            inputs:
-              PathtoPublish: '$(Build.SourcesDirectory)/artifacts/CrashDumps'
-              ArtifactName: TestResults_Windows_$(_BuildConfig)
-            condition: failed()
-
-      - job: Linux
-        timeoutInMinutes: 90
-        pool:
-          name: NetCore-Public
-          demands: ImageOverride -equals build.ubuntu.2004.amd64.open
-        strategy:
-          matrix:
-            Release:
-              _BuildConfig: Release
-            Debug:
-              _BuildConfig: Debug
-        steps:
-        - script: eng/common/cibuild.sh
-            -configuration $(_BuildConfig)
-            -prepareMachine
-            /p:Test=false
-            /p:Publish=false
-            /p:NonWindowsBuild=true
-            /p:FastAcceptanceTest=true
-          displayName: Build
-
-        - task: PublishBuildArtifacts@1
-          displayName: 'Publish NuGet packages'
-          inputs:
-            PathtoPublish: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping'
-            ArtifactName: '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)'
-          condition: always()
-
-        - ${{ if eq(parameters.SkipTests, False) }}:
-          # Because the build step is using -ci flag, restore is done in a local .packages directory.
-          # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them.
-          # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set.
-          - script: dotnet test --solution NonWindowsTests.slnf -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig)\TestStep.binlog --no-progress
-            name: Test
-            displayName: Test
-            env:
-              DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet
-              NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages
-              DOTNET_CLI_CONTEXT_VERBOSE: 1
-
-          - task: PublishBuildArtifacts@1
-            displayName: 'Publish Test Results folders'
-            inputs:
-              PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
-              ArtifactName: TestResults_Linux_$(_BuildConfig)_Attempt$(System.JobAttempt)
-            condition: always()
-
-          - task: CopyFiles@2
-            displayName: 'Copy binlogs'
-            inputs:
-              SourceFolder: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite'
-              Contents: |
-                **/*.binlog
-              TargetFolder: '$(Build.ArtifactStagingDirectory)/binlogs'
-            condition: always()
-
-          - task: PublishBuildArtifacts@1
-            displayName: 'Publish integration tests binlogs'
-            inputs:
-              PathtoPublish: '$(Build.ArtifactStagingDirectory)/binlogs'
-              ArtifactName: Integration_Tests_Linux_Binlogs_$(_BuildConfig)
-            condition: always()
-
-      - job: MacOS
-        timeoutInMinutes: 90
-        pool:
-          name: Azure Pipelines
-          vmImage: macos-latest
-          os: macOS
-        strategy:
-          matrix:
-            Release:
-              _BuildConfig: Release
-            Debug:
-              _BuildConfig: Debug
-        steps:
-        - script: eng/common/cibuild.sh
-            -configuration $(_BuildConfig)
-            -prepareMachine
-            /p:Test=false
-            /p:Publish=false
-            /p:NonWindowsBuild=true
-            /p:FastAcceptanceTest=true
-          displayName: Build
-
-        - task: PublishBuildArtifacts@1
-          displayName: 'Publish NuGet packages'
-          inputs:
-            PathtoPublish: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping'
-            ArtifactName: '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)'
-          condition: always()
-
-        - ${{ if eq(parameters.SkipTests, False) }}:
-          # Because the build step is using -ci flag, restore is done in a local .packages directory.
-          # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them.
-          # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set.
-          - script: dotnet test --solution NonWindowsTests.slnf -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig)\TestStep.binlog --no-progress
-            name: Test
-            displayName: Test
-            env:
-              DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet
-              NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages
-              DOTNET_CLI_CONTEXT_VERBOSE: 1
-
-          - task: PublishBuildArtifacts@1
-            displayName: 'Publish Test Results folders'
-            inputs:
-              PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
-              ArtifactName: TestResults_MacOs_$(_BuildConfig)_Attempt$(System.JobAttempt)
-            condition: always()
-
-          - task: CopyFiles@2
-            displayName: 'Copy binlogs'
-            inputs:
-              SourceFolder: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite'
-              Contents: |
-                **/*.binlog
-              TargetFolder: '$(Build.ArtifactStagingDirectory)/binlogs'
-            condition: always()
-
-          - task: PublishBuildArtifacts@1
-            displayName: 'Publish integration tests binlogs'
-            inputs:
-              PathtoPublish: '$(Build.ArtifactStagingDirectory)/binlogs'
-              ArtifactName: Integration_Tests_MacOS_Binlogs_$(_BuildConfig)
-            condition: always()
+- template: /eng/test-stage-template.yml
+  parameters:
+    stageName: TestWindows
+    displayName: Test Windows
+    os: Windows
+    skipTests: ${{ parameters.SkipTests }}
+
+- template: /eng/test-stage-template.yml
+  parameters:
+    stageName: TestLinux
+    displayName: Test Linux
+    os: Linux
+    skipTests: ${{ parameters.SkipTests }}
+    solutionFile: NonWindowsTests.slnf
+
+- template: /eng/test-stage-template.yml
+  parameters:
+    stageName: TestMacOS
+    displayName: Test macOS
+    os: macOS
+    skipTests: ${{ parameters.SkipTests }}
+    solutionFile: NonWindowsTests.slnf

eng/Build.props

@@ -34,4 +34,22 @@
     <ProjectToBuild Include="$(RepoRoot)Microsoft.Testing.Platform.slnf" />
   </ItemGroup>
 
+  <!-- When building tests only -->
+  <ItemGroup Condition=" '$(ProductsToBuild)' == 'tests' ">
+    <!-- Unit Tests -->
+    <ProjectToBuild Include="$(RepoRoot)test/UnitTests/**/*.csproj" />
+    <!-- Integration Tests -->
+    <ProjectToBuild Include="$(RepoRoot)test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSTest.Acceptance.IntegrationTests.csproj" />
+    <ProjectToBuild Include="$(RepoRoot)test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests.csproj" />
+    <ProjectToBuild Include="$(RepoRoot)test/IntegrationTests/MSTest.IntegrationTests/MSTest.IntegrationTests.csproj" />
+    <ProjectToBuild Include="$(RepoRoot)test/IntegrationTests/PlatformServices.Desktop.IntegrationTests/PlatformServices.Desktop.IntegrationTests.csproj" />
+    <ProjectToBuild Include="$(RepoRoot)test/IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests/MSTest.VstestConsoleWrapper.IntegrationTests.csproj" />
+    <!-- Test Utilities -->
+    <ProjectToBuild Include="$(RepoRoot)test/Utilities/Microsoft.Testing.TestInfrastructure/Microsoft.Testing.TestInfrastructure.csproj" />
+    <ProjectToBuild Include="$(RepoRoot)test/Utilities/TestFramework.ForTestingMSTest/TestFramework.ForTestingMSTest.csproj" />
+    <ProjectToBuild Include="$(RepoRoot)test/Utilities/Automation.CLI/Automation.CLI.csproj" />
+    <!-- Performance Tests -->
+    <ProjectToBuild Include="$(RepoRoot)test/Performance/MSTest.Performance.Runner/MSTest.Performance.Runner.csproj" />
+  </ItemGroup>
+
 </Project>