NOT readay: Add dynamic runtimesettings (#8341)

Build/azuredevops/test/uitest-integration-stage.yml

@@ -0,0 +1,149 @@
+parameters:
+  - name: BinariesDirectory
+    type: string
+    default: '$(Drop)/binaries'
+  - name: CleanupProcesses
+    type: string
+    default: 'devenv, vshub, microsoft.vshub.server.httphost, python, ipy, microsoft.pythontools.analyzer, vstest.executionengine.x86, vstest.discoveryengine.x86'
+  - name: Drop
+    type: string
+    default: '$(System.ArtifactsDirectory)/$(DefinitionName)'
+  - name: TestFilterCriteria
+    type: string
+    default: 'Priority=0'
+  - name: UITestNames
+    type: string
+    default: ''
+
+steps:
+  - task: ExtractFiles@1
+    displayName: Extract files
+    condition: succeeded()
+    inputs:
+      archiveFilePatterns: '${{ parameters.Drop }}/TestData/TestData.zip'
+      destinationFolder: '${{ parameters.Drop }}/TestData'
+      cleanDestinationFolder: false
+      overwriteExistingFiles: false
+
+  - task: PowerShell@2
+    displayName: Cleanup
+    condition: succeeded()
+    continueOnError: true
+    timeoutInMinutes: 5
+    inputs:
+      targetType: inline
+      arguments: '${{ parameters.CleanupProcesses }}'
+      script: |
+        param ([string[]]$names)
+        $running = Get-Process -Name $names -ErrorAction SilentlyContinue
+        while ($running) {
+          Write-Host 'Killing:'
+          $running
+          try {
+            $running | Stop-Process -Force
+          } catch {
+          }
+          Start-Sleep -Seconds 3
+          $running = Get-Process -Name $names -ErrorAction SilentlyContinue
+        }
+      errorActionPreference: stop
+      failOnStderr: true
+
+  - task: CopyFiles@2
+    displayName: Copy TestData into binaries
+    condition: succeeded()
+    continueOnError: true
+    inputs:
+      SourceFolder: '${{ parameters.Drop }}/TestData'
+      Contents: '**'
+      TargetFolder: '${{ parameters.BinariesDirectory }}/TestData'
+      CleanTargetFolder: 'true'
+      OverWrite: 'false'
+      flattenFolders: 'false'
+      preserveTimestamp: 'false'
+      retryCount: '0'
+
+  - task: CopyFiles@2
+    displayName: Copy runsettings to binary folder
+    condition: succeeded()
+    inputs:
+      SourceFolder: '${{ parameters.BinariesDirectory }}/TestData'
+      Contents: 'default.runsettings'
+      TargetFolder: '${{ parameters.BinariesDirectory }}'
+      CleanTargetFolder: 'false'
+      OverWrite: 'false'
+      flattenFolders: 'false'
+      preserveTimestamp: 'false'
+      retryCount: '0'
+
+  - task: PowerShell@2
+    displayName: Skip Verification
+    condition: succeeded()
+    inputs:
+      targetType: inline
+      script: |
+        reg import "${{ parameters.BinariesDirectory }}\TestData\EnableSkipVerification.reg"
+      errorActionPreference: stop
+      failOnStderr: false
+
+  - task: VSTest@2
+    displayName: '[MultiConfiguration] Run UI Test ${{ parameters.UITestNames }}'
+    condition: succeeded()
+    continueOnError: true
+    inputs:
+      testSelector: testAssemblies
+      testAssemblyVer2: '${{ parameters.UITestNames }}UITestsRunner.dll'
+      tcmTestRun: '$(test.RunId)'
+      searchFolder: '${{ parameters.BinariesDirectory }}'
+      resultsFolder: '$(Agent.TempDirectory)/TestResults'
+      testFiltercriteria: '${{ parameters.TestFilterCriteria }}'
+      runOnlyImpactedTests: false
+      runAllTestsAfterXBuilds: 50
+      uiTests: true
+      vstestLocationMethod: version
+      vsTestVersion: latest
+      runSettingsFile: '${{ parameters.BinariesDirectory }}/default.runsettings'
+      overrideTestrunParameters: '-TargetFrameworkVersion .NETFramework,Version=v4.7.2'
+      runInParallel: false
+      runTestsInIsolation: false
+      codeCoverageEnabled: true
+      distributionBatchType: basedOnExecutionTime
+      batchingBasedOnAgentsOption: autoBatchSize
+      customBatchSizeValue: 10
+      batchingBasedOnExecutionTimeOption: customTimeBatchSize
+      customRunTimePerBatchValue: 300
+      dontDistribute: false
+      testRunTitle: '${{ parameters.UITestNames }} UI Tests'
+      publishRunAttachments: true
+      failOnMinTestsNotRun: false
+      minimumExpectedTests: 1
+      diagnosticsEnabled: true
+      collectDumpOn: onAbortOnly
+      rerunFailedTests: true
+      rerunType: basedOnTestFailurePercentage
+      rerunFailedThreshold: 50
+      rerunFailedTestCasesMaxLimit: 5
+      rerunMaxAttempts: 2
+
+  - task: PowerShell@2
+    displayName: Cleanup
+    condition: succeeded()
+    timeoutInMinutes: 5
+    inputs:
+      targetType: inline
+      arguments: '${{ parameters.CleanupProcesses }}'
+      script: |
+        param ([string[]]$names)
+        $running = Get-Process -Name $names -ErrorAction SilentlyContinue
+        while ($running) {
+          Write-Host 'Killing:'
+          $running
+          try {
+            $running | Stop-Process -Force
+          } catch {
+          }
+          Start-Sleep -Seconds 3
+          $running = Get-Process -Name $names -ErrorAction SilentlyContinue
+        }
+      errorActionPreference: stop
+      failOnStderr: true

Build/templates/build.yml

@@ -2,6 +2,9 @@ parameters:
 - name: ptvsPackageVersion
   type: string
   default: $(Build.BuildNumber)
+- name: skipSigning
+  type: boolean
+  default: false
 
 steps:
 
@@ -57,4 +60,5 @@ steps:
                           /p:BuildNumber=${{ parameters.ptvsPackageVersion }} 
                           /p:BUILD_BINARIESDIRECTORY=$(Build.BinariesDirectory)
                           /p:BUILD_STAGINGDIRECTORY=$(Build.StagingDirectory) 
+                          /p:SkipSigning=${{ parameters.skipSigning }}
                           /bl:$(Build.SourcesDirectory)\logs\BuildInstaller.binlog'

Python/Setup/signlayout.proj

@@ -9,6 +9,10 @@
   <PropertyGroup>
     <OutputPath>$(BinariesOutputPath)</OutputPath>
     <OutDir>$(BinariesOutputPath)</OutDir>
+    <!-- Opt-in flag to skip signing work -->
+    <SkipSigning Condition="'$(SkipSigning)'==''">false</SkipSigning>
+    <!-- When skipping, set MicroBuildSignType to None so MicroBuild signing targets are disabled -->
+    <MicroBuildSignType Condition="'$(SkipSigning)'=='true'">None</MicroBuildSignType>
   </PropertyGroup>
 
   <ItemDefinitionGroup>
@@ -17,7 +21,7 @@
     </ManagedFiles>
   </ItemDefinitionGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(SkipSigning)'!='true'">
     <_Languages Include="$(Languages)" />
 
     <ManagedFiles Include="
@@ -73,7 +77,7 @@
     </FilesToSign>
   </ItemGroup>
 
-  <Target Name="_AddLocFiles" BeforeTargets="_PreserveUnsigned;ListFiles">
+  <Target Name="_AddLocFiles" BeforeTargets="_PreserveUnsigned;ListFiles" Condition="'$(SkipSigning)'!='true'">
     <ItemGroup>
       <LocFilesToSign Include="$(BinariesOutputPath)localize\**\*.resources.dll" />
       <FilesToSign Include="@(LocFilesToSign)">
@@ -84,7 +88,7 @@
       </FilesToSign>
     </ItemGroup>
   </Target>
-  <Target Name="_AddPylanceFiles" BeforeTargets="_PreserveUnsigned;ListFiles">
+  <Target Name="_AddPylanceFiles" BeforeTargets="_PreserveUnsigned;ListFiles" Condition="'$(SkipSigning)'!='true'">
     <ItemGroup>
       <PylanceFilesToSign Include="$(BinariesOutputPath)pylance\**\*.js" />
       <FilesToSign Include="@(PylanceFilesToSign)">
@@ -96,17 +100,17 @@
     </ItemGroup>
   </Target>
 
-  <Target Name="_PreserveUnsigned" Inputs="%(FilesToSign.FullPath)" Outputs="%(FilesToSign.UnsignedPath)" BeforeTargets="SignFiles">
+  <Target Name="_PreserveUnsigned" Inputs="%(FilesToSign.FullPath)" Outputs="%(FilesToSign.UnsignedPath)" BeforeTargets="SignFiles" Condition="'$(SkipSigning)'!='true' and '@(FilesToSign)'!=''">
     <Copy SourceFiles="%(FilesToSign.FullPath)" DestinationFiles="%(FilesToSign.UnsignedPath)" />
   </Target>
 
-  <Target Name="ListFiles">
+  <Target Name="ListFiles" Condition="'$(SkipSigning)'!='true'">
     <Message Text="OutputPath: $(OutputPath)" Importance="high" />
     <Message Text="@(FilesToSign->'%(Identity) (%(Authenticode), %(StrongName)) - %(SignedPath)', '
 ')" Importance="high" />
   </Target>
 
-  <Target Name="_GetBinariesInLayout">
+  <Target Name="_GetBinariesInLayout" Condition="'$(SkipSigning)'!='true'">
     <ItemGroup>
       <_BinariesInLayout Include="$(LayoutOutputPath)**\*.exe;$(LayoutOutputPath)**\*.dll" />
       <_BinariesWithSignedSource Include="$(BinariesOutputPath)%(_BinariesInLayout.Filename)%(_BinariesInLayout.Extension)">
@@ -116,7 +120,7 @@
     </ItemGroup>
   </Target>
 
-  <Target Name="_GetJSFilesInLayout">
+  <Target Name="_GetJSFilesInLayout" Condition="'$(SkipSigning)'!='true'">
     <ItemGroup>
       <!-- Special case the pylance files -->
       <_FilesInLayout Include="$(LayoutOutputPath)\Microsoft.PythonTools.Core\pylance\**\*.js" />
@@ -127,10 +131,10 @@
     </ItemGroup>
   </Target>
 
-
   <Target Name="_UpdateLayoutWithSignedBinaries"
           DependsOnTargets="_GetBinariesInLayout;_GetJsFilesInLayout"
-          AfterTargets="SignFiles">
+          AfterTargets="SignFiles"
+          Condition="'$(SkipSigning)'!='true'">
     <Copy SourceFiles="%(_BinariesToCopy.FullPath)" DestinationFiles="%(_BinariesToCopy.TargetPath)" />
   </Target>
 

Python/Tests/Core.UI/TestExplorerTests.cs

@@ -65,7 +65,7 @@ public void RunAllUnittestProject(PythonVisualStudioApp app) {
             var sln = app.CopyProjectForTest(@"TestData\TestExplorerUnittest.sln");
             app.OpenProject(sln);
 
-            RunAllTests(app, AllUnittests);
+            RunAllTests(app, AllUnittests, Path.GetFileNameWithoutExtension(sln));
         }
 
         public void RunAllUnittestWorkspace(PythonVisualStudioApp app) {

Python/Tests/PythonToolsUITestsRunner/TestExplorerTests.cs

@@ -75,12 +75,14 @@ public void DebugPytestWorkspace() {
             _vs.RunTest(nameof(PythonToolsUITests.TestExplorerTests.DebugPytestWorkspace));
         }
 
+        [Ignore]
         [TestMethod, Priority(UITestPriority.P0)]
         [TestCategory("Installed")]
         public void DebugUnittestProject() {
             _vs.RunTest(nameof(PythonToolsUITests.TestExplorerTests.DebugUnittestProject));
         }
 
+        [Ignore]
         [TestMethod, Priority(UITestPriority.P0)]
         [TestCategory("Installed")]
         public void DebugUnittestWorkspace() {