Add azure pipeline yaml files for GHAzDO

CIGate.ps1

@@ -0,0 +1,177 @@
+# This script is used as part of our PR gating strategy. It takes advantage of the GHAzDO REST API to check for CodeQL issues a PR source and target branch. 
+# If there are 'new' issues in the source branch, the script will fail with error code 1. 
+# The script will also log errors, 1 per new CodeQL alert, it will also add PR annotations for the alert
+$pass = ${env:MAPPED_ADO_PAT}
+$orgUri = ${env:SYSTEM_COLLECTIONURI}
+$orgName = $orgUri -replace "^https://dev.azure.com/|/$"
+$project = ${env:SYSTEM_TEAMPROJECT} 
+$repositoryId = ${env:BUILD_REPOSITORY_ID} 
+$prTargetBranch = ${env:SYSTEM_PULLREQUEST_TARGETBRANCH}
+$prSourceBranch = ${env:BUILD_SOURCEBRANCH}
+$prId = ${env:SYSTEM_PULLREQUEST_PULLREQUESTID}
+$prInteration  = ${env:SYSTEM_PULLREQUEST_PULLREQUESTITERATION}
+$pair = ":${pass}"
+$bytes = [System.Text.Encoding]::ASCII.GetBytes($pair)
+$base64 = [System.Convert]::ToBase64String($bytes)
+$basicAuthValue = "Basic $base64"
+$headers = @{ Authorization = $basicAuthValue }
+
+$urlTargetAlerts = "https://advsec.dev.azure.com/{0}/{1}/_apis/Alert/repositories/{2}/Alerts?top=500&orderBy=lastSeen&criteria.alertType=3&criteria.ref={3}&criteria.states=1" -f $orgName, $project, $repositoryId, $prTargetBranch
+$urlSourceAlerts = "https://advsec.dev.azure.com/{0}/{1}/_apis/Alert/repositories/{2}/Alerts?top=500&orderBy=lastSeen&criteria.alertType=3&criteria.ref={3}&criteria.states=1" -f $orgName, $project, $repositoryId, $prSourceBranch
+$urlComment =   "https://dev.azure.com/{0}/{1}/_apis/git/repositories/{2}/pullRequests/{3}/threads?api-version=7.1-preview.1" -f $orgName, $project, $repositoryId, $prId
+$urlIteration = "https://dev.azure.com/{0}/{1}/_apis/git/repositories/{2}/pullRequests/{3}/iterations/{4}/changes?api-version=7.1-preview.1&`$compareTo={5}" -f $orgName, $project, $repositoryId, $prId, $prInteration, ($prInteration-1) 
+
+#Get-ChildItem Env: | Format-Table -AutoSize
+
+# Add a PR annotations for the Alert in the changed file. 
+function AddPRComment($prAlert, $urlAlert) {
+    # Get Pull Request iterations, we need this to map the file to a changeTrackingId
+    $prIterations = Invoke-RestMethod -Uri $urlIteration -Method Get -Headers $headers
+
+    # Find the changeTrackingId mapping to the file with the CodeQL alert
+    $iterationItem = $prIterations.changeEntries | Where-Object { $_.item.path -like "/$($prAlert.physicalLocations[-1].filePath)" } | Select-Object -First 1
+
+    # Any change to the file with the CodeQL alert in this PR iteration?
+    if ($null -eq $iterationItem) {
+        Write-Host "In this iteration of the PR, there is no change to the file with the CodeQL alert. "
+        return
+    }
+
+    $lineEnd = $($prAlert.physicalLocations[-1].region.lineEnd)
+    $lineStart = $($prAlert.physicalLocations[-1].region.lineStart)
+
+    if ($lineEnd -eq 0) {
+        $lineEnd = $lineStart
+    }
+
+    # Define the Body hashtable
+    $body = @{
+        "comments" = @(
+            @{
+                "content" = "**$($prAlert.title)**
+                $($prAlert.tools.rules.description)
+                See details [here]($($urlAlert))"
+                "commentType" = 1
+            }
+        )
+        "status" = 1
+        "threadContext" = @{
+            "filePath" = "./$($prAlert.physicalLocations[-1].filePath)"
+            "rightFileStart" = @{
+                "line" = $lineEnd
+                "offset" = $($prAlert.physicalLocations[-1].region.columnStart)
+            }
+            "rightFileEnd" = @{
+                "line" = $lineStart
+                "offset" = $($prAlert.physicalLocations[-1].region.columnEnd)
+            }
+        }
+        "pullRequestThreadContext" = @{
+            "changeTrackingId" = $($iterationItem.changeTrackingId)
+            "iterationContext" = @{
+                "firstComparingIteration" = $($prInteration)
+                "secondComparingIteration" = $($prInteration)
+              }
+        }
+    }
+
+    # Convert the hashtable to a JSON string
+    $bodyJson = $body | ConvertTo-Json -Depth 10
+
+    # Print the JSON string
+    # Write-Output $bodyJson
+
+    # Send the POST request
+    $response = Invoke-RestMethod -Uri $urlComment -Method Post -Headers $headers -Body $bodyJson -ContentType "application/json"
+
+    # Write-Output $response
+
+}
+
+Write-Host "Will check to see if there are any new CodeQL issues in this PR branch" 
+Write-Host "PR source : $($prSourceBranch). PR target: $($prTargetBranch)"
+
+if (${env:BUILD_REASON} -ne 'PullRequest'){
+   Write-Host "This build is not part of a Pull Request so all is ok"
+   exit 0
+}
+
+# Get the alerts on the pr target branch (all without filter) and the PR source branch (only currently open)
+$alertsPRSource = Invoke-WebRequest -Uri $urlSourceAlerts -Headers $headers -Method Get
+
+# The CodeQL scanning of the target branch runs in a separate pipeline. This scan might not have been completed. 
+# Try to get the results 10 times with a 1 min wait between each try.
+$retries = 10
+while ($retries -gt 0) {
+    try {
+        $alertsPRTarget = Invoke-WebRequest -Uri $urlTargetAlerts -Headers $headers -Method Get -ErrorAction Stop
+        # Success
+        break
+    }
+    catch  {
+        # No GHAzDO results on the target branch, wait and retry?
+        if($_.ErrorDetails.Message.Split("`"") -contains "BranchNotFoundException"){
+            $retries--
+            if($retries -eq 0){
+                # We have retried the maximum number of times, give up
+                Write-Host "##vso[task.logissue type=error] We have retried the maximum number of times, give up."          
+                throw $_
+            }
+
+            # Wait and then try again 
+            Write-Host "There are no GHAzDO results on the target branch, wait and try again."          
+            Start-Sleep -Seconds 60
+        }
+        else {
+            # Something else is wrong, give up
+            Write-Host "##vso[task.logissue type=error] There was an unexpected error."          
+            throw $_
+        }
+    }
+}
+
+if ($alertsPRTarget.StatusCode -ne 200){
+   Write-Host "##vso[task.logissue type=error] Error getting alerts from Azure DevOps Advanced Security PR target branch:", $alertsPRTarget.StatusCode, $alertsPRTarget.StatusDescription
+   exit 1
+}
+
+if ($alertsPRSource.StatusCode -ne 200){
+   Write-Host "##vso[task.logissue type=error] Error getting alerts from Azure DevOps Advanced Security PR source branch:", $alertsPRSource.StatusCode, $alertsPRSource.StatusDescription
+   exit 1
+}
+
+$jsonPRTarget = $alertsPRTarget.Content | ConvertFrom-Json
+$jsonPRSource = $alertsPRSource.Content | ConvertFrom-Json 
+
+# Extract alert ids from the list of alerts on pr target/source branch.
+$prTargetAlertIds = $jsonPRTarget.value | Select-Object -ExpandProperty alertId
+$prSourceAlertIds = $jsonPRSource.value | Select-Object -ExpandProperty alertId
+
+# Check for alert ids that are reported in the PR source branch but not the pr target branch
+$newAlertIds = Compare-Object $prSourceAlertIds $prTargetAlertIds -PassThru | Where-Object { $_.SideIndicator -eq '<=' }
+
+# Are there any new alert ids in the PR source branch?
+if($newAlertIds.length -gt 0) {
+    Write-Host "##[error] The code changes in this PR looks to be introducing new CodeQL alerts:"
+
+    # Loop over the objects in the prAlerts JSON object
+    foreach ($prAlert in $jsonPRSource.value) {
+        if ($newAlertIds -contains $prAlert.alertId) {
+            # This is a new Alert for this PR. Log and report it.
+            Write-Host  ""
+            Write-Host  "##vso[task.logissue type=error;sourcepath=$($prAlert.physicalLocations[-1].filePath);linenumber=$($prAlert.physicalLocations[-1].region.lineStart);columnnumber=$($prAlert.physicalLocations[-1].region.columnStart)] New $alertType alert detected #$($prAlert.alertId) : $($prAlert.title)."
+            Write-Host  "##[error] Fix or dismiss this new alert in the Advanced Security UI for pr branch $($prSourceBranch)."
+            $urlAlert = "https://dev.azure.com/{0}/{1}/_git/{2}/alerts/{3}?branch={4}" -f $orgName, $project, $repositoryId, $prAlert.alertId, $prSourceBranch
+            Write-Host  "##[error] Details for this new alert:  $($urlAlert)"
+
+            AddPRComment $prAlert $urlAlert
+        }
+    }
+    Write-Host
+    Write-Host "##[error] Please review these Code Scanning alerts for the $($prBranch) branch using the regular Advanced Security UI" 
+    Write-Host "##[error] Dissmiss or fix the alerts listed and try re-queue the CIVerify task."
+    exit 1
+} else {
+    Write-Output "No new CodeQL alerts - all is fine"
+    exit 0
+}

CIVerify.yml

@@ -0,0 +1,50 @@
+# This is a starter pipeline to handle Gated PRs using GHAzDO. Currently (October 2023) this is not supported out of the box by the product, 
+# it probably will be supported in the future, this project can be used as a workaround until that time. 
+# We want to restrict new code going into main and only allow PRs if the new code does not introduce any new CodeQL issues. 
+# The idea is to set a branch protection policy (for main), forcing this pipeline to succeed before a PR into main can happen. 
+# The pipeline will run CodeQL on the source branch of the PR. Later, using a PowerShell script, the CodeQL issues of the PR source and target will be compared. 
+# If there are issues in the PR source that are not in main, this pipeline will fail. 
+#
+# If new alerts are detected these needs to be analysed using the regular Advanced Security Code Scanning UI. Set the filter to the pr branch and fix or dissmiss all the new issue. 
+# After that, the PR check CIVerify can be requeued in the PR. Hopefully this time, without any issues. 
+# 
+# The script needs a PAT to run (for accessing the REST API). This PAT should be setup as a secret variable for the pipleline (name: GATING_PAT). 
+# The PAT needs the access right - Advanced Security - Read
+#
+# More on ADO build verifications: https://learn.microsoft.com/en-us/azure/devops/repos/git/branch-policies?view=azure-devops&tabs=browser#build-validation 
+
+trigger:
+- none
+
+pool:
+  vmImage: ubuntu-latest
+
+
+steps:
+- task: AdvancedSecurity-Codeql-Init@1
+  inputs:
+    languages: "java"
+
+- task: Maven@4
+  inputs:
+    mavenPomFile: 'pom.xml'
+    publishJUnitResults: true
+    testResultsFiles: '**/TEST-*.xml'
+    javaHomeOption: 'JDKVersion'
+    jdkVersionOption: '1.17'
+    mavenVersionOption: 'Default'
+    mavenAuthenticateFeed: false
+    effectivePomSkip: false
+    sonarQubeRunAnalysis: false
+
+- task: AdvancedSecurity-Codeql-Analyze@1
+
+# Compair CodeQL issues on the PR source branch and main. 
+# Fail if there are new issues.  
+- task: PowerShell@2
+  displayName: 'CI Gating - verify there are no new CodeQL issues introduced in this PR'
+  inputs:
+    targetType: filePath
+    filePath: CIGate.ps1
+  env:
+    MAPPED_ADO_PAT: $(System.AccessToken)

Demoscript.md

@@ -0,0 +1,36 @@
+This repository has been setup for Microsoft personal to play with an demo GHAzDO. 
+
+## Resources ##
+ - [GHAzDO pitch](https://microsoft.sharepoint.com/teams/GithubSales/_layouts/15/search.aspx/siteall?q=GHAZDO%20Pitch%20Deck) deck on [ghsales](aka.ms/ghsales)
+ - [GHAzDO intro demo video](https://www.youtube.com/watch?v=cTkUhKkMD_c), covering why a tool like this is needed and how to set it up. This video is usually hidden behind a form to collect leads so do not spread it to far.
+ - [ADO roadmap](https://aka.ms/azdo-roadmap)
+ - [PR Gating Script](https://gh.io/GHAzDO_pr_gating). PR Gating is added to this repo via scripts and configurations. The product team plans to add this feature at a later state, the current setup is a temporary workaround. 
+ - [GHAzDO documentation](https://learn.microsoft.com/en-us/azure/devops/repos/security/configure-github-advanced-security-features).
+
+
+## Preparations ##
+
+Please review the GHAzDO pitch deck and the intro demo video. The pitch deck is useful if you need to explain the why of Advanced Security. The demo is a good next step after the customer understands the need. 
+Advanced customers might want to dig deeper into the GHAzDO security capabilities. For larger customers, you will get support from the GitHub account team in these discussions if you reach out. 
+
+Create a new ADO PAT that can be used for showing off push protection
+[Setup a new ADO PAT](https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows#create-a-pat). 
+Give it minimal access rights. Copy the key value. Revoke the new PAT right away.
+
+## Demo for the customer ##
+
+Show how GHAzDO is enabled on the repo in the ADO UI. This is a competitive advantage, most of our competitors have clunky setups and extra installs for this. You can also switch over to the security tab of the repo and who the GHAzDO security roles and how they are integrated with ADO.
+
+Talk about what happens when GHAzDO is enabled. How we will 'start a background process' to scan for forgotten secrets in the codebase. Leaked credentials is one of the main attack vectors and most of our competitors does not do it. 
+Show Push Protection. I usually setup a new file (config.txt) and add ADO_PAT = [the key from your pre prep]. This is obviously a simplification of a more realistic scenario but it usually gets the point across. Try to commit the new file and notice the push protection UI. Show how you can change just 1 char of the PAT and the push will not be blocked. This shows how we can keep our false positive rate low.
+
+Switch over to the GHAzDO report UI and show the secrets that was added to the project before push protection was enabled. If asked, show the secret providers we work with from the GHAzDO documentation.
+
+Talk about how secret scanning is the feature that can be enabled with just a click of a checkbox. For dependency scanning and code scanning, a pipelines needs to be extended. Show the GHAS pipeline and note the CodeQL and dependency scanning tasks. 
+
+Show in the GHAS pipeline build logs the result of a dependency scan. Show the number of direct and transient dependencies. We will check all of these libraries against the GitHub advisory database. Any libraries used with known vulnerabilities will be flagged and show in the GHAzDO UI. Show the dependency alerts. Dig into one vulnerability. Talk about the details and that this was a transient dependency, the UI shows the root dependency that brought it in.
+
+Switch to code scanning. Talk about how CodeQL is a two step process, of first building a Database describing the dataflow of the application and that we then run queries against this database. Show the results of the code scanning. Dig into one vulnerability and show the details. Jump into the code that is referenced, I usually pick a SQL injection. Show the code and explain the issue, mention that we know in this instance that the SQL query is user controlled, that is, the tool has detected that some parts of the SQL query comes unfiltered from user input. Again, since we can follow the data from source to sink we can keep our false positive rate low. 
+
+PR gating come up in most discussions. That is, the ability to block new code going into the main branch if it adds any new vulnerabilities. If this come up, show the PR that is there and blocked. Show how a branch policy has been set on the main branch. Show the result of the PR - how it has been annotated with information about the vulnerabilities. Jump into the details referenced in the PR annotation and show how the scan has happened on the PR branch. 
+

azure-pipelines.yml

@@ -0,0 +1,26 @@
+trigger:
+  - main
+
+pool:
+  vmImage: windows-latest
+
+steps:
+  - task: AdvancedSecurity-Codeql-Init@1
+    inputs:
+      languages: "java"
+
+  - task: Maven@4
+    inputs:
+      mavenPomFile: 'pom.xml'
+      publishJUnitResults: true
+      testResultsFiles: '**/TEST-*.xml'
+      javaHomeOption: 'JDKVersion'
+      jdkVersionOption: '1.17'
+      mavenVersionOption: 'Default'
+      mavenAuthenticateFeed: false
+      effectivePomSkip: false
+      sonarQubeRunAnalysis: false
+
+  - task: AdvancedSecurity-Codeql-Analyze@1
+
+  - task: AdvancedSecurity-Dependency-Scanning@1