From 470b27e8bae25863d55808ea41d32a0138b9d3a9 Mon Sep 17 00:00:00 2001 From: Nan Zang Date: Thu, 8 Dec 2022 17:49:19 -0800 Subject: [PATCH 1/8] remove workflow get from RI --- ...y-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.ps1 | 587 ------------------ ...y-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.yml | 81 --- 2 files changed, 668 deletions(-) delete mode 100644 .github/workflows/azure-sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.ps1 delete mode 100644 .github/workflows/sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.yml diff --git a/.github/workflows/azure-sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.ps1 b/.github/workflows/azure-sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.ps1 deleted file mode 100644 index 9269e304c..000000000 --- a/.github/workflows/azure-sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.ps1 +++ /dev/null @@ -1,587 +0,0 @@ -## Globals ## -$CloudEnv = $Env:cloudEnv -$ResourceGroupName = $Env:resourceGroupName -$WorkspaceName = $Env:workspaceName -$WorkspaceId = $Env:workspaceId -$Directory = $Env:directory -$Creds = $Env:creds -$contentTypes = $Env:contentTypes -$contentTypeMapping = @{ - "AnalyticsRule"=@("Microsoft.OperationalInsights/workspaces/providers/alertRules", "Microsoft.OperationalInsights/workspaces/providers/alertRules/actions"); - "AutomationRule"=@("Microsoft.OperationalInsights/workspaces/providers/automationRules"); - "HuntingQuery"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); - "Parser"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); - "Playbook"=@("Microsoft.Web/connections", "Microsoft.Logic/workflows", "Microsoft.Web/customApis"); - "Workbook"=@("Microsoft.Insights/workbooks"); -} -$sourceControlId = $Env:sourceControlId -$rootDirectory = $Env:rootDirectory -$githubAuthToken = $Env:githubAuthToken -$githubRepository = $Env:GITHUB_REPOSITORY -$branchName = $Env:branch -$smartDeployment = $Env:smartDeployment -$csvPath = "$rootDirectory\.sentinel\tracking_table_$sourceControlId.csv" -$configPath = "$rootDirectory\sentinel-deployment.config" -$global:localCsvTablefinal = @{} -$global:updatedCsvTable = @{} -$global:parameterFileMapping = @{} -$global:prioritizedContentFiles = @() -$global:excludeContentFiles = @() - -$guidPattern = '(\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)' -$namePattern = '([-\w\._\(\)]+)' -$sentinelResourcePatterns = @{ - "AnalyticsRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/alertRules/$namePattern" - "AutomationRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/automationRules/$namePattern" - "HuntingQuery" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" - "Parser" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" - "Playbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Logic/workflows/$namePattern" - "Workbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Insights/workbooks/$namePattern" -} - -if ([string]::IsNullOrEmpty($contentTypes)) { - $contentTypes = "AnalyticsRule" -} - -$metadataFilePath = "metadata.json" -@" -{ - "`$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "parentResourceId": { - "type": "string" - }, - "kind": { - "type": "string" - }, - "sourceControlId": { - "type": "string" - }, - "workspace": { - "type": "string" - }, - "contentId": { - "type": "string" - } - }, - "variables": { - "metadataName": "[concat(toLower(parameters('kind')), '-', parameters('contentId'))]" - }, - "resources": [ - { - "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", - "apiVersion": "2022-01-01-preview", - "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('metadataName'))]", - "properties": { - "parentId": "[parameters('parentResourceId')]", - "kind": "[parameters('kind')]", - "source": { - "kind": "SourceRepository", - "name": "Repositories", - "sourceId": "[parameters('sourceControlId')]" - } - } - } - ] -} -"@ | Out-File -FilePath $metadataFilePath - -$resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping[$_] } | ForEach-Object { $_.ToLower() } -$MaxRetries = 3 -$secondsBetweenAttempts = 5 - -#Converts hashtable to string that can be set as content when pushing csv file -function ConvertTableToString { - $output = "FileName, CommitSha`n" - $global:updatedCsvTable.GetEnumerator() | ForEach-Object { - $key = RelativePathWithBackslash $_.Key - $output += "{0},{1}`n" -f $key, $_.Value - } - return $output -} - -$header = @{ - "authorization" = "Bearer $githubAuthToken" -} - -#Gets all files and commit shas using Get Trees API -function GetGithubTree { - $branchResponse = AttemptInvokeRestMethod "Get" "https://api.github.com/repos/$githubRepository/branches/$branchName" $null $null 3 - $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" - $getTreeResponse = AttemptInvokeRestMethod "Get" $treeUrl $null $null 3 - return $getTreeResponse -} - -#Gets blob commit sha of the csv file, used when updating csv file to repo -function GetCsvCommitSha($getTreeResponse) { - $relativeCsvPath = RelativePathWithBackslash $csvPath - $shaObject = $getTreeResponse.tree | Where-Object { $_.path -eq $relativeCsvPath } - return $shaObject.sha -} - -#Creates a table using the reponse from the tree api, creates a table -function GetCommitShaTable($getTreeResponse) { - $shaTable = @{} - $getTreeResponse.tree | ForEach-Object { - $truePath = AbsolutePathWithSlash $_.path - if (([System.IO.Path]::GetExtension($_.path) -eq ".json") -or ($truePath -eq $configPath)) - { - $shaTable.Add($truePath, $_.sha) - } - } - return $shaTable -} - -#Pushes new/updated csv file to the user's repository. If updating file, will need csv commit sha. -function PushCsvToRepo($getTreeResponse) { - $relativeCsvPath = RelativePathWithBackslash $csvPath - $sha = GetCsvCommitSha $getTreeResponse - $createFileUrl = "https://api.github.com/repos/$githubRepository/contents/$relativeCsvPath" - $content = ConvertTableToString - $encodedContent = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($content)) - - $body = @{ - message = "trackingTable.csv created." - content = $encodedContent - branch = $branchName - sha = $sha - } | ConvertTo-Json - - $Parameters = @{ - Method = "PUT" - Uri = $createFileUrl - Headers = $header - Body = $body | ConvertTo-Json - } - AttemptInvokeRestMethod "Put" $createFileUrl $body $null 3 -} - -function ReadCsvToTable { - $csvTable = Import-Csv -Path $csvPath - $HashTable=@{} - foreach($r in $csvTable) - { - $key = AbsolutePathWithSlash $r.FileName - $HashTable[$key]=$r.CommitSha - } - return $HashTable -} - -function AttemptInvokeRestMethod($method, $url, $body, $contentTypes, $maxRetries) { - $Stoploop = $false - $retryCount = 0 - do { - try { - $result = Invoke-RestMethod -Uri $url -Method $method -Headers $header -Body $body -ContentType $contentTypes - $Stoploop = $true - } - catch { - if ($retryCount -gt $maxRetries) { - Write-Host "[Error] API call failed after $retryCount retries: $_" - $Stoploop = $true - } - else { - Write-Host "[Warning] API call failed: $_.`n Conducting retry #$retryCount." - Start-Sleep -Seconds 5 - $retryCount = $retryCount + 1 - } - } - } - While ($Stoploop -eq $false) - return $result -} - -function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { - $maxLoginRetries = 3 - $delayInSeconds = 30 - $retryCount = 1 - $stopTrying = $false - do { - try { - Connect-AzAccount -ServicePrincipal -Tenant $tenantId -Credential $psCredential -Environment $cloudEnv | out-null; - Write-Host "Login Successful" - $stopTrying = $true - } - catch { - if ($retryCount -ge $maxLoginRetries) { - Write-Host "Login failed after $maxLoginRetries attempts." - $stopTrying = $true - } - else { - Write-Host "Login attempt failed, retrying in $delayInSeconds seconds." - Start-Sleep -Seconds $delayInSeconds - $retryCount++ - } - } - } - while (-not $stopTrying) -} - -function ConnectAzCloud { - $RawCreds = $Creds | ConvertFrom-Json - - Clear-AzContext -Scope Process; - Clear-AzContext -Scope CurrentUser -Force -ErrorAction SilentlyContinue; - - Add-AzEnvironment ` - -Name $CloudEnv ` - -ActiveDirectoryEndpoint $RawCreds.activeDirectoryEndpointUrl ` - -ResourceManagerEndpoint $RawCreds.resourceManagerEndpointUrl ` - -ActiveDirectoryServiceEndpointResourceId $RawCreds.activeDirectoryServiceEndpointResourceId ` - -GraphEndpoint $RawCreds.graphEndpointUrl | out-null; - - $servicePrincipalKey = ConvertTo-SecureString $RawCreds.clientSecret.replace("'", "''") -AsPlainText -Force - $psCredential = New-Object System.Management.Automation.PSCredential($RawCreds.clientId, $servicePrincipalKey) - - AttemptAzLogin $psCredential $RawCreds.tenantId $CloudEnv - Set-AzContext -Tenant $RawCreds.tenantId | out-null; -} - -function AttemptDeployMetadata($deploymentName, $resourceGroupName, $templateObject) { - $deploymentInfo = $null - try { - $deploymentInfo = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Ignore - } - catch { - Write-Host "[Warning] Unable to fetch deployment info for $deploymentName, no metadata was created for the resources in the file. Error: $_" - return - } - $deploymentInfo | Where-Object { $_.TargetResource -ne "" } | ForEach-Object { - $resource = $_.TargetResource - $sentinelContentKinds = GetContentKinds $resource - if ($sentinelContentKinds.Count -gt 0) { - $contentKind = ToContentKind $sentinelContentKinds $resource $templateObject - $contentId = $resource.Split("/")[-1] - try { - New-AzResourceGroupDeployment -Name "md-$deploymentName" -ResourceGroupName $ResourceGroupName -TemplateFile $metadataFilePath ` - -parentResourceId $resource ` - -kind $contentKind ` - -contentId $contentId ` - -sourceControlId $sourceControlId ` - -workspace $workspaceName ` - -ErrorAction Stop | Out-Host - Write-Host "[Info] Created metadata metadata for $contentKind with parent resource id $resource" - } - catch { - Write-Host "[Warning] Failed to deploy metadata for $contentKind with parent resource id $resource with error $_" - } - } - } -} - -function GetContentKinds($resource) { - return $sentinelResourcePatterns.Keys | Where-Object { $resource -match $sentinelResourcePatterns[$_] } -} - -function ToContentKind($contentKinds, $resource, $templateObject) { - if ($contentKinds.Count -eq 1) { - return $contentKinds - } - if ($null -ne $resource -and $resource.Contains('savedSearches')) { - if ($templateObject.resources.properties.Category -eq "Hunting Queries") { - return "HuntingQuery" - } - return "Parser" - } - return $null -} - -function IsValidTemplate($path, $templateObject) { - Try { - if (DoesContainWorkspaceParam $templateObject) { - Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $WorkspaceName - } - else { - Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path - } - - return $true - } - Catch { - Write-Host "[Warning] The file $path is not valid: $_" - return $false - } -} - -function IsRetryable($deploymentName) { - $retryableStatusCodes = "Conflict","TooManyRequests","InternalServerError","DeploymentActive" - Try { - $deploymentResult = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Stop - return $retryableStatusCodes -contains $deploymentResult.StatusCode - } - Catch { - return $false - } -} - -function IsValidResourceType($template) { - try { - $isAllowedResources = $true - $template.resources | ForEach-Object { - $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources - } - } - catch { - Write-Host "[Error] Failed to check valid resource type." - $isAllowedResources = $false - } - return $isAllowedResources -} - -function DoesContainWorkspaceParam($templateObject) { - $templateObject.parameters.PSobject.Properties.Name -contains "workspace" -} - -function AttemptDeployment($path, $parameterFile, $deploymentName, $templateObject) { - Write-Host "[Info] Deploying $path with deployment name $deploymentName" - - $isValid = IsValidTemplate $path $templateObject - if (-not $isValid) { - return $false - } - $isSuccess = $false - $currentAttempt = 0 - While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) - { - $currentAttempt ++ - Try - { - Write-Host "[Info] Deploy $path with parameter file: [$parameterFile]" - if (DoesContainWorkspaceParam $templateObject) - { - if ($parameterFile) { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host - } - else - { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -ErrorAction Stop | Out-Host - } - } - else - { - if ($parameterFile) { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host - } - else - { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -ErrorAction Stop | Out-Host - } - } - AttemptDeployMetadata $deploymentName $ResourceGroupName $templateObject - - $isSuccess = $true - } - Catch [Exception] - { - $err = $_ - if (-not (IsRetryable $deploymentName)) - { - Write-Host "[Warning] Failed to deploy $path with error: $err" - break - } - else - { - if ($currentAttempt -le $MaxRetries) - { - Write-Host "[Warning] Failed to deploy $path with error: $err. Retrying in $secondsBetweenAttempts seconds..." - Start-Sleep -Seconds $secondsBetweenAttempts - } - else - { - Write-Host "[Warning] Failed to deploy $path after $currentAttempt attempts with error: $err" - } - } - } - } - return $isSuccess -} - -function GenerateDeploymentName() { - $randomId = [guid]::NewGuid() - return "Sentinel_Deployment_$randomId" -} - -#Load deployment configuration -function LoadDeploymentConfig() { - Write-Host "[Info] load the deployment configuration from [$configPath]" - $global:parameterFileMapping = @{} - $global:prioritizedContentFiles = @() - $global:excludeContentFiles = @() - try { - if (Test-Path $configPath) { - $deployment_config = Get-Content $configPath | Out-String | ConvertFrom-Json - $parameterFileMappings = @{} - if ($deployment_config.parameterfilemappings) { - $deployment_config.parameterfilemappings.psobject.properties | ForEach { $parameterFileMappings[$_.Name] = $_.Value } - } - $key = ($parameterFileMappings.Keys | ? { $_ -eq $workspaceId }) - if ($null -ne $key) { - $parameterFileMappings[$key].psobject.properties | ForEach { $global:parameterFileMapping[$_.Name] = $_.Value } - } - if ($deployment_config.prioritizedcontentfiles) { - $global:prioritizedContentFiles = $deployment_config.prioritizedcontentfiles - } - $excludeList = $global:parameterFileMapping.Values + $global:prioritizedcontentfiles - if ($deployment_config.excludecontentfiles) { - $excludeList = $excludeList + $deployment_config.excludecontentfiles - } - $global:excludeContentFiles = $excludeList | Where-Object { Test-Path (AbsolutePathWithSlash $_) } - } - } - catch { - Write-Host "[Warning] An error occurred while trying to load deployment configuration." - Write-Host "Exception details: $_" - Write-Host $_.ScriptStackTrace - } -} - -function filterContentFile($fullPath) { - $temp = RelativePathWithBackslash $fullPath - return $global:excludeContentFiles | ? {$temp.StartsWith($_, 'CurrentCultureIgnoreCase')} -} - -function RelativePathWithBackslash($absolutePath) { - return $absolutePath.Replace($rootDirectory + "\", "").Replace("\", "/") -} - -function AbsolutePathWithSlash($relativePath) { - return Join-Path -Path $rootDirectory -ChildPath $relativePath -} - -#resolve parameter file name, return $null if there is none. -function GetParameterFile($path) { - $index = RelativePathWithBackslash $path - $key = ($global:parameterFileMapping.Keys | ? { $_ -eq $index }) - if ($key) { - $mappedParameterFile = AbsolutePathWithSlash $global:parameterFileMapping[$key] - if (Test-Path $mappedParameterFile) { - return $mappedParameterFile - } - } - - $parameterFilePrefix = $path.TrimEnd(".json") - - $workspaceParameterFile = $parameterFilePrefix + ".parameters-$WorkspaceId.json" - if (Test-Path $workspaceParameterFile) { - return $workspaceParameterFile - } - - $defaultParameterFile = $parameterFilePrefix + ".parameters.json" - if (Test-Path $defaultParameterFile) { - return $defaultParameterFile - } - - return $null -} - -function Deployment($fullDeploymentFlag, $remoteShaTable, $tree) { - Write-Host "Starting Deployment for Files in path: $Directory" - if (Test-Path -Path $Directory) - { - $totalFiles = 0; - $totalFailed = 0; - $iterationList = @() - $global:prioritizedContentFiles | ForEach-Object { $iterationList += (AbsolutePathWithSlash $_) } - Get-ChildItem -Path $Directory -Recurse -Filter *.json -exclude *metadata.json, *.parameters*.json | - Where-Object { $null -eq ( filterContentFile $_.FullName ) } | - Select-Object -Property FullName | - ForEach-Object { $iterationList += $_.FullName } - $iterationList | ForEach-Object { - $path = $_ - Write-Host "[Info] Try to deploy $path" - if (-not (Test-Path $path)) { - Write-Host "[Warning] Skipping deployment for $path. The file doesn't exist." - return - } - $templateObject = Get-Content $path | Out-String | ConvertFrom-Json - if (-not (IsValidResourceType $templateObject)) - { - Write-Host "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." - return - } - $parameterFile = GetParameterFile $path - $result = SmartDeployment $fullDeploymentFlag $remoteShaTable $path $parameterFile $templateObject - if ($result.isSuccess -eq $false) { - $totalFailed++ - } - if (-not $result.skip) { - $totalFiles++ - } - if ($result.isSuccess -or $result.skip) { - $global:updatedCsvTable[$path] = $remoteShaTable[$path] - if ($parameterFile) { - $global:updatedCsvTable[$parameterFile] = $remoteShaTable[$parameterFile] - } - } - } - PushCsvToRepo $tree - if ($totalFiles -gt 0 -and $totalFailed -gt 0) - { - $err = "$totalFailed of $totalFiles deployments failed." - Throw $err - } - } - else - { - Write-Output "[Warning] $Directory not found. nothing to deploy" - } -} - -function SmartDeployment($fullDeploymentFlag, $remoteShaTable, $path, $parameterFile, $templateObject) { - try { - $skip = $false - $isSuccess = $null - if (!$fullDeploymentFlag) { - $existingSha = $global:localCsvTablefinal[$path] - $remoteSha = $remoteShaTable[$path] - $skip = (($existingSha) -and ($existingSha -eq $remoteSha)) - if ($skip -and $parameterFile) { - $existingShaForParameterFile = $global:localCsvTablefinal[$parameterFile] - $remoteShaForParameterFile = $remoteShaTable[$parameterFile] - $skip = (($existingShaForParameterFile) -and ($existingShaForParameterFile -eq $remoteShaForParameterFile)) - } - } - if (!$skip) { - $deploymentName = GenerateDeploymentName - $isSuccess = AttemptDeployment $path $parameterFile $deploymentName $templateObject - } - return @{ - skip = $skip - isSuccess = $isSuccess - } - } - catch { - Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" - Write-Host $_.ScriptStackTrace - } -} - -function main() { - if ($CloudEnv -ne 'AzureCloud') - { - Write-Output "Attempting Sign In to Azure Cloud" - ConnectAzCloud - } - - if (Test-Path $csvPath) { - $global:localCsvTablefinal = ReadCsvToTable - } - - LoadDeploymentConfig - - $tree = GetGithubTree - $remoteShaTable = GetCommitShaTable $tree - - $existingConfigSha = $global:localCsvTablefinal[$configPath] - $remoteConfigSha = $remoteShaTable[$configPath] - $modifiedConfig = ($existingConfigSha -xor $remoteConfigSha) -or ($existingConfigSha -and $remoteConfigSha -and ($existingConfigSha -ne $remoteConfigSha)) - if ($remoteConfigSha) { - $global:updatedCsvTable[$configPath] = $remoteConfigSha - } - - $fullDeploymentFlag = $modifiedConfig -or (-not (Test-Path $csvPath)) -or ($smartDeployment -eq "false") - Deployment $fullDeploymentFlag $remoteShaTable $tree -} - -main \ No newline at end of file diff --git a/.github/workflows/sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.yml b/.github/workflows/sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.yml deleted file mode 100644 index 563a79d99..000000000 --- a/.github/workflows/sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Deploy Content to sentinel-eco-test-nazang [bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6] -# Note: This workflow will deploy everything in the root directory. -# To deploy content only from a specific path (for example SentinelContent): -# 1. Add the target path to the "paths" property like such -# paths: -# - 'SentinelContent/**' -# - '!.github/workflows/**' -# - '.github/workflows/sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.yml' -# 2. Append the path to the directory environment variable below -# directory: '${{ github.workspace }}/SentinelContent' - -on: - push: - branches: [ TestWorkspaceMappingBranch ] - paths: - - '**' - - '!.github/workflows/**' # this filter prevents other workflow changes from triggering this workflow - - '.github/workflows/sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.yml' - -jobs: - deploy-content: - runs-on: windows-latest - env: - resourceGroupName: 'sentinel-eco-test-nazang' - workspaceName: 'sentinel-eco-test-nazang' - workspaceId: '40d1b346-fe1b-4503-8f8e-cfbbf4d7f4b3' - directory: '${{ github.workspace }}' - cloudEnv: 'AzureCloud' - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_bdbbbbc27a6145f0bd306969c73eb7b6 }} - contentTypes: 'Workbook' - branch: 'TestWorkspaceMappingBranch' - sourceControlId: 'bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6' - rootDirectory: '${{ github.workspace }}' - githubAuthToken: ${{ secrets.GITHUB_TOKEN }} - smartDeployment: 'true' - - steps: - - name: Login to Azure (Attempt 1) - continue-on-error: true - id: login1 - uses: azure/login@v1 - if: ${{ env.cloudEnv == 'AzureCloud' }} - with: - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_bdbbbbc27a6145f0bd306969c73eb7b6 }} - enable-AzPSSession: true - - - name: Wait 30 seconds if login attempt 1 failed - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login1.outcome=='failure' }} - run: powershell Start-Sleep -s 30 - - - name: Login to Azure (Attempt 2) - continue-on-error: true - id: login2 - uses: azure/login@v1 - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login1.outcome=='failure' }} - with: - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_bdbbbbc27a6145f0bd306969c73eb7b6 }} - enable-AzPSSession: true - - - name: Wait 30 seconds if login attempt 2 failed - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login2.outcome=='failure' }} - run: powershell Start-Sleep -s 30 - - - name: Login to Azure (Attempt 3) - continue-on-error: false - id: login3 - uses: azure/login@v1 - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login2.outcome=='failure' }} - with: - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_bdbbbbc27a6145f0bd306969c73eb7b6 }} - enable-AzPSSession: true - - - name: Checkout - uses: actions/checkout@v1 - - - name: Deploy Content to Azure Sentinel - uses: azure/powershell@v1 - with: - azPSVersion: 'latest' - inlineScript: | - ${{ github.workspace }}//.github/workflows/azure-sentinel-deploy-bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.ps1 \ No newline at end of file From cd8033253ba3043958f447b438462d782a264c5f Mon Sep 17 00:00:00 2001 From: Nan Zang Date: Fri, 9 Dec 2022 13:20:13 -0800 Subject: [PATCH 2/8] prepare prod workspace --- .../tracking_table_bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.csv | 2 -- ...on => workspace-sentinel-eco-test-nazang2-parameters.json} | 2 +- sentinel-deployment.config | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 .sentinel/tracking_table_bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.csv rename Parameters/{workspace-sentinel-eco-test-nazang-parameters.json => workspace-sentinel-eco-test-nazang2-parameters.json} (89%) diff --git a/.sentinel/tracking_table_bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.csv b/.sentinel/tracking_table_bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.csv deleted file mode 100644 index be7df43a3..000000000 --- a/.sentinel/tracking_table_bdbbbbc2-7a61-45f0-bd30-6969c73eb7b6.csv +++ /dev/null @@ -1,2 +0,0 @@ -FileName, CommitSha -sentinel-deployment.config,b1bc6767afa169977d57de61e5e8be3d8de7f51e diff --git a/Parameters/workspace-sentinel-eco-test-nazang-parameters.json b/Parameters/workspace-sentinel-eco-test-nazang2-parameters.json similarity index 89% rename from Parameters/workspace-sentinel-eco-test-nazang-parameters.json rename to Parameters/workspace-sentinel-eco-test-nazang2-parameters.json index 8138974b5..b0cffd1ce 100644 --- a/Parameters/workspace-sentinel-eco-test-nazang-parameters.json +++ b/Parameters/workspace-sentinel-eco-test-nazang2-parameters.json @@ -9,7 +9,7 @@ "value": "sentinel-eco-test-nazang" }, "workspaceName": { - "value": "sentinel-eco-test-nazang" + "value": "sentinel-eco-test-nazang2" } } } \ No newline at end of file diff --git a/sentinel-deployment.config b/sentinel-deployment.config index b1bc6767a..92fc1d25d 100644 --- a/sentinel-deployment.config +++ b/sentinel-deployment.config @@ -3,8 +3,8 @@ "Parameters" ], "parameterfilemappings": { - "40d1b346-fe1b-4503-8f8e-cfbbf4d7f4b3": { - "Workbooks/workbook_ADXvsLA.json": "Parameters/workspace-sentinel-eco-test-nazang-parameters.json" + "a2c3834b-49dd-410b-87aa-99f878346c91": { + "Workbooks/workbook_ADXvsLA.json": "Parameters/workspace-sentinel-eco-test-nazang2-parameters.json" } }, "DummySection": "This shouldn't impact deployment" From 2e746d5748e0252156ae814cc9bcc88a024e09ed Mon Sep 17 00:00:00 2001 From: Nan Zang Date: Fri, 9 Dec 2022 13:36:36 -0800 Subject: [PATCH 3/8] merge test into prod branch --- ...space-sentinel-eco-test-nazang-parameters.json | 15 +++++++++++++++ sentinel-deployment.config | 3 +++ 2 files changed, 18 insertions(+) create mode 100644 Parameters/workspace-sentinel-eco-test-nazang-parameters.json diff --git a/Parameters/workspace-sentinel-eco-test-nazang-parameters.json b/Parameters/workspace-sentinel-eco-test-nazang-parameters.json new file mode 100644 index 000000000..8138974b5 --- /dev/null +++ b/Parameters/workspace-sentinel-eco-test-nazang-parameters.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "subscriptionId": { + "value": "419581d6-4853-49bd-83b6-d94bb8a77887" + }, + "resourcegroupName": { + "value": "sentinel-eco-test-nazang" + }, + "workspaceName": { + "value": "sentinel-eco-test-nazang" + } + } +} \ No newline at end of file diff --git a/sentinel-deployment.config b/sentinel-deployment.config index 92fc1d25d..88f3cb402 100644 --- a/sentinel-deployment.config +++ b/sentinel-deployment.config @@ -3,6 +3,9 @@ "Parameters" ], "parameterfilemappings": { + "40d1b346-fe1b-4503-8f8e-cfbbf4d7f4b3": { + "Workbooks/workbook_ADXvsLA.json": "Parameters/workspace-sentinel-eco-test-nazang-parameters.json" + }, "a2c3834b-49dd-410b-87aa-99f878346c91": { "Workbooks/workbook_ADXvsLA.json": "Parameters/workspace-sentinel-eco-test-nazang2-parameters.json" } From ec634fa0e5d7ad2e256cfab44544c2d15e0b5276 Mon Sep 17 00:00:00 2001 From: Nan Zang Date: Fri, 9 Dec 2022 15:26:26 -0800 Subject: [PATCH 4/8] add AMA workbook into the content hub repo --- ...e-sentinel-eco-test-nazang-parameters.json | 6 ++ ...-sentinel-eco-test-nazang2-parameters.json | 6 ++ Workbooks/workbook_ADXvsLA.json | 2 +- Workbooks/workbook_AMA_migration_tracker.json | 69 +++++++++++++++++++ sentinel-deployment.config | 6 +- 5 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 Workbooks/workbook_AMA_migration_tracker.json diff --git a/Parameters/workspace-sentinel-eco-test-nazang-parameters.json b/Parameters/workspace-sentinel-eco-test-nazang-parameters.json index 8138974b5..e0fb57ea1 100644 --- a/Parameters/workspace-sentinel-eco-test-nazang-parameters.json +++ b/Parameters/workspace-sentinel-eco-test-nazang-parameters.json @@ -10,6 +10,12 @@ }, "workspaceName": { "value": "sentinel-eco-test-nazang" + }, + "ADXvsLA-WorkbookId": { + "value": "8dd3bb4d-62fd-4b40-939c-64edf3d2fdff" + }, + "AMAMigrationTracker-WorkbookId": { + "value": "56dbbdde-991b-4fe1-ad01-5c72f84d9200" } } } \ No newline at end of file diff --git a/Parameters/workspace-sentinel-eco-test-nazang2-parameters.json b/Parameters/workspace-sentinel-eco-test-nazang2-parameters.json index b0cffd1ce..587e4ef64 100644 --- a/Parameters/workspace-sentinel-eco-test-nazang2-parameters.json +++ b/Parameters/workspace-sentinel-eco-test-nazang2-parameters.json @@ -10,6 +10,12 @@ }, "workspaceName": { "value": "sentinel-eco-test-nazang2" + }, + "ADXvsLA-WorkbookId": { + "value": "9af7013f-5c80-4c3e-8b0a-ad69a8c35063" + }, + "AMAMigrationTracker-WorkbookId": { + "value": "43e29093-8cdd-4568-b2a0-503a7737ffbc" } } } \ No newline at end of file diff --git a/Workbooks/workbook_ADXvsLA.json b/Workbooks/workbook_ADXvsLA.json index 32cc27274..5cd3eeabb 100644 --- a/Workbooks/workbook_ADXvsLA.json +++ b/Workbooks/workbook_ADXvsLA.json @@ -34,7 +34,7 @@ "description": "The id of resource instance to which the workbook will be associate." } }, - "workbookId": { + "ADXvsLA-WorkbookId": { "type": "string", "defaultValue": "[newGuid()]", "metadata": { diff --git a/Workbooks/workbook_AMA_migration_tracker.json b/Workbooks/workbook_AMA_migration_tracker.json new file mode 100644 index 000000000..2f55e8340 --- /dev/null +++ b/Workbooks/workbook_AMA_migration_tracker.json @@ -0,0 +1,69 @@ +{ + "contentVersion": "1.0.0.0", + "parameters": { + "subscriptionId": { + "type": "string", + "defaultValue": "" + }, + "resourceGroupName": { + "type": "string", + "defaultValue": "" + }, + "workspaceName": { + "type": "string", + "defaultValue": "" + }, + "workbookDisplayName": { + "type": "string", + "defaultValue": "[concat('AMA migration tracker - ', parameters('workspaceName'))]", + "metadata": { + "description": "The friendly name for the workbook that is used in the Gallery or Saved List. This name must be unique within a resource group." + } + }, + "workbookType": { + "type": "string", + "defaultValue": "sentinel", + "metadata": { + "description": "The gallery that the workbook will been shown under. Supported values include workbook, tsg, etc. Usually, this is 'workbook'." + } + }, + "workbookSourceId": { + "type": "string", + "defaultValue": "[concat('/subscriptions/', parameters('subscriptionId'), '/resourcegroups/', parameters('resourcegroupName'), '/providers/microsoft.operationalinsights/workspaces/', parameters('workspaceName'))]", + "metadata": { + "description": "The id of resource instance to which the workbook will be associate." + } + }, + "AMAMigrationTracker-WorkbookId": { + "type": "string", + "defaultValue": "[newGuid()]", + "metadata": { + "description": "The unique guid for this workbook instance." + } + } + }, + "resources": [ + { + "name": "[parameters('workbookId')]", + "type": "microsoft.insights/workbooks", + "location": "[resourceGroup().location]", + "apiVersion": "2021-03-08", + "dependsOn": [], + "kind": "shared", + "properties": { + "displayName": "[parameters('workbookDisplayName')]", + "serializedData": "{\"version\":\"Notebook/1.0\",\"items\":[{\"type\":11,\"content\":{\"version\":\"LinkItem/1.0\",\"style\":\"tabs\",\"links\":[{\"id\":\"4df9243a-749d-4698-98f6-188e0b687e13\",\"cellValue\":\"selectedTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Servers and extensions installed\",\"subTarget\":\"ServersExtensions\",\"style\":\"link\"},{\"id\":\"4c0faa80-5c85-4d02-989d-37921b12ae87\",\"cellValue\":\"selectedTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Data Collection Rules under Subscription\",\"subTarget\":\"DCRs\",\"style\":\"link\"},{\"id\":\"ffceb6e6-3756-466e-860b-c017f0421e9f\",\"cellValue\":\"selectedTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"VMs with AMA\",\"subTarget\":\"AMAvms\",\"style\":\"link\"}]},\"name\":\"links - 19\"},{\"type\":9,\"content\":{\"version\":\"KqlParameterItem/1.0\",\"parameters\":[{\"id\":\"cd3be59c-92d1-4c08-945e-bd9420459a0a\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"Subscription\",\"label\":\"Subscriptions\",\"type\":6,\"multiSelect\":true,\"quote\":\"'\",\"delimiter\":\",\",\"typeSettings\":{\"includeAll\":true,\"showDefault\":false},\"timeContext\":{\"durationMs\":86400000}}],\"style\":\"pills\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"ServersExtensions\"},\"name\":\"parameters - 21 - only one\"},{\"type\":9,\"content\":{\"version\":\"KqlParameterItem/1.0\",\"parameters\":[{\"id\":\"bcd3ee68-9319-4de6-8904-eae492693e64\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"Subscription2\",\"label\":\"Subscription\",\"type\":6,\"isRequired\":true,\"typeSettings\":{\"includeAll\":false},\"timeContext\":{\"durationMs\":86400000},\"value\":null}],\"style\":\"pills\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isNotEqualTo\",\"value\":\"ServersExtensions\"},\"name\":\"parameters - 21 \"},{\"type\":9,\"content\":{\"version\":\"KqlParameterItem/1.0\",\"parameters\":[{\"id\":\"521cf079-688f-4f0c-b446-64a353ac8aa1\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"InternalWSs\",\"type\":1,\"query\":\"SecurityIncident\\r\\n| take 1\\r\\n| parse IncidentUrl with * \\\"/workspaces/\\\" Workspace \\\"/\\\" *\\r\\n| project Workspace\",\"isHiddenWhenLocked\":true,\"timeContext\":{\"durationMs\":86400000},\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"},{\"id\":\"eaa69221-591b-448c-b18b-5828487030ab\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"Workspace\",\"type\":5,\"isRequired\":true,\"multiSelect\":true,\"quote\":\"'\",\"delimiter\":\",\",\"query\":\"where type =~ \\\"microsoft.operationalinsights/workspaces\\\"\\r\\n| project value =id, label = name, selected = iff(name =~ '{InternalWSs}', true, false)\",\"crossComponentResources\":[\"value::all\"],\"value\":[\"value::all\"],\"typeSettings\":{\"additionalResourceOptions\":[\"value::all\"],\"showDefault\":false},\"timeContext\":{\"durationMs\":86400000},\"defaultValue\":\"value::all\",\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\"},{\"id\":\"8f323b26-e505-40f3-8ea8-4d0fd69dcc9b\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"InternalRG\",\"type\":1,\"query\":\"where type =~ \\\"microsoft.operationalinsights/workspaces\\\"\\r\\n| where id =~ \\\"{Workspace}\\\"\\r\\n| project resourceGroup\",\"isHiddenWhenLocked\":true,\"timeContext\":{\"durationMs\":86400000},\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\"},{\"id\":\"2209d79a-3a7d-4840-a45e-579b98c35cfc\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"resourceGroup\",\"label\":\"Resource group\",\"type\":2,\"multiSelect\":true,\"quote\":\"'\",\"delimiter\":\",\",\"query\":\"Resources\\r\\n| summarize Count = count() by subscriptionId, resourceGroup\\r\\n| order by Count desc\\r\\n| extend Rank = row_number()\\r\\n| project value = strcat('/subscriptions/', subscriptionId, '/resourceGroups/', resourceGroup), label = resourceGroup, selected = false\",\"crossComponentResources\":[\"{Subscription}\"],\"typeSettings\":{\"showDefault\":false},\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\"},{\"id\":\"4335a281-83dd-4a49-91e8-4f93a4241ca2\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"Help\",\"label\":\"Show Help\",\"type\":10,\"description\":\"This will show some help information to help you understand the page you are on\",\"isRequired\":true,\"value\":\"Yes\",\"jsonData\":\"[\\r\\n { \\\"value\\\": \\\"Yes\\\", \\\"label\\\": \\\"Yes\\\"},\\r\\n { \\\"value\\\": \\\"No\\\", \\\"label\\\": \\\"No\\\", \\\"selected\\\":true }\\r\\n]\"}],\"style\":\"pills\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"},\"name\":\"parameters - 1 - Copy\"},{\"type\":1,\"content\":{\"json\":\"Use this section to see all the servers (either from Azure, other clouds, on-prem and Arc enabled (prerequisite to install the Azure Monitoring agent) under the selected subscription.\\r\\nUse the Subscription and Resource group filters to narrow down your results.\",\"style\":\"info\"},\"conditionalVisibilities\":[{\"parameterName\":\"Help\",\"comparison\":\"isEqualTo\",\"value\":\"Yes\"},{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"ServersExtensions\"}],\"name\":\"text - 18\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources\\r\\n| where \\\"{Subscription:Id}\\\" has subscriptionId or \\\"{Subscription}\\\" == \\\"\\\"\\r\\n| where \\\"{resourceGroup:label}\\\" contains resourceGroup or \\\"{resourceGroup}\\\" == \\\"\\\"\\r\\n| where type == \\\"microsoft.compute/virtualmachines\\\"\\r\\n| summarize TotalCountofVMs = count()\",\"size\":4,\"title\":\"Current VMs in {Subscription:label}\",\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{Subscription}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"leftContent\":{\"columnMatch\":\"TotalCountofVMs\",\"formatter\":3,\"formatOptions\":{\"palette\":\"blue\"}},\"showBorder\":false}},\"customWidth\":\"20\",\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"ServersExtensions\"},\"name\":\"query - 0\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"Heartbeat\\r\\n| where isempty(_ResourceId) == true or ResourceProvider =~ \\\"microsoft.hybridcompute\\\"\\r\\n| distinct Computer\\r\\n| summarize count()\",\"size\":4,\"title\":\"Arc or non-Azure with Heartbeat\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{Workspace}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"leftContent\":{\"columnMatch\":\"count_\",\"formatter\":3,\"formatOptions\":{\"palette\":\"blue\"}},\"showBorder\":false}},\"customWidth\":\"20\",\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"ServersExtensions\"},\"name\":\"query - 5\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources\\r\\n| where \\\"{Subscription:Id}\\\" has subscriptionId or \\\"{Subscription}\\\" == \\\"\\\"\\r\\n| where \\\"{resourceGroup:label}\\\" contains resourceGroup or \\\"{resourceGroup}\\\" == \\\"\\\"\\r\\n| where type has 'microsoft.compute/virtualmachines/extensions' or type has 'microsoft.hybridcompute/machines/extensions'\\r\\n| where name has 'MicrosoftMonitoringAgent' or name has 'OmsAgentForLinux' \\r\\n| extend Computer = extract('virtualMachines/(.*)/extensions',1,id) \\r\\n| summarize count()\",\"size\":4,\"title\":\"Machines with MMA\",\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{Subscription}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"leftContent\":{\"columnMatch\":\"count_\",\"formatter\":3,\"formatOptions\":{\"palette\":\"blue\"}},\"showBorder\":false}},\"customWidth\":\"20\",\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"ServersExtensions\"},\"name\":\"query - 6\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources \\r\\n| where \\\"{Subscription:Id}\\\" has subscriptionId or \\\"{Subscription}\\\" == \\\"\\\"\\r\\n| where type has 'microsoft.compute/virtualmachines/extensions' or type has 'microsoft.hybridcompute/machines/extensions'\\r\\n| where name has 'AzureMonitorWindowsAgent' or name has 'AzureMonitorLinuxAgent' \\r\\n| extend AzureVM = extract('virtualMachines/(.*)/extensions',1,id), ArcVM = extract('machines/(.*)/extensions',1,id) \\r\\n| summarize count() by AzureVM, ArcVM, subscriptionId, resourceGroup \\r\\n| project AzureVM, ArcVM, resourceGroup\\r\\n| summarize count()\",\"size\":4,\"title\":\"Machines with AMA\",\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{Subscription}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"leftContent\":{\"columnMatch\":\"count_\",\"formatter\":3,\"formatOptions\":{\"palette\":\"blue\"}},\"showBorder\":false}},\"customWidth\":\"20\",\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"ServersExtensions\"},\"name\":\"query - 4\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources\\r\\n| where \\\"{Subscription:Id}\\\" has subscriptionId or \\\"{Subscription}\\\" == \\\"\\\"\\r\\n| where type has 'microsoft.compute/virtualmachines/extensions' or type has 'microsoft.hybridcompute/machines/extensions'\\r\\n| where name has 'MicrosoftMonitoringAgent' or name has 'OmsAgentForLinux' or name has 'AzureMonitorWindowsAgent' or name has 'AzureMonitorLinuxAgent'\\r\\n| extend AzureVM = extract('virtualMachines/(.*)/extensions',1,id), ArcVM = extract('machines/(.*)/extensions',1,id)\\r\\n| summarize count() by AzureVM=tolower(AzureVM), ArcVM=tolower(ArcVM), subscriptionId, resourceGroup \\r\\n| extend hasBoth = iff(count_ > 1, 'Yes', 'No') | where count_ > 1 \\r\\n| join (resources | where type has 'microsoft.compute/virtualmachines/extensions' or type has 'microsoft.hybridcompute/machines/extensions'\\r\\n| where name has 'MicrosoftMonitoringAgent' or name has 'OmsAgentForLinux' \\r\\n| extend AzureVM = extract('virtualMachines/(.*)/extensions',1,id)) on AzureVM \\r\\n| summarize count()\",\"size\":4,\"title\":\"Machines with Both Agents\",\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{Subscription}\"],\"visualization\":\"tiles\",\"tileSettings\":{\"leftContent\":{\"columnMatch\":\"count_\",\"formatter\":3,\"formatOptions\":{\"palette\":\"blue\"}},\"showBorder\":false}},\"customWidth\":\"20\",\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"ServersExtensions\"},\"name\":\"query - 7\"},{\"type\":12,\"content\":{\"version\":\"NotebookGroup/1.0\",\"groupType\":\"editable\",\"items\":[{\"type\":11,\"content\":{\"version\":\"LinkItem/1.0\",\"style\":\"tabs\",\"links\":[{\"id\":\"ba5c2789-99fc-4a62-bfa7-a145dd3e0855\",\"cellValue\":\"selectedTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Azure machines within {Subscription:label}\",\"subTarget\":\"AzMachines\",\"style\":\"link\"},{\"id\":\"34116526-233e-417e-966f-df3bf94fe65c\",\"cellValue\":\"selectedTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Arc machines within {Subscription:label}\",\"subTarget\":\"ArcMachines\",\"style\":\"link\"},{\"id\":\"c3d204ef-88d2-487b-86d0-0d854aac4389\",\"cellValue\":\"selectedTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Azure or Arc machines with MMA\",\"subTarget\":\"MMA\",\"style\":\"link\"},{\"id\":\"eec95288-367a-4ecd-a359-7f1bf22beb9c\",\"cellValue\":\"selectedTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Machines with AMA\",\"subTarget\":\"AMA\",\"style\":\"link\"},{\"id\":\"d0e07018-4d0d-4caf-b0a6-1d1b67becb84\",\"cellValue\":\"selectedTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Machines with Both Agents under any RG\",\"subTarget\":\"Both\",\"style\":\"link\"},{\"id\":\"cfe87184-be1d-4a06-a87b-f5fbd310d7e7\",\"cellValue\":\"selectedTab\",\"linkTarget\":\"parameter\",\"linkLabel\":\"Non-Azure or hybrid VMs under any RG reporting to {Workspace:label} in the last 30d\",\"subTarget\":\"NAorHybrid\",\"style\":\"link\"}]},\"name\":\"Selection of Machines Tabs\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources\\r\\n| where \\\"{Subscription:Id}\\\" has subscriptionId or \\\"{Subscription}\\\" == \\\"\\\"\\r\\n| where \\\"{resourceGroup:label}\\\" contains resourceGroup or \\\"{resourceGroup}\\\" == \\\"\\\"\\r\\n| where type == \\\"microsoft.compute/virtualmachines\\\"\\r\\n| extend PowerStatus = properties.extended.instanceView.powerState.displayStatus,\\r\\n\\tOSType = properties.storageProfile.osDisk.osType\\r\\n| project name, VM=id, location, ResourceGroup=resourceGroup, PowerStatus, OSType\\r\\n| join kind=leftouter (\\r\\nresources\\r\\n| where type contains \\\"microsoft.compute/virtualmachines/extensions\\\" and (name == \\\"MicrosoftMonitoringAgent\\\" or name == \\\"OmsAgentForLinux\\\" or name == \\\"MMAExtension\\\")\\r\\n| parse id with * \\\"/virtualMachines/\\\" ComputerName \\\"/\\\" *\\r\\n| extend extensionType = properties.type, \\r\\n\\tstatus = properties.provisioningState,\\r\\n\\tversion = properties.typeHandlerVersion\\r\\n| project ComputerName, MMA = name, status, version\\r\\n) on $left.name == $right.ComputerName\\r\\n| join kind=leftouter (\\r\\nresources\\r\\n| where type contains \\\"microsoft.compute/virtualmachines/extensions\\\" and (name == \\\"AzureMonitorWindowsAgent\\\" or name == \\\"AzureMonitorLinuxAgent\\\")\\r\\n| parse id with * \\\"/virtualMachines/\\\" ComputerName \\\"/\\\" *\\r\\n| extend extensionType = properties.type, \\r\\n\\tstatus = properties.provisioningState,\\r\\n\\tversion = properties.typeHandlerVersion\\r\\n| project ComputerName, AMA = name, status, version\\r\\n) on $left.name == $right.ComputerName\",\"size\":2,\"showAnalytics\":true,\"noDataMessage\":\"No Azure Virtual Machines in the selected subscriptions. Please select the subscription(s) that contain your virtual machine(s)\",\"showRefreshButton\":true,\"showExportToExcel\":true,\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{Subscription}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"name\",\"formatter\":5},{\"columnMatch\":\"location\",\"formatter\":17},{\"columnMatch\":\"resourceGroup\",\"formatter\":14,\"formatOptions\":{\"showIcon\":true}},{\"columnMatch\":\"PowerStatus\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"VM running\",\"representation\":\"Available\",\"text\":\"{0}{1}\"},{\"operator\":\"Default\",\"representation\":\"2\",\"text\":\"{0}{1}\"}]}},{\"columnMatch\":\"OSType\",\"formatter\":1},{\"columnMatch\":\"ComputerName\",\"formatter\":5},{\"columnMatch\":\"MMA\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"contains\",\"thresholdValue\":\"Agent\",\"representation\":\"success\",\"text\":\"Deployed\"},{\"operator\":\"contains\",\"thresholdValue\":\"MMA\",\"representation\":\"success\",\"text\":\"Deployed\"},{\"operator\":\"Default\",\"representation\":\"Unavailable\",\"text\":\"Not Deployed\"}]}},{\"columnMatch\":\"status\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"Failed\",\"representation\":\"3\",\"text\":\"{0}{1}\"},{\"operator\":\"is Empty\",\"representation\":\"Blank\",\"text\":\"{0}{1}\"},{\"operator\":\"Default\",\"representation\":\"success\",\"text\":\"{0}{1}\"}]}},{\"columnMatch\":\"version\",\"formatter\":1},{\"columnMatch\":\"ComputerName1\",\"formatter\":5},{\"columnMatch\":\"AMA\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"contains\",\"thresholdValue\":\"Agent\",\"representation\":\"success\",\"text\":\"Deployed\"},{\"operator\":\"Default\",\"representation\":\"unknown\",\"text\":\"Not Deployed\"}]}},{\"columnMatch\":\"status1\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"Failed\",\"representation\":\"3\",\"text\":\"{0}{1}\"},{\"operator\":\"is Empty\",\"representation\":\"Blank\",\"text\":\"{0}{1}\"},{\"operator\":\"Default\",\"representation\":\"success\",\"text\":\"{0}{1}\"}]}},{\"columnMatch\":\"version1\",\"formatter\":1}],\"rowLimit\":5000,\"filter\":true,\"sortBy\":[{\"itemKey\":\"$gen_thresholds_AMA_11\",\"sortOrder\":2}],\"labelSettings\":[{\"columnId\":\"location\",\"label\":\"Location\"},{\"columnId\":\"PowerStatus\",\"label\":\"VM Status\"},{\"columnId\":\"status\",\"label\":\"Status\"},{\"columnId\":\"version\",\"label\":\"Version\"},{\"columnId\":\"status1\",\"label\":\"Status\"},{\"columnId\":\"version1\",\"label\":\"Version\"}]},\"sortBy\":[{\"itemKey\":\"$gen_thresholds_AMA_11\",\"sortOrder\":2}]},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"AzMachines\"},\"name\":\"azurevmquery-arg\"},{\"type\":12,\"content\":{\"version\":\"NotebookGroup/1.0\",\"groupType\":\"editable\",\"items\":[{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"Heartbeat\\r\\n| where ResourceProvider == \\\"Microsoft.HybridCompute\\\" and _ResourceId != \\\"\\\"\\r\\n| summarize LastHeartBeat = arg_max(TimeGenerated, *) by _ResourceId\\r\\n| extend TimeFromNow = now() - LastHeartBeat\\r\\n| extend [\\\"TimeAgo\\\"] = strcat(case(TimeFromNow < 2m, strcat(toint(TimeFromNow / 1m), ' seconds'), TimeFromNow < 2h, strcat(toint(TimeFromNow / 1m), ' minutes'), TimeFromNow < 2d, strcat(toint(TimeFromNow / 1h), ' hours'), strcat(toint(TimeFromNow / 1d), ' days')), ' ago')\\r\\n| project LastHeartBeat, [\\\"Time\\\"]=strcat('🕒 ', TimeAgo), _ResourceId, ResourceGroup, Computer, ManagementGroupName, OSName, Agent=Category, Version=Version\",\"size\":0,\"title\":\"Arc machines within {Subscription:label}\",\"timeContext\":{\"durationMs\":604800000},\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{Workspace}\"]},\"conditionalVisibility\":{\"parameterName\":\"0\",\"comparison\":\"isEqualTo\",\"value\":\"0\"},\"name\":\"Arc-Heartbeats\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources\\r\\n| where type == \\\"microsoft.hybridcompute/machines\\\"\\r\\n| extend provisioningState = properties.provisioningState,\\r\\n\\tstatus = properties.status,\\r\\n\\tagentVersion = properties.agentVersion,\\r\\n\\tlastStatusChange = properties.lastStatusChange,\\r\\n\\terrorDetails = properties.errorDetails,\\r\\n\\tosName = properties.osName,\\r\\n\\tosSku = properties.osSku\\r\\n| extend name = tolower(name)\\r\\n| project id, name, location, resourceGroup, provisioningState, status, agentVersion, lastStatusChange, osName, osSku\\r\\n| join kind=leftouter (\\r\\nresources\\r\\n| where type contains \\\"microsoft.hybridcompute/machines/extensions\\\" and (name == \\\"MicrosoftMonitoringAgent\\\" or name == \\\"OmsAgentForLinux\\\" or name == \\\"OMSAgentForLinux\\\")\\r\\n| parse id with * \\\"/machines/\\\" ComputerName \\\"/\\\" *\\r\\n| extend extensionType = properties.type, \\r\\n\\tstatus = properties.provisioningState,\\r\\n\\tversion = properties.typeHandlerVersion\\r\\n| extend ComputerName = tolower(ComputerName)\\r\\n| project ComputerName, name, status, version\\r\\n| order by ComputerName, name\\r\\n) on $left.name == $right.ComputerName\\r\\n| join kind=leftouter (\\r\\nresources\\r\\n| where type contains \\\"microsoft.hybridcompute/machines/extensions\\\" and (name == \\\"AzureMonitorWindowsAgent\\\" or name == \\\"AzureMonitorLinuxAgent\\\")\\r\\n| parse id with * \\\"/machines/\\\" ComputerName \\\"/\\\" *\\r\\n| extend extensionType = properties.type, \\r\\n\\tstatus = properties.provisioningState,\\r\\n\\tversion = properties.typeHandlerVersion\\r\\n| extend ComputerName = tolower(ComputerName)\\r\\n| project ComputerName, name, status, version\\r\\n| order by ComputerName, name\\r\\n) on $left.name == $right.ComputerName\",\"size\":1,\"showAnalytics\":true,\"noDataMessage\":\"No Arc-enabled Virtual Machines in the selected subscriptions. Please select the subscription(s) that contain your Arc-enabled virtual machine(s)\",\"noDataMessageStyle\":2,\"showRefreshButton\":true,\"showExportToExcel\":true,\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{Subscription}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"name\",\"formatter\":5},{\"columnMatch\":\"location\",\"formatter\":17},{\"columnMatch\":\"provisioningState\",\"formatter\":5},{\"columnMatch\":\"status\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"Connected\",\"representation\":\"success\",\"text\":\"{0}{1}\"},{\"operator\":\"==\",\"thresholdValue\":\"Disconnected\",\"representation\":\"warning\",\"text\":\"{0}{1}\"},{\"operator\":\"==\",\"thresholdValue\":\"Expired\",\"representation\":\"Degraded\",\"text\":\"{0}{1}\"},{\"operator\":\"Default\",\"representation\":\"Blank\",\"text\":\"{0}{1}\"}]}},{\"columnMatch\":\"agentVersion\",\"formatter\":1},{\"columnMatch\":\"lastStatusChange\",\"formatter\":6,\"dateFormat\":{\"formatName\":\"longDatePattern\"}},{\"columnMatch\":\"osName\",\"formatter\":1},{\"columnMatch\":\"osSku\",\"formatter\":5},{\"columnMatch\":\"ComputerName\",\"formatter\":5},{\"columnMatch\":\"name1\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"contains\",\"thresholdValue\":\"Agent\",\"representation\":\"success\",\"text\":\"Deployed\"},{\"operator\":\"Default\",\"representation\":\"Blank\",\"text\":\"Not Deployed\"}]}},{\"columnMatch\":\"status1\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"Succeeded\",\"representation\":\"success\",\"text\":\"{0}{1}\"},{\"operator\":\"==\",\"thresholdValue\":\"Failed\",\"representation\":\"3\",\"text\":\"{0}{1}\"},{\"operator\":\"Default\",\"representation\":\"Blank\",\"text\":\"{0}{1}\"}]}},{\"columnMatch\":\"version\",\"formatter\":1},{\"columnMatch\":\"ComputerName1\",\"formatter\":5},{\"columnMatch\":\"name2\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"contains\",\"thresholdValue\":\"Agent\",\"representation\":\"success\",\"text\":\"Deployed\"},{\"operator\":\"Default\",\"representation\":\"Blank\",\"text\":\"Not Deployed\"}]}},{\"columnMatch\":\"status2\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"Succeeded\",\"representation\":\"success\",\"text\":\"{0}{1}\"},{\"operator\":\"==\",\"thresholdValue\":\"Failed\",\"representation\":\"3\",\"text\":\"{0}{1}\"},{\"operator\":\"Default\",\"representation\":\"Blank\",\"text\":\"{0}{1}\"}]}},{\"columnMatch\":\"version1\",\"formatter\":1}],\"rowLimit\":5000,\"labelSettings\":[{\"columnId\":\"id\",\"label\":\"Server Name\"},{\"columnId\":\"location\",\"label\":\"Location\"},{\"columnId\":\"status\",\"label\":\"Arc Status\"},{\"columnId\":\"name1\",\"label\":\"MMA\"},{\"columnId\":\"status1\",\"label\":\"Status\"},{\"columnId\":\"version\",\"label\":\"Version\"},{\"columnId\":\"name2\",\"label\":\"AMA\"},{\"columnId\":\"status2\",\"label\":\"Status\"},{\"columnId\":\"version1\",\"label\":\"Version\"}]}},\"conditionalVisibility\":{\"parameterName\":\"0\",\"comparison\":\"isEqualTo\",\"value\":\"0\"},\"name\":\"ArcVMQuery-arg\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"{\\\"version\\\":\\\"Merge/1.0\\\",\\\"merges\\\":[{\\\"id\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\",\\\"mergeType\\\":\\\"leftouter\\\",\\\"leftTable\\\":\\\"Arc-Heartbeats\\\",\\\"rightTable\\\":\\\"ArcVMQuery-arg\\\",\\\"leftColumn\\\":\\\"_ResourceId\\\",\\\"rightColumn\\\":\\\"id\\\"}],\\\"projectRename\\\":[{\\\"originalName\\\":\\\"[ArcVMQuery-arg].id\\\",\\\"mergedName\\\":\\\"Server Name\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[Arc-Heartbeats].ManagementGroupName\\\",\\\"mergedName\\\":\\\"ManagementGroupName\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Arc-Heartbeats].Time\\\",\\\"mergedName\\\":\\\"Time\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[Arc-Heartbeats]._ResourceId\\\",\\\"mergedName\\\":\\\"_ResourceId\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[Arc-Heartbeats].Computer\\\",\\\"mergedName\\\":\\\"Computer\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[Arc-Heartbeats].LastHeartBeat\\\",\\\"mergedName\\\":\\\"LastHeartBeat\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].name\\\",\\\"mergedName\\\":\\\"name\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].location\\\",\\\"mergedName\\\":\\\"Location\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].resourceGroup\\\",\\\"mergedName\\\":\\\"resourceGroup\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].provisioningState\\\",\\\"mergedName\\\":\\\"provisioningState\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].status\\\",\\\"mergedName\\\":\\\"Arc Status\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].agentVersion\\\",\\\"mergedName\\\":\\\"agentVersion\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].lastStatusChange\\\",\\\"mergedName\\\":\\\"lastStatusChange\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].osName\\\",\\\"mergedName\\\":\\\"osName\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].osSku\\\",\\\"mergedName\\\":\\\"osSku\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].ComputerName\\\",\\\"mergedName\\\":\\\"ComputerName\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].name1\\\",\\\"mergedName\\\":\\\"MMA\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].status1\\\",\\\"mergedName\\\":\\\"Status\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].version\\\",\\\"mergedName\\\":\\\"Version\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].ComputerName1\\\",\\\"mergedName\\\":\\\"ComputerName1\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].name2\\\",\\\"mergedName\\\":\\\"AMA\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].status2\\\",\\\"mergedName\\\":\\\"Status1\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[ArcVMQuery-arg].version1\\\",\\\"mergedName\\\":\\\"Version1\\\",\\\"fromId\\\":\\\"fec3768e-dd50-424e-a15f-0919f1cc2122\\\"},{\\\"originalName\\\":\\\"[Arc-Heartbeats].ResourceGroup\\\",\\\"mergedName\\\":\\\"ResourceGroup\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Arc-Heartbeats].OSName\\\",\\\"mergedName\\\":\\\"OSName\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Arc-Heartbeats].Agent\\\",\\\"mergedName\\\":\\\"Agent\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Arc-Heartbeats].Version\\\",\\\"mergedName\\\":\\\"Version\\\",\\\"fromId\\\":\\\"unknown\\\"}]}\",\"size\":0,\"title\":\"{$rowCount} Azure Arc-enabled servers\",\"noDataMessage\":\"No Arc-enabled Virtual Machines with a relationship to the selected workspace. Please select the workspace that contain your Arc-enabled virtual machine(s)\",\"showRefreshButton\":true,\"showExportToExcel\":true,\"queryType\":7,\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"Server Name\",\"formatter\":13,\"formatOptions\":{\"linkTarget\":\"Resource\",\"linkIsContextBlade\":true,\"showIcon\":true}},{\"columnMatch\":\"ManagementGroupName\",\"formatter\":5},{\"columnMatch\":\"_ResourceId\",\"formatter\":5},{\"columnMatch\":\"Computer\",\"formatter\":5},{\"columnMatch\":\"LastHeartBeat\",\"formatter\":5},{\"columnMatch\":\"name\",\"formatter\":5},{\"columnMatch\":\"Location\",\"formatter\":17},{\"columnMatch\":\"resourceGroup\",\"formatter\":14,\"formatOptions\":{\"showIcon\":true}},{\"columnMatch\":\"provisioningState\",\"formatter\":5},{\"columnMatch\":\"Arc Status\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"==\",\"thresholdValue\":\"Expired\",\"representation\":\"2\",\"text\":\"{0}{1}\"},{\"operator\":\"==\",\"thresholdValue\":\"Disconnected\",\"representation\":\"3\",\"text\":\"{0}{1}\"},{\"operator\":\"Default\",\"representation\":\"success\",\"text\":\"{0}{1}\"}]}},{\"columnMatch\":\"agentVersion\",\"formatter\":1},{\"columnMatch\":\"lastStatusChange\",\"formatter\":6},{\"columnMatch\":\"osName\",\"formatter\":5},{\"columnMatch\":\"osSku\",\"formatter\":1},{\"columnMatch\":\"ComputerName\",\"formatter\":5},{\"columnMatch\":\"MMA\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"contains\",\"thresholdValue\":\"Agent\",\"representation\":\"success\",\"text\":\"Deployed\"},{\"operator\":\"Default\",\"representation\":\"unknown\",\"text\":\"Not Deployed\"}]}},{\"columnMatch\":\"Status\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"contains\",\"thresholdValue\":\"Succeeded\",\"representation\":\"success\",\"text\":\"{0}{1}\"},{\"operator\":\"is Empty\",\"representation\":\"Blank\",\"text\":\"{0}{1}\"},{\"operator\":\"Default\",\"representation\":\"2\",\"text\":\"{0}{1}\"}]}},{\"columnMatch\":\"Version\",\"formatter\":1},{\"columnMatch\":\"ComputerName1\",\"formatter\":5},{\"columnMatch\":\"AMA\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"contains\",\"thresholdValue\":\"Agent\",\"representation\":\"success\",\"text\":\"Deployed\"},{\"operator\":\"Default\",\"representation\":\"unknown\",\"text\":\"Not Deployed\"}]}},{\"columnMatch\":\"Status1\",\"formatter\":18,\"formatOptions\":{\"thresholdsOptions\":\"icons\",\"thresholdsGrid\":[{\"operator\":\"is Empty\",\"representation\":\"Blank\",\"text\":\"{0}{1}\"},{\"operator\":\"contains\",\"thresholdValue\":\"Succeeded\",\"representation\":\"success\",\"text\":\"{0}{1}\"},{\"operator\":\"Default\",\"representation\":\"2\",\"text\":\"{0}{1}\"}]}},{\"columnMatch\":\"Version1\",\"formatter\":1}],\"rowLimit\":5000,\"filter\":true,\"labelSettings\":[{\"columnId\":\"Time\",\"label\":\"Last Heartbeat\"},{\"columnId\":\"resourceGroup\",\"label\":\"Resource Group\"},{\"columnId\":\"osSku\",\"label\":\"Operating System\"},{\"columnId\":\"Status\",\"label\":\"MMA Status\"},{\"columnId\":\"Status1\",\"label\":\"AMA Status\"},{\"columnId\":\"Version1\",\"label\":\"Version\"}]}},\"name\":\"ArcAgents-Full\"}]},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"ArcMachines\"},\"name\":\"Arc tab\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources\\r\\n| where \\\"{Subscription:Id}\\\" has subscriptionId or \\\"{Subscription}\\\" == \\\"\\\"\\r\\n| where \\\"{resourceGroup:label}\\\" contains resourceGroup or \\\"{resourceGroup}\\\" == \\\"\\\" \\r\\n| where type has 'microsoft.compute/virtualmachines/extensions' or type has 'microsoft.hybridcompute/machines/extensions'\\r\\n| where name has 'MicrosoftMonitoringAgent' or name has 'OmsAgentForLinux' \\r\\n| extend Server = extract('(.*)/extensions',1,id)\\r\\n| extend Subscription = extract('(/subscriptions/.*)/resource.*',1,id)\\r\\n| summarize count() by Server, subscriptionId, resourceGroup, Subscription\\r\\n| project Server, resourceGroup, Subscription\\r\\n| sort by Server asc\",\"size\":0,\"title\":\"Azure or Arc machines with MMA\",\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{Subscription}\"]},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"MMA\"},\"name\":\"query - 1\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources \\r\\n| where \\\"{Subscription:Id}\\\" has subscriptionId or \\\"{Subscription}\\\" == \\\"\\\"\\r\\n| where \\\"{resourceGroup:label}\\\" contains resourceGroup or \\\"{resourceGroup}\\\" == \\\"\\\"\\r\\n| where type has 'microsoft.compute/virtualmachines/extensions' or type has 'microsoft.hybridcompute/machines/extensions'\\r\\n| where name has 'AzureMonitorWindowsAgent' or name has 'AzureMonitorLinuxAgent' \\r\\n| extend VM = extract('(/subscriptions.*)/extensions',1,id)\\r\\n| extend Subscription = extract('(/subscriptions/.*)/resource.*',1,id)\\r\\n| summarize count() by VM, subscriptionId, resourceGroup, Subscription\\r\\n| project VM, resourceGroup, Subscription\\r\\n| sort by VM asc\",\"size\":0,\"title\":\"Machines with AMA\",\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{Subscription}\"]},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"AMA\"},\"name\":\"query - 2\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources\\r\\n| where \\\"{resourceGroup:label}\\\" contains resourceGroup or \\\"{resourceGroup}\\\" == \\\"\\\"\\r\\n| where type has 'microsoft.compute/virtualmachines/extensions' or type has 'microsoft.hybridcompute/machines/extensions'\\r\\n| where name has 'MicrosoftMonitoringAgent' or name has 'OmsAgentForLinux' or name has 'AzureMonitorWindowsAgent' or name has 'AzureMonitorLinuxAgent'\\r\\n| extend AzureVM = extract('virtualmachines|virtualMachines/(.*)/extensions',1,id), ArcVM = extract('machines/(.*)/extensions',1,id)\\r\\n| summarize count() by AzureVM=tolower(AzureVM), ArcVM=tolower(ArcVM), subscriptionId, resourceGroup \\r\\n| extend hasBoth = iff(count_ > 1, 'Yes', 'No') | where count_ > 1 \\r\\n| join (resources | where \\\"{resourceGroup:label}\\\" contains resourceGroup or \\\"{resourceGroup}\\\" == \\\"\\\" | where type has 'microsoft.compute/virtualmachines/extensions' or type has 'microsoft.hybridcompute/machines/extensions'\\r\\n| where name has 'MicrosoftMonitoringAgent' or name has 'OmsAgentForLinux' \\r\\n| extend AzureVM = extract('virtualmachines|virtualMachines/(.*)/extensions',1,id)) on AzureVM \\r\\n| project AzureVM, ArcVM, resourceGroup, MMAVersion=name, hasBoth\",\"size\":0,\"title\":\"Machines with Both Agents under any RG\",\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\"},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"Both\"},\"name\":\"query - 3\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"let onprem = Heartbeat | where isempty(_ResourceId) == true;\\r\\nlet Arc = Heartbeat | where ResourceProvider =~ \\\"microsoft.hybridcompute\\\";\\r\\nunion withsource=\\\"Heartbeat\\\"\\r\\nonprem, Arc\\r\\n| summarize by Computer, OSType, ResourceProvider\",\"size\":0,\"title\":\"Non-Azure or hybrid VMs under any RG reporting to {Workspace:label} in the last 30d\",\"timeContext\":{\"durationMs\":2592000000},\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{Workspace}\"],\"gridSettings\":{\"sortBy\":[{\"itemKey\":\"ResourceProvider\",\"sortOrder\":2}]},\"sortBy\":[{\"itemKey\":\"ResourceProvider\",\"sortOrder\":2}]},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"NAorHybrid\"},\"name\":\"query - 9\"}]},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"ServersExtensions\"},\"name\":\"Selection of Machines\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"{\\\"version\\\":\\\"ARMEndpoint/1.0\\\",\\\"data\\\":null,\\\"headers\\\":[],\\\"method\\\":\\\"GET\\\",\\\"path\\\":\\\"/subscriptions/{Subscription2:Id}/providers/Microsoft.Insights/dataCollectionRules\\\",\\\"urlParams\\\":[{\\\"key\\\":\\\"api-version\\\",\\\"value\\\":\\\"2019-11-01-preview\\\"}],\\\"batchDisabled\\\":false,\\\"transformers\\\":[{\\\"type\\\":\\\"jsonpath\\\",\\\"settings\\\":{\\\"tablePath\\\":\\\"$.value\\\",\\\"columns\\\":[{\\\"path\\\":\\\"name\\\",\\\"columnid\\\":\\\"Name\\\"},{\\\"path\\\":\\\"$..workspaceResourceId\\\",\\\"columnid\\\":\\\"Workspace\\\"},{\\\"path\\\":\\\"kind\\\",\\\"columnid\\\":\\\"OS\\\"},{\\\"path\\\":\\\"properties.dataFlows[*].streams[0]\\\",\\\"columnid\\\":\\\"Streams\\\",\\\"substringRegexMatch\\\":\\\"(Microsoft-)*(\\\\\\\\w+)\\\",\\\"substringReplace\\\":\\\"$2\\\"},{\\\"path\\\":\\\"$..xPathQueries\\\",\\\"columnid\\\":\\\"xPath\\\"},{\\\"path\\\":\\\"location\\\",\\\"columnid\\\":\\\"Location\\\"},{\\\"path\\\":\\\"$..facilityNames\\\",\\\"columnid\\\":\\\"SyslogFacilities\\\"}]}}]}\",\"size\":0,\"title\":\"Get all DCRs under {Subscription:label}\",\"exportedParameters\":[{\"fieldName\":\"Name\",\"parameterName\":\"dcrName\",\"parameterType\":1},{\"fieldName\":\"resourceGroup\",\"parameterName\":\"resourceGroup\",\"parameterType\":1}],\"queryType\":12,\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"workspace2\",\"formatter\":5}]}},\"conditionalVisibility\":{\"parameterName\":\"0\",\"comparison\":\"isEqualTo\",\"value\":\"0\"},\"name\":\"Get DCRs and associations\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources\\r\\n| where type has 'microsoft.insights/datacollectionrules'\\r\\n| extend Subscription = extract('(/subscriptions/.*)/resource.*',1,id)\\r\\n| project name, Subscription, resourceGroup\",\"size\":0,\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"value::all\"]},\"conditionalVisibility\":{\"parameterName\":\"0\",\"comparison\":\"isEqualTo\",\"value\":\"0\"},\"name\":\"Get DCRs from Graph\"},{\"type\":1,\"content\":{\"json\":\"Use this section to see all the DCRs under the selected subscription. You will find Streams and the xPath query.\\r\\nThe stream will normally match the destination table in the Microsoft Sentinel workspace. If it starts with \\\"Microsoft-\\\", the destination table is what comes after. For example, if the stream says \\\"Microsoft-Perf\\\", then the destination table is \\\"Perf\\\".\\r\\nClick on the DCRs to see associated VMs.\",\"style\":\"info\"},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"DCRs\"},\"name\":\"Help tab 2\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"{\\\"version\\\":\\\"Merge/1.0\\\",\\\"merges\\\":[{\\\"id\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\",\\\"mergeType\\\":\\\"leftouter\\\",\\\"leftTable\\\":\\\"Get DCRs and associations\\\",\\\"rightTable\\\":\\\"Get DCRs from Graph\\\",\\\"leftColumn\\\":\\\"Name\\\",\\\"rightColumn\\\":\\\"name\\\"}],\\\"projectRename\\\":[{\\\"originalName\\\":\\\"[Get DCRs and associations].Name\\\",\\\"mergedName\\\":\\\"Name\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs and associations].Workspace\\\",\\\"mergedName\\\":\\\"Workspace\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs and associations].OS\\\",\\\"mergedName\\\":\\\"OS\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs and associations].Streams\\\",\\\"mergedName\\\":\\\"Streams\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs and associations].xPath\\\",\\\"mergedName\\\":\\\"xPath\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs and associations].Location\\\",\\\"mergedName\\\":\\\"Location\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs and associations].SyslogFacilities\\\",\\\"mergedName\\\":\\\"SyslogFacilities\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs from Graph].id\\\",\\\"mergedName\\\":\\\"id\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs from Graph].name\\\",\\\"mergedName\\\":\\\"name\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs from Graph].kind\\\",\\\"mergedName\\\":\\\"kind\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs from Graph].resourceGroup\\\",\\\"mergedName\\\":\\\"resourceGroup\\\",\\\"fromId\\\":\\\"bee4471d-99cc-4c86-9f3a-1e4f0a05e32a\\\"},{\\\"originalName\\\":\\\"[Get DCRs from Graph].Subscription\\\",\\\"mergedName\\\":\\\"Subscription\\\",\\\"fromId\\\":\\\"unknown\\\"}]}\",\"size\":0,\"title\":\"Select a DCR to view associated VMs\",\"exportedParameters\":[{\"fieldName\":\"Name\",\"parameterName\":\"DCR\",\"parameterType\":1},{\"fieldName\":\"resourceGroup\",\"parameterName\":\"DCRrg\",\"parameterType\":1}],\"queryType\":7},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"DCRs\"},\"name\":\"Merge - DCRs and RG\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"{\\\"version\\\":\\\"ARMEndpoint/1.0\\\",\\\"data\\\":null,\\\"headers\\\":[],\\\"method\\\":\\\"GET\\\",\\\"path\\\":\\\"/subscriptions/{Subscription2:Id}/resourceGroups/{DCRrg}/providers/Microsoft.Insights/dataCollectionRules/{DCR}/associations\\\",\\\"urlParams\\\":[{\\\"key\\\":\\\"api-version\\\",\\\"value\\\":\\\"2019-11-01-preview\\\"}],\\\"batchDisabled\\\":false,\\\"transformers\\\":[{\\\"type\\\":\\\"jsonpath\\\",\\\"settings\\\":{\\\"tablePath\\\":\\\"$.value\\\",\\\"columns\\\":[{\\\"path\\\":\\\"id\\\",\\\"columnid\\\":\\\"VMs\\\",\\\"columnType\\\":\\\"string\\\",\\\"substringRegexMatch\\\":\\\"(\\\\\\\\/subscriptions.*)(\\\\\\\\/providers.*|Providers.*)\\\",\\\"substringReplace\\\":\\\"$1\\\"}]}}]}\",\"size\":4,\"title\":\"VMs associated to the selected DCR\",\"noDataMessage\":\"No VMs found or no DCR selected above\",\"queryType\":12},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"DCRs\"},\"name\":\"query - 18\"},{\"type\":12,\"content\":{\"version\":\"NotebookGroup/1.0\",\"groupType\":\"editable\",\"title\":\"View all VMs with AMA under the {Subscription:label} subscription and their data collection rules\",\"items\":[{\"type\":1,\"content\":{\"json\":\"These are all the servers with AMA installed. Select each one individually to see the Data Collection Rules associated to them.
You will find information such as what is being collected and to what workspace those logs are being sent. The stream will normally match the destination table in the Microsoft Sentinel workspace. If it starts with \\\"Microsoft-\\\", the destination table is what comes after. For example, if the stream says \\\"Microsoft-Perf\\\", then the destination table is \\\"Perf\\\".
To learn more about DCRs, please check https://docs.microsoft.com/azure/azure-monitor/agents/data-collection-rule-azure-monitor-agent\",\"style\":\"info\"},\"conditionalVisibilities\":[{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"AMAvms\"},{\"parameterName\":\"Help\",\"comparison\":\"isEqualTo\",\"value\":\"Yes\"}],\"name\":\"text - 19\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources\\r\\n| where type has 'microsoft.compute/virtualmachines' or type has \\\"microsoft.hybridcompute/machines\\\" \\r\\n| project Computer=tolower(name)\\r\\n| join (resources | where type has 'microsoft.compute/virtualmachines/extensions' \\r\\n| where name has 'azuremonitorwindowsagent' or name has 'azuremonitorlinuxagent' \\r\\n| extend type, temp = extract('virtualmachines|virtualMachines/(.*)/extensions',1,id), vmid = extract('(/subscriptions/.*achines/.*)(/extensions*)',1,id)\\r\\n| extend Subscription = extract('(/subscriptions/.*)/resource.*',1,id)\\r\\n| extend Computer = tolower(temp)\\r\\n)\\r\\non Computer\\r\\n| union (resources | where type has 'microsoft.hybridcompute/machines/extensions'\\r\\n| where name has 'azuremonitorwindowsagent' or name has 'azuremonitorlinuxagent'\\r\\n| extend id, Computer = extract('machines/(.*)/extensions',1,id), vmid = extract('(/subscriptions/.*achines/.*)(/extensions*)',1,id))\\r\\n| project vmid, Computer, resourceGroup, location, Subscription\",\"size\":0,\"title\":\"Get Azure and hybrid VMs with AMA\",\"exportedParameters\":[{\"fieldName\":\"Computer\",\"parameterName\":\"vmname\",\"parameterType\":1},{\"fieldName\":\"resourceGroup\",\"parameterName\":\"vmrg\",\"parameterType\":1},{\"fieldName\":\"vmid\",\"parameterName\":\"vmid1\",\"parameterType\":1}],\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\",\"crossComponentResources\":[\"{Subscription}\"],\"gridSettings\":{\"formatters\":[{\"columnMatch\":\"vmname\",\"formatter\":5}],\"sortBy\":[{\"itemKey\":\"$gen_link_vmid_0\",\"sortOrder\":2}]},\"sortBy\":[{\"itemKey\":\"$gen_link_vmid_0\",\"sortOrder\":2}]},\"name\":\"Azure and hybrid VMs with AMA\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"resources \\r\\n| where type has 'microsoft.compute/virtualmachines/extensions' or type has 'microsoft.hybridcompute/machines/extensions'\\r\\n| where name has 'AzureMonitorWindowsAgent' or name has 'AzureMonitorLinuxAgent' \\r\\n| extend AzureVM = extract('virtualMachines/(.*)/extensions',1,id), ArcVM = extract('machines/(.*)/extensions',1,id) \\r\\n| summarize count() by AzureVM, ArcVM, subscriptionId, resourceGroup \\r\\n| project AzureVM, ArcVM, resourceGroup\",\"size\":0,\"queryType\":1,\"resourceType\":\"microsoft.resourcegraph/resources\"},\"conditionalVisibility\":{\"parameterName\":\"0\",\"comparison\":\"isEqualTo\",\"value\":\"0\"},\"name\":\"Azure or Arc machines with AMA\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"{\\\"version\\\":\\\"ARMEndpoint/1.0\\\",\\\"data\\\":null,\\\"headers\\\":[],\\\"method\\\":\\\"GET\\\",\\\"path\\\":\\\"/{vmid1}/providers/Microsoft.Insights/dataCollectionRuleAssociations\\\",\\\"urlParams\\\":[{\\\"key\\\":\\\"api-version\\\",\\\"value\\\":\\\"2019-11-01-preview\\\"}],\\\"batchDisabled\\\":false,\\\"transformers\\\":[{\\\"type\\\":\\\"jsonpath\\\",\\\"settings\\\":{\\\"tablePath\\\":\\\"$.value\\\",\\\"columns\\\":[{\\\"path\\\":\\\"$..dataCollectionRuleId\\\",\\\"columnid\\\":\\\"DCRname\\\",\\\"substringRegexMatch\\\":\\\"\\\\\\\\/subscriptions\\\\\\\\/[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}\\\\\\\\/\\\\\\\\w+\\\\\\\\/[a-zA-Z0-9\\\\\\\\.\\\\\\\\_\\\\\\\\-\\\\\\\\(\\\\\\\\)]{1,90}\\\\\\\\/providers\\\\\\\\/\\\\\\\\w+\\\\\\\\.\\\\\\\\w+\\\\\\\\/\\\\\\\\w+\\\\\\\\/([a-zA-Z0-9\\\\\\\\.\\\\\\\\_\\\\\\\\-\\\\\\\\(\\\\\\\\)]{1,64})\\\",\\\"substringReplace\\\":\\\"$1\\\"},{\\\"path\\\":\\\"$..dataCollectionRuleId\\\",\\\"columnid\\\":\\\"DCR\\\"},{\\\"path\\\":\\\"$..dataCollectionRuleId\\\",\\\"columnid\\\":\\\"DCRrg\\\",\\\"substringRegexMatch\\\":\\\"\\\\\\\\/.*\\\\\\\\/resource.*s\\\\\\\\/(.*)\\\\\\\\/providers.*\\\",\\\"substringReplace\\\":\\\"$1\\\"}]}}]}\",\"size\":0,\"exportedParameters\":[{\"fieldName\":\"DCRname\",\"parameterName\":\"DCRname\",\"parameterType\":1},{\"fieldName\":\"DCRrg\",\"parameterName\":\"DCRrg\",\"parameterType\":1}],\"queryType\":12},\"conditionalVisibility\":{\"parameterName\":\"a\",\"comparison\":\"isEqualTo\",\"value\":\"a\"},\"name\":\"Get DCR associations for a given VM\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"{\\\"version\\\":\\\"ARMEndpoint/1.0\\\",\\\"data\\\":null,\\\"headers\\\":[],\\\"method\\\":\\\"GET\\\",\\\"path\\\":\\\"/subscriptions/{Subscription2:Id}/providers/Microsoft.Insights/dataCollectionRules\\\",\\\"urlParams\\\":[{\\\"key\\\":\\\"api-version\\\",\\\"value\\\":\\\"2019-11-01-preview\\\"}],\\\"batchDisabled\\\":false,\\\"transformers\\\":[{\\\"type\\\":\\\"jsonpath\\\",\\\"settings\\\":{\\\"tablePath\\\":\\\"$.value\\\",\\\"columns\\\":[{\\\"path\\\":\\\"name\\\",\\\"columnid\\\":\\\"Name\\\"},{\\\"path\\\":\\\"properties.dataFlows[*].streams[0]\\\",\\\"columnid\\\":\\\"Streams\\\",\\\"substringRegexMatch\\\":\\\"(Microsoft-)*(\\\\\\\\w+)\\\",\\\"substringReplace\\\":\\\"$2\\\"},{\\\"path\\\":\\\"$..workspaceResourceId\\\",\\\"columnid\\\":\\\"Workspace\\\"},{\\\"path\\\":\\\"kind\\\",\\\"columnid\\\":\\\"OS\\\"},{\\\"path\\\":\\\"$..xPathQueries\\\",\\\"columnid\\\":\\\"xPath\\\",\\\"columnType\\\":\\\"string\\\"},{\\\"path\\\":\\\"$..facilityNames\\\",\\\"columnid\\\":\\\"SyslogFacilities\\\"},{\\\"path\\\":\\\"$..counterSpecifiers\\\",\\\"columnid\\\":\\\"CounterSpecifiers\\\"},{\\\"path\\\":\\\"properties.dataFlows[*].streams[0]\\\",\\\"columnid\\\":\\\"Custom\\\"}]}}]}\",\"size\":0,\"queryType\":12},\"conditionalVisibility\":{\"parameterName\":\"a\",\"comparison\":\"isEqualTo\",\"value\":\"a\"},\"name\":\"Get ALL DCRs under subscription\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"{\\\"version\\\":\\\"Merge/1.0\\\",\\\"merges\\\":[{\\\"id\\\":\\\"24881457-2964-4063-b883-330dad837178\\\",\\\"mergeType\\\":\\\"innerunique\\\",\\\"leftTable\\\":\\\"Get DCR associations for a given VM\\\",\\\"rightTable\\\":\\\"Get ALL DCRs under subscription\\\",\\\"leftColumn\\\":\\\"DCRname\\\",\\\"rightColumn\\\":\\\"Name\\\"}],\\\"projectRename\\\":[{\\\"originalName\\\":\\\"[Get DCR associations for a given VM].DCR\\\",\\\"mergedName\\\":\\\"DCR\\\",\\\"fromId\\\":\\\"24881457-2964-4063-b883-330dad837178\\\"},{\\\"originalName\\\":\\\"[Get DCR associations for a given VM].DCRrg\\\",\\\"mergedName\\\":\\\"DCR RG\\\",\\\"fromId\\\":\\\"24881457-2964-4063-b883-330dad837178\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].OS\\\",\\\"mergedName\\\":\\\"OS\\\",\\\"fromId\\\":\\\"24881457-2964-4063-b883-330dad837178\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].Workspace\\\",\\\"mergedName\\\":\\\"Workspace\\\",\\\"fromId\\\":\\\"24881457-2964-4063-b883-330dad837178\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].Streams\\\",\\\"mergedName\\\":\\\"Streams\\\",\\\"fromId\\\":\\\"24881457-2964-4063-b883-330dad837178\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].xPath\\\",\\\"mergedName\\\":\\\"xPath\\\",\\\"fromId\\\":\\\"24881457-2964-4063-b883-330dad837178\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].SyslogFacilities\\\",\\\"mergedName\\\":\\\"SyslogFacilities\\\",\\\"fromId\\\":\\\"24881457-2964-4063-b883-330dad837178\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].CounterSpecifiers\\\",\\\"mergedName\\\":\\\"CounterSpecifiers\\\",\\\"fromId\\\":\\\"24881457-2964-4063-b883-330dad837178\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].Parsed streams\\\",\\\"mergedName\\\":\\\"Parsed streams\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].ParsedStreams\\\",\\\"mergedName\\\":\\\"ParsedStreams\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].test\\\",\\\"mergedName\\\":\\\"test\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].WindowsStreams\\\",\\\"mergedName\\\":\\\"WindowsStreams\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].SyslogStreams\\\",\\\"mergedName\\\":\\\"SyslogStreams\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].DataFlows\\\",\\\"mergedName\\\":\\\"DataFlows\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].LinuxStreams\\\",\\\"mergedName\\\":\\\"LinuxStreams\\\",\\\"fromId\\\":\\\"unknown\\\"},{\\\"originalName\\\":\\\"[Get DCR associations for a given VM].DCRname\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].Name\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].Custom\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].Streams2\\\"},{\\\"originalName\\\":\\\"[Get ALL DCRs under subscription].StreamsRegex\\\"}]}\",\"size\":1,\"queryType\":7},\"showPin\":false,\"name\":\"query - 4\"},{\"type\":9,\"content\":{\"version\":\"KqlParameterItem/1.0\",\"parameters\":[{\"id\":\"beb26b8d-3f76-4177-8dac-5c7f7e19791d\",\"version\":\"KqlParameterItem/1.0\",\"name\":\"TimeRange\",\"label\":\"Time range\",\"type\":4,\"isRequired\":true,\"value\":{\"durationMs\":2592000000},\"typeSettings\":{\"selectableValues\":[{\"durationMs\":86400000},{\"durationMs\":259200000},{\"durationMs\":604800000},{\"durationMs\":1209600000},{\"durationMs\":2592000000},{\"durationMs\":7776000000}],\"allowCustom\":true},\"timeContext\":{\"durationMs\":86400000}}],\"style\":\"pills\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\"},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"AMAvms\"},\"name\":\"Time Picker\"},{\"type\":1,\"content\":{\"json\":\"Below you can find the number of events submitted by the selected VM to some of the main native tables in Microsoft Sentinel and Log Analytics.
The first graph includes events logged in the tables: SecurityEvent, Heartbeat, Perf and Event.
The second graph shows the number of events forwarded by the selected VM to the WindowsEvent table if you are using the Windows Forwarded Events connector.
Note that if the selected VM also has MMA installed, the logs could come from either one of the agents, AMA or MMA\",\"style\":\"info\"},\"conditionalVisibilities\":[{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"AMAvms\"},{\"parameterName\":\"Help\",\"comparison\":\"isEqualTo\",\"value\":\"Yes\"}],\"name\":\"text - 19 - Copy\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"SecurityEvent | union Heartbeat, Event, Perf\\r\\n| extend _ResourceId, WEC = extract('Machines|machines/(.*)',1,_ResourceId)\\r\\n| where TimeGenerated {TimeRange} and Computer == \\\"{vmname}\\\" or WEC == \\\"{vmname}\\\"\\r\\n| summarize count() by bin(TimeGenerated, 1d), Type\\r\\n| project Table=Type, TimeGenerated, count_\",\"size\":0,\"title\":\"{TimeRange:label}: Number of events in {Workspace:label} in for {vmname} in the SecurityEvent, Heartbeat, Perf or Event tables\",\"timeContextFromParameter\":\"TimeRange\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{Workspace}\"],\"visualization\":\"linechart\",\"chartSettings\":{\"showLegend\":true}},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"AMAvms\"},\"name\":\"LA volume by Table\"},{\"type\":3,\"content\":{\"version\":\"KqlItem/1.0\",\"query\":\"WindowsEvent\\r\\n| extend _ResourceId, WEC = extract('Machines|machines/(.*)',1,_ResourceId)\\r\\n| where TimeGenerated {TimeRange} and WEC == \\\"{vmname}\\\"\\r\\n| summarize count() by bin(TimeGenerated, 1d)\\r\\n| project TimeGenerated, count_\",\"size\":0,\"title\":\"{TimeRange:label}: Number of WEF events in {Workspace:label} forwarded by {vmname} to the WindowsEvent table\",\"timeContextFromParameter\":\"TimeRange\",\"queryType\":0,\"resourceType\":\"microsoft.operationalinsights/workspaces\",\"crossComponentResources\":[\"{Workspace}\"],\"visualization\":\"linechart\",\"chartSettings\":{\"showLegend\":true}},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"AMAvms\"},\"name\":\"WEF\"}]},\"conditionalVisibility\":{\"parameterName\":\"selectedTab\",\"comparison\":\"isEqualTo\",\"value\":\"AMAvms\"},\"name\":\"VMs and DCRA\"}],\"isLocked\":false,\"fallbackResourceIds\":[\"/subscriptions/419581d6-4853-49bd-83b6-d94bb8a77887/resourcegroups/sentinel-eco-test-nazang/providers/microsoft.operationalinsights/workspaces/sentinel-eco-test-nazang\"],\"fromTemplateId\":\"sentinel-AMAmigrationTracker\"}", + "version": "1.0", + "sourceId": "[parameters('workbookSourceId')]", + "category": "[parameters('workbookType')]" + } + } + ], + "outputs": { + "workbookId": { + "type": "string", + "value": "[resourceId( 'microsoft.insights/workbooks', parameters('workbookId'))]" + } + }, + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#" +} \ No newline at end of file diff --git a/sentinel-deployment.config b/sentinel-deployment.config index 88f3cb402..b79600995 100644 --- a/sentinel-deployment.config +++ b/sentinel-deployment.config @@ -4,10 +4,12 @@ ], "parameterfilemappings": { "40d1b346-fe1b-4503-8f8e-cfbbf4d7f4b3": { - "Workbooks/workbook_ADXvsLA.json": "Parameters/workspace-sentinel-eco-test-nazang-parameters.json" + "Workbooks/workbook_ADXvsLA.json": "Parameters/workspace-sentinel-eco-test-nazang-parameters.json", + "workbook_AMA_migration_tracker.json": "Parameters/workspace-sentinel-eco-test-nazang-parameters.json" }, "a2c3834b-49dd-410b-87aa-99f878346c91": { - "Workbooks/workbook_ADXvsLA.json": "Parameters/workspace-sentinel-eco-test-nazang2-parameters.json" + "Workbooks/workbook_ADXvsLA.json": "Parameters/workspace-sentinel-eco-test-nazang2-parameters.json", + "workbook_AMA_migration_tracker.json": "Parameters/workspace-sentinel-eco-test-nazang-parameters.json" } }, "DummySection": "This shouldn't impact deployment" From 9ee363cca63ffb1969b651d7da51c07e6a56950b Mon Sep 17 00:00:00 2001 From: "azure-sentinel-canary[bot]" <81647351+azure-sentinel-canary[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 21:48:19 +0000 Subject: [PATCH 5/8] Sentinel Content Deployment Script --- ...y-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 | 602 ++++++++++++++++++ 1 file changed, 602 insertions(+) create mode 100644 .github/workflows/azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 diff --git a/.github/workflows/azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 b/.github/workflows/azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 new file mode 100644 index 000000000..7aa9adf09 --- /dev/null +++ b/.github/workflows/azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 @@ -0,0 +1,602 @@ +## Globals ## +$CloudEnv = $Env:cloudEnv +$ResourceGroupName = $Env:resourceGroupName +$WorkspaceName = $Env:workspaceName +$WorkspaceId = $Env:workspaceId +$Directory = $Env:directory +$Creds = $Env:creds +$contentTypes = $Env:contentTypes +$contentTypeMapping = @{ + "AnalyticsRule"=@("Microsoft.OperationalInsights/workspaces/providers/alertRules", "Microsoft.OperationalInsights/workspaces/providers/alertRules/actions"); + "AutomationRule"=@("Microsoft.OperationalInsights/workspaces/providers/automationRules"); + "HuntingQuery"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); + "Parser"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); + "Playbook"=@("Microsoft.Web/connections", "Microsoft.Logic/workflows", "Microsoft.Web/customApis"); + "Workbook"=@("Microsoft.Insights/workbooks"); +} +$sourceControlId = $Env:sourceControlId +$rootDirectory = $Env:rootDirectory +$githubAuthToken = $Env:githubAuthToken +$githubRepository = $Env:GITHUB_REPOSITORY +$branchName = $Env:branch +$smartDeployment = $Env:smartDeployment +$newResourceBranch = $branchName + "-sentinel-deployment" +$csvPath = "$rootDirectory\.sentinel\tracking_table_$sourceControlId.csv" +$configPath = "$rootDirectory\sentinel-deployment.config" +$global:localCsvTablefinal = @{} +$global:updatedCsvTable = @{} +$global:parameterFileMapping = @{} +$global:prioritizedContentFiles = @() +$global:excludeContentFiles = @() + +$guidPattern = '(\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)' +$namePattern = '([-\w\._\(\)]+)' +$sentinelResourcePatterns = @{ + "AnalyticsRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/alertRules/$namePattern" + "AutomationRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/automationRules/$namePattern" + "HuntingQuery" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" + "Parser" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" + "Playbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Logic/workflows/$namePattern" + "Workbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Insights/workbooks/$namePattern" +} + +if ([string]::IsNullOrEmpty($contentTypes)) { + $contentTypes = "AnalyticsRule" +} + +$metadataFilePath = "metadata.json" +@" +{ + "`$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "parentResourceId": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "sourceControlId": { + "type": "string" + }, + "workspace": { + "type": "string" + }, + "contentId": { + "type": "string" + } + }, + "variables": { + "metadataName": "[concat(toLower(parameters('kind')), '-', parameters('contentId'))]" + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2022-01-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('metadataName'))]", + "properties": { + "parentId": "[parameters('parentResourceId')]", + "kind": "[parameters('kind')]", + "source": { + "kind": "SourceRepository", + "name": "Repositories", + "sourceId": "[parameters('sourceControlId')]" + } + } + } + ] +} +"@ | Out-File -FilePath $metadataFilePath + +$resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping[$_] } | ForEach-Object { $_.ToLower() } +$MaxRetries = 3 +$secondsBetweenAttempts = 5 + +#Converts hashtable to string that can be set as content when pushing csv file +function ConvertTableToString { + $output = "FileName, CommitSha`n" + $global:updatedCsvTable.GetEnumerator() | ForEach-Object { + $key = RelativePathWithBackslash $_.Key + $output += "{0},{1}`n" -f $key, $_.Value + } + return $output +} + +$header = @{ + "authorization" = "Bearer $githubAuthToken" +} + +#Gets all files and commit shas using Get Trees API +function GetGithubTree { + $branchResponse = AttemptInvokeRestMethod "Get" "https://api.github.com/repos/$githubRepository/branches/$branchName" $null $null 3 + $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" + $getTreeResponse = AttemptInvokeRestMethod "Get" $treeUrl $null $null 3 + return $getTreeResponse +} + +#Creates a table using the reponse from the tree api, creates a table +function GetCommitShaTable($getTreeResponse) { + $shaTable = @{} + $getTreeResponse.tree | ForEach-Object { + $truePath = AbsolutePathWithSlash $_.path + if (([System.IO.Path]::GetExtension($_.path) -eq ".json") -or ($truePath -eq $configPath)) + { + $shaTable.Add($truePath, $_.sha) + } + } + return $shaTable +} + +function PushCsvToRepo() { + $content = ConvertTableToString + $relativeCsvPath = RelativePathWithBackslash $csvPath + $resourceBranchExists = git ls-remote --heads "https://github.com/$githubRepository" $newResourceBranch | wc -l + + if ($resourceBranchExists -eq 0) { + git switch --orphan $newResourceBranch + git commit --allow-empty -m "Initial commit on orphan branch" + git push -u origin $newResourceBranch + New-Item -ItemType "directory" -Path ".sentinel" + } else { + git fetch > $null + git checkout $newResourceBranch + } + + Write-Output $content > $relativeCsvPath + git add $relativeCsvPath + git commit -m "Modified tracking table" + git push -u origin $newResourceBranch + git checkout $branchName +} + +function ReadCsvToTable { + $csvTable = Import-Csv -Path $csvPath + $HashTable=@{} + foreach($r in $csvTable) + { + $key = AbsolutePathWithSlash $r.FileName + $HashTable[$key]=$r.CommitSha + } + return $HashTable +} + +function AttemptInvokeRestMethod($method, $url, $body, $contentTypes, $maxRetries) { + $Stoploop = $false + $retryCount = 0 + do { + try { + $result = Invoke-RestMethod -Uri $url -Method $method -Headers $header -Body $body -ContentType $contentTypes + $Stoploop = $true + } + catch { + if ($retryCount -gt $maxRetries) { + Write-Host "[Error] API call failed after $retryCount retries: $_" + $Stoploop = $true + } + else { + Write-Host "[Warning] API call failed: $_.`n Conducting retry #$retryCount." + Start-Sleep -Seconds 5 + $retryCount = $retryCount + 1 + } + } + } + While ($Stoploop -eq $false) + return $result +} + +function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { + $maxLoginRetries = 3 + $delayInSeconds = 30 + $retryCount = 1 + $stopTrying = $false + do { + try { + Connect-AzAccount -ServicePrincipal -Tenant $tenantId -Credential $psCredential -Environment $cloudEnv | out-null; + Write-Host "Login Successful" + $stopTrying = $true + } + catch { + if ($retryCount -ge $maxLoginRetries) { + Write-Host "Login failed after $maxLoginRetries attempts." + $stopTrying = $true + } + else { + Write-Host "Login attempt failed, retrying in $delayInSeconds seconds." + Start-Sleep -Seconds $delayInSeconds + $retryCount++ + } + } + } + while (-not $stopTrying) +} + +function ConnectAzCloud { + $RawCreds = $Creds | ConvertFrom-Json + + Clear-AzContext -Scope Process; + Clear-AzContext -Scope CurrentUser -Force -ErrorAction SilentlyContinue; + + Add-AzEnvironment ` + -Name $CloudEnv ` + -ActiveDirectoryEndpoint $RawCreds.activeDirectoryEndpointUrl ` + -ResourceManagerEndpoint $RawCreds.resourceManagerEndpointUrl ` + -ActiveDirectoryServiceEndpointResourceId $RawCreds.activeDirectoryServiceEndpointResourceId ` + -GraphEndpoint $RawCreds.graphEndpointUrl | out-null; + + $servicePrincipalKey = ConvertTo-SecureString $RawCreds.clientSecret.replace("'", "''") -AsPlainText -Force + $psCredential = New-Object System.Management.Automation.PSCredential($RawCreds.clientId, $servicePrincipalKey) + + AttemptAzLogin $psCredential $RawCreds.tenantId $CloudEnv + Set-AzContext -Tenant $RawCreds.tenantId | out-null; +} + +function AttemptDeployMetadata($deploymentName, $resourceGroupName, $templateObject) { + $deploymentInfo = $null + try { + $deploymentInfo = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Ignore + } + catch { + Write-Host "[Warning] Unable to fetch deployment info for $deploymentName, no metadata was created for the resources in the file. Error: $_" + return + } + $deploymentInfo | Where-Object { $_.TargetResource -ne "" } | ForEach-Object { + $resource = $_.TargetResource + $sentinelContentKinds = GetContentKinds $resource + if ($sentinelContentKinds.Count -gt 0) { + $contentKind = ToContentKind $sentinelContentKinds $resource $templateObject + $contentId = $resource.Split("/")[-1] + try { + New-AzResourceGroupDeployment -Name "md-$deploymentName" -ResourceGroupName $ResourceGroupName -TemplateFile $metadataFilePath ` + -parentResourceId $resource ` + -kind $contentKind ` + -contentId $contentId ` + -sourceControlId $sourceControlId ` + -workspace $workspaceName ` + -ErrorAction Stop | Out-Host + Write-Host "[Info] Created metadata metadata for $contentKind with parent resource id $resource" + } + catch { + Write-Host "[Warning] Failed to deploy metadata for $contentKind with parent resource id $resource with error $_" + } + } + } +} + +function GetContentKinds($resource) { + return $sentinelResourcePatterns.Keys | Where-Object { $resource -match $sentinelResourcePatterns[$_] } +} + +function ToContentKind($contentKinds, $resource, $templateObject) { + if ($contentKinds.Count -eq 1) { + return $contentKinds + } + if ($null -ne $resource -and $resource.Contains('savedSearches')) { + if ($templateObject.resources.properties.Category -eq "Hunting Queries") { + return "HuntingQuery" + } + return "Parser" + } + return $null +} + +function IsValidTemplate($path, $templateObject) { + Try { + if (DoesContainWorkspaceParam $templateObject) { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $WorkspaceName + } + else { + Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path + } + + return $true + } + Catch { + Write-Host "[Warning] The file $path is not valid: $_" + return $false + } +} + +function IsRetryable($deploymentName) { + $retryableStatusCodes = "Conflict","TooManyRequests","InternalServerError","DeploymentActive" + Try { + $deploymentResult = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Stop + return $retryableStatusCodes -contains $deploymentResult.StatusCode + } + Catch { + return $false + } +} + +function IsValidResourceType($template) { + try { + $isAllowedResources = $true + $template.resources | ForEach-Object { + $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources + } + } + catch { + Write-Host "[Error] Failed to check valid resource type." + $isAllowedResources = $false + } + return $isAllowedResources +} + +function DoesContainWorkspaceParam($templateObject) { + $templateObject.parameters.PSobject.Properties.Name -contains "workspace" +} + +function AttemptDeployment($path, $parameterFile, $deploymentName, $templateObject) { + Write-Host "[Info] Deploying $path with deployment name $deploymentName" + + $isValid = IsValidTemplate $path $templateObject + if (-not $isValid) { + return $false + } + $isSuccess = $false + $currentAttempt = 0 + While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) + { + $currentAttempt ++ + Try + { + Write-Host "[Info] Deploy $path with parameter file: [$parameterFile]" + if (DoesContainWorkspaceParam $templateObject) + { + if ($parameterFile) { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host + } + else + { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -ErrorAction Stop | Out-Host + } + } + else + { + if ($parameterFile) { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host + } + else + { + New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -ErrorAction Stop | Out-Host + } + } + AttemptDeployMetadata $deploymentName $ResourceGroupName $templateObject + + $isSuccess = $true + } + Catch [Exception] + { + $err = $_ + if (-not (IsRetryable $deploymentName)) + { + Write-Host "[Warning] Failed to deploy $path with error: $err" + break + } + else + { + if ($currentAttempt -le $MaxRetries) + { + Write-Host "[Warning] Failed to deploy $path with error: $err. Retrying in $secondsBetweenAttempts seconds..." + Start-Sleep -Seconds $secondsBetweenAttempts + } + else + { + Write-Host "[Warning] Failed to deploy $path after $currentAttempt attempts with error: $err" + } + } + } + } + return $isSuccess +} + +function GenerateDeploymentName() { + $randomId = [guid]::NewGuid() + return "Sentinel_Deployment_$randomId" +} + +#Load deployment configuration +function LoadDeploymentConfig() { + Write-Host "[Info] load the deployment configuration from [$configPath]" + $global:parameterFileMapping = @{} + $global:prioritizedContentFiles = @() + $global:excludeContentFiles = @() + try { + if (Test-Path $configPath) { + $deployment_config = Get-Content $configPath | Out-String | ConvertFrom-Json + $parameterFileMappings = @{} + if ($deployment_config.parameterfilemappings) { + $deployment_config.parameterfilemappings.psobject.properties | ForEach { $parameterFileMappings[$_.Name] = $_.Value } + } + $key = ($parameterFileMappings.Keys | ? { $_ -eq $workspaceId }) + if ($null -ne $key) { + $parameterFileMappings[$key].psobject.properties | ForEach { $global:parameterFileMapping[$_.Name] = $_.Value } + } + if ($deployment_config.prioritizedcontentfiles) { + $global:prioritizedContentFiles = $deployment_config.prioritizedcontentfiles + } + $excludeList = $global:parameterFileMapping.Values + $global:prioritizedcontentfiles + if ($deployment_config.excludecontentfiles) { + $excludeList = $excludeList + $deployment_config.excludecontentfiles + } + $global:excludeContentFiles = $excludeList | Where-Object { Test-Path (AbsolutePathWithSlash $_) } + } + } + catch { + Write-Host "[Warning] An error occurred while trying to load deployment configuration." + Write-Host "Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function filterContentFile($fullPath) { + $temp = RelativePathWithBackslash $fullPath + return $global:excludeContentFiles | ? {$temp.StartsWith($_, 'CurrentCultureIgnoreCase')} +} + +function RelativePathWithBackslash($absolutePath) { + return $absolutePath.Replace($rootDirectory + "\", "").Replace("\", "/") +} + +function AbsolutePathWithSlash($relativePath) { + return Join-Path -Path $rootDirectory -ChildPath $relativePath +} + +#resolve parameter file name, return $null if there is none. +function GetParameterFile($path) { + $index = RelativePathWithBackslash $path + $key = ($global:parameterFileMapping.Keys | ? { $_ -eq $index }) + if ($key) { + $mappedParameterFile = AbsolutePathWithSlash $global:parameterFileMapping[$key] + if (Test-Path $mappedParameterFile) { + return $mappedParameterFile + } + } + + $parameterFilePrefix = $path.TrimEnd(".json") + + $workspaceParameterFile = $parameterFilePrefix + ".parameters-$WorkspaceId.json" + if (Test-Path $workspaceParameterFile) { + return $workspaceParameterFile + } + + $defaultParameterFile = $parameterFilePrefix + ".parameters.json" + if (Test-Path $defaultParameterFile) { + return $defaultParameterFile + } + + return $null +} + +function Deployment($fullDeploymentFlag, $remoteShaTable, $tree) { + Write-Host "Starting Deployment for Files in path: $Directory" + if (Test-Path -Path $Directory) + { + $totalFiles = 0; + $totalFailed = 0; + $iterationList = @() + $global:prioritizedContentFiles | ForEach-Object { $iterationList += (AbsolutePathWithSlash $_) } + Get-ChildItem -Path $Directory -Recurse -Filter *.json -exclude *metadata.json, *.parameters*.json | + Where-Object { $null -eq ( filterContentFile $_.FullName ) } | + Select-Object -Property FullName | + ForEach-Object { $iterationList += $_.FullName } + $iterationList | ForEach-Object { + $path = $_ + Write-Host "[Info] Try to deploy $path" + if (-not (Test-Path $path)) { + Write-Host "[Warning] Skipping deployment for $path. The file doesn't exist." + return + } + $templateObject = Get-Content $path | Out-String | ConvertFrom-Json + if (-not (IsValidResourceType $templateObject)) + { + Write-Host "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." + return + } + $parameterFile = GetParameterFile $path + $result = SmartDeployment $fullDeploymentFlag $remoteShaTable $path $parameterFile $templateObject + if ($result.isSuccess -eq $false) { + $totalFailed++ + } + if (-not $result.skip) { + $totalFiles++ + } + if ($result.isSuccess -or $result.skip) { + $global:updatedCsvTable[$path] = $remoteShaTable[$path] + if ($parameterFile) { + $global:updatedCsvTable[$parameterFile] = $remoteShaTable[$parameterFile] + } + } + } + PushCsvToRepo + if ($totalFiles -gt 0 -and $totalFailed -gt 0) + { + $err = "$totalFailed of $totalFiles deployments failed." + Throw $err + } + } + else + { + Write-Output "[Warning] $Directory not found. nothing to deploy" + } +} + +function SmartDeployment($fullDeploymentFlag, $remoteShaTable, $path, $parameterFile, $templateObject) { + try { + $skip = $false + $isSuccess = $null + if (!$fullDeploymentFlag) { + $existingSha = $global:localCsvTablefinal[$path] + $remoteSha = $remoteShaTable[$path] + $skip = (($existingSha) -and ($existingSha -eq $remoteSha)) + if ($skip -and $parameterFile) { + $existingShaForParameterFile = $global:localCsvTablefinal[$parameterFile] + $remoteShaForParameterFile = $remoteShaTable[$parameterFile] + $skip = (($existingShaForParameterFile) -and ($existingShaForParameterFile -eq $remoteShaForParameterFile)) + } + } + if (!$skip) { + $deploymentName = GenerateDeploymentName + $isSuccess = AttemptDeployment $path $parameterFile $deploymentName $templateObject + } + return @{ + skip = $skip + isSuccess = $isSuccess + } + } + catch { + Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" + Write-Host $_.ScriptStackTrace + } +} + +function TryGetCsvFile { + if (Test-Path $csvPath) { + $global:localCsvTablefinal = ReadCsvToTable + Remove-Item -Path $csvPath + git add $csvPath + git commit -m "Removed tracking file and moved to new sentinel created branch" + git push origin $branchName + } + + $relativeCsvPath = RelativePathWithBackslash $csvPath + $resourceBranchExists = git ls-remote --heads "https://github.com/$githubRepository" $newResourceBranch | wc -l + + if ($resourceBranchExists -eq 1) { + git fetch > $null + git checkout $newResourceBranch + + if (Test-Path $relativeCsvPath) { + $global:localCsvTablefinal = ReadCsvToTable + } + git checkout $branchName + } +} + +function main() { + git config --global user.email "donotreply@microsoft.com" + git config --global user.name "Sentinel" + + if ($CloudEnv -ne 'AzureCloud') + { + Write-Output "Attempting Sign In to Azure Cloud" + ConnectAzCloud + } + + TryGetCsvFile + LoadDeploymentConfig + $tree = GetGithubTree + $remoteShaTable = GetCommitShaTable $tree + + $existingConfigSha = $global:localCsvTablefinal[$configPath] + $remoteConfigSha = $remoteShaTable[$configPath] + $modifiedConfig = ($existingConfigSha -xor $remoteConfigSha) -or ($existingConfigSha -and $remoteConfigSha -and ($existingConfigSha -ne $remoteConfigSha)) + + if ($remoteConfigSha) { + $global:updatedCsvTable[$configPath] = $remoteConfigSha + } + + $fullDeploymentFlag = $modifiedConfig -or ($smartDeployment -eq "false") + Deployment $fullDeploymentFlag $remoteShaTable $tree +} + +main \ No newline at end of file From 641168f67bb3721977b84ce6bc1afb94aedd0b15 Mon Sep 17 00:00:00 2001 From: "azure-sentinel-canary[bot]" <81647351+azure-sentinel-canary[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 21:48:20 +0000 Subject: [PATCH 6/8] Workflow file for Sentinel-Deploy --- ...y-95880f15-991d-4939-bb1c-67fa26d2196c.yml | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml diff --git a/.github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml b/.github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml new file mode 100644 index 000000000..e20c2f288 --- /dev/null +++ b/.github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml @@ -0,0 +1,81 @@ +name: Deploy Content to nazang-ch-test1 [95880f15-991d-4939-bb1c-67fa26d2196c] +# Note: This workflow will deploy everything in the root directory. +# To deploy content only from a specific path (for example SentinelContent): +# 1. Add the target path to the "paths" property like such +# paths: +# - 'SentinelContent/**' +# - '!.github/workflows/**' +# - '.github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml' +# 2. Append the path to the directory environment variable below +# directory: '${{ github.workspace }}/SentinelContent' + +on: + push: + branches: [ ContentDevelopmentBranch ] + paths: + - '**' + - '!.github/workflows/**' # this filter prevents other workflow changes from triggering this workflow + - '.github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml' + +jobs: + deploy-content: + runs-on: windows-latest + env: + resourceGroupName: 'nazang-ch-test1' + workspaceName: 'nazang-ch-test1' + workspaceId: 'f0b96d37-a773-4624-907f-574f093ab71c' + directory: '${{ github.workspace }}' + cloudEnv: 'AzureCloud' + creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_95880f15991d4939bb1c67fa26d2196c }} + contentTypes: 'AnalyticsRule,AutomationRule,Playbook,Workbook,Parser,HuntingQuery' + branch: 'ContentDevelopmentBranch' + sourceControlId: '95880f15-991d-4939-bb1c-67fa26d2196c' + rootDirectory: '${{ github.workspace }}' + githubAuthToken: ${{ secrets.GITHUB_TOKEN }} + smartDeployment: 'true' + + steps: + - name: Login to Azure (Attempt 1) + continue-on-error: true + id: login1 + uses: azure/login@v1 + if: ${{ env.cloudEnv == 'AzureCloud' }} + with: + creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_95880f15991d4939bb1c67fa26d2196c }} + enable-AzPSSession: true + + - name: Wait 30 seconds if login attempt 1 failed + if: ${{ env.cloudEnv == 'AzureCloud' && steps.login1.outcome=='failure' }} + run: powershell Start-Sleep -s 30 + + - name: Login to Azure (Attempt 2) + continue-on-error: true + id: login2 + uses: azure/login@v1 + if: ${{ env.cloudEnv == 'AzureCloud' && steps.login1.outcome=='failure' }} + with: + creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_95880f15991d4939bb1c67fa26d2196c }} + enable-AzPSSession: true + + - name: Wait 30 seconds if login attempt 2 failed + if: ${{ env.cloudEnv == 'AzureCloud' && steps.login2.outcome=='failure' }} + run: powershell Start-Sleep -s 30 + + - name: Login to Azure (Attempt 3) + continue-on-error: false + id: login3 + uses: azure/login@v1 + if: ${{ env.cloudEnv == 'AzureCloud' && steps.login2.outcome=='failure' }} + with: + creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_95880f15991d4939bb1c67fa26d2196c }} + enable-AzPSSession: true + + - name: Checkout + uses: actions/checkout@v3 + + - name: Deploy Content to Azure Sentinel + uses: azure/powershell@v1 + with: + azPSVersion: 'latest' + inlineScript: | + ${{ github.workspace }}//.github/workflows/azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 \ No newline at end of file From 23d1f2f18c66d5ed164fa3338d9c68b2c8067968 Mon Sep 17 00:00:00 2001 From: "azure-sentinel-canary[bot]" <81647351+azure-sentinel-canary[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 22:00:31 +0000 Subject: [PATCH 7/8] Remove sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml --- ...y-95880f15-991d-4939-bb1c-67fa26d2196c.yml | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 .github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml diff --git a/.github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml b/.github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml deleted file mode 100644 index e20c2f288..000000000 --- a/.github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Deploy Content to nazang-ch-test1 [95880f15-991d-4939-bb1c-67fa26d2196c] -# Note: This workflow will deploy everything in the root directory. -# To deploy content only from a specific path (for example SentinelContent): -# 1. Add the target path to the "paths" property like such -# paths: -# - 'SentinelContent/**' -# - '!.github/workflows/**' -# - '.github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml' -# 2. Append the path to the directory environment variable below -# directory: '${{ github.workspace }}/SentinelContent' - -on: - push: - branches: [ ContentDevelopmentBranch ] - paths: - - '**' - - '!.github/workflows/**' # this filter prevents other workflow changes from triggering this workflow - - '.github/workflows/sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.yml' - -jobs: - deploy-content: - runs-on: windows-latest - env: - resourceGroupName: 'nazang-ch-test1' - workspaceName: 'nazang-ch-test1' - workspaceId: 'f0b96d37-a773-4624-907f-574f093ab71c' - directory: '${{ github.workspace }}' - cloudEnv: 'AzureCloud' - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_95880f15991d4939bb1c67fa26d2196c }} - contentTypes: 'AnalyticsRule,AutomationRule,Playbook,Workbook,Parser,HuntingQuery' - branch: 'ContentDevelopmentBranch' - sourceControlId: '95880f15-991d-4939-bb1c-67fa26d2196c' - rootDirectory: '${{ github.workspace }}' - githubAuthToken: ${{ secrets.GITHUB_TOKEN }} - smartDeployment: 'true' - - steps: - - name: Login to Azure (Attempt 1) - continue-on-error: true - id: login1 - uses: azure/login@v1 - if: ${{ env.cloudEnv == 'AzureCloud' }} - with: - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_95880f15991d4939bb1c67fa26d2196c }} - enable-AzPSSession: true - - - name: Wait 30 seconds if login attempt 1 failed - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login1.outcome=='failure' }} - run: powershell Start-Sleep -s 30 - - - name: Login to Azure (Attempt 2) - continue-on-error: true - id: login2 - uses: azure/login@v1 - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login1.outcome=='failure' }} - with: - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_95880f15991d4939bb1c67fa26d2196c }} - enable-AzPSSession: true - - - name: Wait 30 seconds if login attempt 2 failed - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login2.outcome=='failure' }} - run: powershell Start-Sleep -s 30 - - - name: Login to Azure (Attempt 3) - continue-on-error: false - id: login3 - uses: azure/login@v1 - if: ${{ env.cloudEnv == 'AzureCloud' && steps.login2.outcome=='failure' }} - with: - creds: ${{ secrets.AZURE_SENTINEL_CREDENTIALS_95880f15991d4939bb1c67fa26d2196c }} - enable-AzPSSession: true - - - name: Checkout - uses: actions/checkout@v3 - - - name: Deploy Content to Azure Sentinel - uses: azure/powershell@v1 - with: - azPSVersion: 'latest' - inlineScript: | - ${{ github.workspace }}//.github/workflows/azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 \ No newline at end of file From 6bfabff734034dd125387cfcf2f4acf728a154f5 Mon Sep 17 00:00:00 2001 From: "azure-sentinel-canary[bot]" <81647351+azure-sentinel-canary[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 22:00:31 +0000 Subject: [PATCH 8/8] Remove azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 --- ...y-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 | 602 ------------------ 1 file changed, 602 deletions(-) delete mode 100644 .github/workflows/azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 diff --git a/.github/workflows/azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 b/.github/workflows/azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 deleted file mode 100644 index 7aa9adf09..000000000 --- a/.github/workflows/azure-sentinel-deploy-95880f15-991d-4939-bb1c-67fa26d2196c.ps1 +++ /dev/null @@ -1,602 +0,0 @@ -## Globals ## -$CloudEnv = $Env:cloudEnv -$ResourceGroupName = $Env:resourceGroupName -$WorkspaceName = $Env:workspaceName -$WorkspaceId = $Env:workspaceId -$Directory = $Env:directory -$Creds = $Env:creds -$contentTypes = $Env:contentTypes -$contentTypeMapping = @{ - "AnalyticsRule"=@("Microsoft.OperationalInsights/workspaces/providers/alertRules", "Microsoft.OperationalInsights/workspaces/providers/alertRules/actions"); - "AutomationRule"=@("Microsoft.OperationalInsights/workspaces/providers/automationRules"); - "HuntingQuery"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); - "Parser"=@("Microsoft.OperationalInsights/workspaces/savedSearches"); - "Playbook"=@("Microsoft.Web/connections", "Microsoft.Logic/workflows", "Microsoft.Web/customApis"); - "Workbook"=@("Microsoft.Insights/workbooks"); -} -$sourceControlId = $Env:sourceControlId -$rootDirectory = $Env:rootDirectory -$githubAuthToken = $Env:githubAuthToken -$githubRepository = $Env:GITHUB_REPOSITORY -$branchName = $Env:branch -$smartDeployment = $Env:smartDeployment -$newResourceBranch = $branchName + "-sentinel-deployment" -$csvPath = "$rootDirectory\.sentinel\tracking_table_$sourceControlId.csv" -$configPath = "$rootDirectory\sentinel-deployment.config" -$global:localCsvTablefinal = @{} -$global:updatedCsvTable = @{} -$global:parameterFileMapping = @{} -$global:prioritizedContentFiles = @() -$global:excludeContentFiles = @() - -$guidPattern = '(\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)' -$namePattern = '([-\w\._\(\)]+)' -$sentinelResourcePatterns = @{ - "AnalyticsRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/alertRules/$namePattern" - "AutomationRule" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/providers/Microsoft.SecurityInsights/automationRules/$namePattern" - "HuntingQuery" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" - "Parser" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.OperationalInsights/workspaces/$namePattern/savedSearches/$namePattern" - "Playbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Logic/workflows/$namePattern" - "Workbook" = "/subscriptions/$guidPattern/resourceGroups/$namePattern/providers/Microsoft.Insights/workbooks/$namePattern" -} - -if ([string]::IsNullOrEmpty($contentTypes)) { - $contentTypes = "AnalyticsRule" -} - -$metadataFilePath = "metadata.json" -@" -{ - "`$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "parentResourceId": { - "type": "string" - }, - "kind": { - "type": "string" - }, - "sourceControlId": { - "type": "string" - }, - "workspace": { - "type": "string" - }, - "contentId": { - "type": "string" - } - }, - "variables": { - "metadataName": "[concat(toLower(parameters('kind')), '-', parameters('contentId'))]" - }, - "resources": [ - { - "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", - "apiVersion": "2022-01-01-preview", - "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('metadataName'))]", - "properties": { - "parentId": "[parameters('parentResourceId')]", - "kind": "[parameters('kind')]", - "source": { - "kind": "SourceRepository", - "name": "Repositories", - "sourceId": "[parameters('sourceControlId')]" - } - } - } - ] -} -"@ | Out-File -FilePath $metadataFilePath - -$resourceTypes = $contentTypes.Split(",") | ForEach-Object { $contentTypeMapping[$_] } | ForEach-Object { $_.ToLower() } -$MaxRetries = 3 -$secondsBetweenAttempts = 5 - -#Converts hashtable to string that can be set as content when pushing csv file -function ConvertTableToString { - $output = "FileName, CommitSha`n" - $global:updatedCsvTable.GetEnumerator() | ForEach-Object { - $key = RelativePathWithBackslash $_.Key - $output += "{0},{1}`n" -f $key, $_.Value - } - return $output -} - -$header = @{ - "authorization" = "Bearer $githubAuthToken" -} - -#Gets all files and commit shas using Get Trees API -function GetGithubTree { - $branchResponse = AttemptInvokeRestMethod "Get" "https://api.github.com/repos/$githubRepository/branches/$branchName" $null $null 3 - $treeUrl = "https://api.github.com/repos/$githubRepository/git/trees/" + $branchResponse.commit.sha + "?recursive=true" - $getTreeResponse = AttemptInvokeRestMethod "Get" $treeUrl $null $null 3 - return $getTreeResponse -} - -#Creates a table using the reponse from the tree api, creates a table -function GetCommitShaTable($getTreeResponse) { - $shaTable = @{} - $getTreeResponse.tree | ForEach-Object { - $truePath = AbsolutePathWithSlash $_.path - if (([System.IO.Path]::GetExtension($_.path) -eq ".json") -or ($truePath -eq $configPath)) - { - $shaTable.Add($truePath, $_.sha) - } - } - return $shaTable -} - -function PushCsvToRepo() { - $content = ConvertTableToString - $relativeCsvPath = RelativePathWithBackslash $csvPath - $resourceBranchExists = git ls-remote --heads "https://github.com/$githubRepository" $newResourceBranch | wc -l - - if ($resourceBranchExists -eq 0) { - git switch --orphan $newResourceBranch - git commit --allow-empty -m "Initial commit on orphan branch" - git push -u origin $newResourceBranch - New-Item -ItemType "directory" -Path ".sentinel" - } else { - git fetch > $null - git checkout $newResourceBranch - } - - Write-Output $content > $relativeCsvPath - git add $relativeCsvPath - git commit -m "Modified tracking table" - git push -u origin $newResourceBranch - git checkout $branchName -} - -function ReadCsvToTable { - $csvTable = Import-Csv -Path $csvPath - $HashTable=@{} - foreach($r in $csvTable) - { - $key = AbsolutePathWithSlash $r.FileName - $HashTable[$key]=$r.CommitSha - } - return $HashTable -} - -function AttemptInvokeRestMethod($method, $url, $body, $contentTypes, $maxRetries) { - $Stoploop = $false - $retryCount = 0 - do { - try { - $result = Invoke-RestMethod -Uri $url -Method $method -Headers $header -Body $body -ContentType $contentTypes - $Stoploop = $true - } - catch { - if ($retryCount -gt $maxRetries) { - Write-Host "[Error] API call failed after $retryCount retries: $_" - $Stoploop = $true - } - else { - Write-Host "[Warning] API call failed: $_.`n Conducting retry #$retryCount." - Start-Sleep -Seconds 5 - $retryCount = $retryCount + 1 - } - } - } - While ($Stoploop -eq $false) - return $result -} - -function AttemptAzLogin($psCredential, $tenantId, $cloudEnv) { - $maxLoginRetries = 3 - $delayInSeconds = 30 - $retryCount = 1 - $stopTrying = $false - do { - try { - Connect-AzAccount -ServicePrincipal -Tenant $tenantId -Credential $psCredential -Environment $cloudEnv | out-null; - Write-Host "Login Successful" - $stopTrying = $true - } - catch { - if ($retryCount -ge $maxLoginRetries) { - Write-Host "Login failed after $maxLoginRetries attempts." - $stopTrying = $true - } - else { - Write-Host "Login attempt failed, retrying in $delayInSeconds seconds." - Start-Sleep -Seconds $delayInSeconds - $retryCount++ - } - } - } - while (-not $stopTrying) -} - -function ConnectAzCloud { - $RawCreds = $Creds | ConvertFrom-Json - - Clear-AzContext -Scope Process; - Clear-AzContext -Scope CurrentUser -Force -ErrorAction SilentlyContinue; - - Add-AzEnvironment ` - -Name $CloudEnv ` - -ActiveDirectoryEndpoint $RawCreds.activeDirectoryEndpointUrl ` - -ResourceManagerEndpoint $RawCreds.resourceManagerEndpointUrl ` - -ActiveDirectoryServiceEndpointResourceId $RawCreds.activeDirectoryServiceEndpointResourceId ` - -GraphEndpoint $RawCreds.graphEndpointUrl | out-null; - - $servicePrincipalKey = ConvertTo-SecureString $RawCreds.clientSecret.replace("'", "''") -AsPlainText -Force - $psCredential = New-Object System.Management.Automation.PSCredential($RawCreds.clientId, $servicePrincipalKey) - - AttemptAzLogin $psCredential $RawCreds.tenantId $CloudEnv - Set-AzContext -Tenant $RawCreds.tenantId | out-null; -} - -function AttemptDeployMetadata($deploymentName, $resourceGroupName, $templateObject) { - $deploymentInfo = $null - try { - $deploymentInfo = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Ignore - } - catch { - Write-Host "[Warning] Unable to fetch deployment info for $deploymentName, no metadata was created for the resources in the file. Error: $_" - return - } - $deploymentInfo | Where-Object { $_.TargetResource -ne "" } | ForEach-Object { - $resource = $_.TargetResource - $sentinelContentKinds = GetContentKinds $resource - if ($sentinelContentKinds.Count -gt 0) { - $contentKind = ToContentKind $sentinelContentKinds $resource $templateObject - $contentId = $resource.Split("/")[-1] - try { - New-AzResourceGroupDeployment -Name "md-$deploymentName" -ResourceGroupName $ResourceGroupName -TemplateFile $metadataFilePath ` - -parentResourceId $resource ` - -kind $contentKind ` - -contentId $contentId ` - -sourceControlId $sourceControlId ` - -workspace $workspaceName ` - -ErrorAction Stop | Out-Host - Write-Host "[Info] Created metadata metadata for $contentKind with parent resource id $resource" - } - catch { - Write-Host "[Warning] Failed to deploy metadata for $contentKind with parent resource id $resource with error $_" - } - } - } -} - -function GetContentKinds($resource) { - return $sentinelResourcePatterns.Keys | Where-Object { $resource -match $sentinelResourcePatterns[$_] } -} - -function ToContentKind($contentKinds, $resource, $templateObject) { - if ($contentKinds.Count -eq 1) { - return $contentKinds - } - if ($null -ne $resource -and $resource.Contains('savedSearches')) { - if ($templateObject.resources.properties.Category -eq "Hunting Queries") { - return "HuntingQuery" - } - return "Parser" - } - return $null -} - -function IsValidTemplate($path, $templateObject) { - Try { - if (DoesContainWorkspaceParam $templateObject) { - Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $WorkspaceName - } - else { - Test-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -TemplateFile $path - } - - return $true - } - Catch { - Write-Host "[Warning] The file $path is not valid: $_" - return $false - } -} - -function IsRetryable($deploymentName) { - $retryableStatusCodes = "Conflict","TooManyRequests","InternalServerError","DeploymentActive" - Try { - $deploymentResult = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName -ErrorAction Stop - return $retryableStatusCodes -contains $deploymentResult.StatusCode - } - Catch { - return $false - } -} - -function IsValidResourceType($template) { - try { - $isAllowedResources = $true - $template.resources | ForEach-Object { - $isAllowedResources = $resourceTypes.contains($_.type.ToLower()) -and $isAllowedResources - } - } - catch { - Write-Host "[Error] Failed to check valid resource type." - $isAllowedResources = $false - } - return $isAllowedResources -} - -function DoesContainWorkspaceParam($templateObject) { - $templateObject.parameters.PSobject.Properties.Name -contains "workspace" -} - -function AttemptDeployment($path, $parameterFile, $deploymentName, $templateObject) { - Write-Host "[Info] Deploying $path with deployment name $deploymentName" - - $isValid = IsValidTemplate $path $templateObject - if (-not $isValid) { - return $false - } - $isSuccess = $false - $currentAttempt = 0 - While (($currentAttempt -lt $MaxRetries) -and (-not $isSuccess)) - { - $currentAttempt ++ - Try - { - Write-Host "[Info] Deploy $path with parameter file: [$parameterFile]" - if (DoesContainWorkspaceParam $templateObject) - { - if ($parameterFile) { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host - } - else - { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -workspace $workspaceName -ErrorAction Stop | Out-Host - } - } - else - { - if ($parameterFile) { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -TemplateParameterFile $parameterFile -ErrorAction Stop | Out-Host - } - else - { - New-AzResourceGroupDeployment -Name $deploymentName -ResourceGroupName $ResourceGroupName -TemplateFile $path -ErrorAction Stop | Out-Host - } - } - AttemptDeployMetadata $deploymentName $ResourceGroupName $templateObject - - $isSuccess = $true - } - Catch [Exception] - { - $err = $_ - if (-not (IsRetryable $deploymentName)) - { - Write-Host "[Warning] Failed to deploy $path with error: $err" - break - } - else - { - if ($currentAttempt -le $MaxRetries) - { - Write-Host "[Warning] Failed to deploy $path with error: $err. Retrying in $secondsBetweenAttempts seconds..." - Start-Sleep -Seconds $secondsBetweenAttempts - } - else - { - Write-Host "[Warning] Failed to deploy $path after $currentAttempt attempts with error: $err" - } - } - } - } - return $isSuccess -} - -function GenerateDeploymentName() { - $randomId = [guid]::NewGuid() - return "Sentinel_Deployment_$randomId" -} - -#Load deployment configuration -function LoadDeploymentConfig() { - Write-Host "[Info] load the deployment configuration from [$configPath]" - $global:parameterFileMapping = @{} - $global:prioritizedContentFiles = @() - $global:excludeContentFiles = @() - try { - if (Test-Path $configPath) { - $deployment_config = Get-Content $configPath | Out-String | ConvertFrom-Json - $parameterFileMappings = @{} - if ($deployment_config.parameterfilemappings) { - $deployment_config.parameterfilemappings.psobject.properties | ForEach { $parameterFileMappings[$_.Name] = $_.Value } - } - $key = ($parameterFileMappings.Keys | ? { $_ -eq $workspaceId }) - if ($null -ne $key) { - $parameterFileMappings[$key].psobject.properties | ForEach { $global:parameterFileMapping[$_.Name] = $_.Value } - } - if ($deployment_config.prioritizedcontentfiles) { - $global:prioritizedContentFiles = $deployment_config.prioritizedcontentfiles - } - $excludeList = $global:parameterFileMapping.Values + $global:prioritizedcontentfiles - if ($deployment_config.excludecontentfiles) { - $excludeList = $excludeList + $deployment_config.excludecontentfiles - } - $global:excludeContentFiles = $excludeList | Where-Object { Test-Path (AbsolutePathWithSlash $_) } - } - } - catch { - Write-Host "[Warning] An error occurred while trying to load deployment configuration." - Write-Host "Exception details: $_" - Write-Host $_.ScriptStackTrace - } -} - -function filterContentFile($fullPath) { - $temp = RelativePathWithBackslash $fullPath - return $global:excludeContentFiles | ? {$temp.StartsWith($_, 'CurrentCultureIgnoreCase')} -} - -function RelativePathWithBackslash($absolutePath) { - return $absolutePath.Replace($rootDirectory + "\", "").Replace("\", "/") -} - -function AbsolutePathWithSlash($relativePath) { - return Join-Path -Path $rootDirectory -ChildPath $relativePath -} - -#resolve parameter file name, return $null if there is none. -function GetParameterFile($path) { - $index = RelativePathWithBackslash $path - $key = ($global:parameterFileMapping.Keys | ? { $_ -eq $index }) - if ($key) { - $mappedParameterFile = AbsolutePathWithSlash $global:parameterFileMapping[$key] - if (Test-Path $mappedParameterFile) { - return $mappedParameterFile - } - } - - $parameterFilePrefix = $path.TrimEnd(".json") - - $workspaceParameterFile = $parameterFilePrefix + ".parameters-$WorkspaceId.json" - if (Test-Path $workspaceParameterFile) { - return $workspaceParameterFile - } - - $defaultParameterFile = $parameterFilePrefix + ".parameters.json" - if (Test-Path $defaultParameterFile) { - return $defaultParameterFile - } - - return $null -} - -function Deployment($fullDeploymentFlag, $remoteShaTable, $tree) { - Write-Host "Starting Deployment for Files in path: $Directory" - if (Test-Path -Path $Directory) - { - $totalFiles = 0; - $totalFailed = 0; - $iterationList = @() - $global:prioritizedContentFiles | ForEach-Object { $iterationList += (AbsolutePathWithSlash $_) } - Get-ChildItem -Path $Directory -Recurse -Filter *.json -exclude *metadata.json, *.parameters*.json | - Where-Object { $null -eq ( filterContentFile $_.FullName ) } | - Select-Object -Property FullName | - ForEach-Object { $iterationList += $_.FullName } - $iterationList | ForEach-Object { - $path = $_ - Write-Host "[Info] Try to deploy $path" - if (-not (Test-Path $path)) { - Write-Host "[Warning] Skipping deployment for $path. The file doesn't exist." - return - } - $templateObject = Get-Content $path | Out-String | ConvertFrom-Json - if (-not (IsValidResourceType $templateObject)) - { - Write-Host "[Warning] Skipping deployment for $path. The file contains resources for content that was not selected for deployment. Please add content type to connection if you want this file to be deployed." - return - } - $parameterFile = GetParameterFile $path - $result = SmartDeployment $fullDeploymentFlag $remoteShaTable $path $parameterFile $templateObject - if ($result.isSuccess -eq $false) { - $totalFailed++ - } - if (-not $result.skip) { - $totalFiles++ - } - if ($result.isSuccess -or $result.skip) { - $global:updatedCsvTable[$path] = $remoteShaTable[$path] - if ($parameterFile) { - $global:updatedCsvTable[$parameterFile] = $remoteShaTable[$parameterFile] - } - } - } - PushCsvToRepo - if ($totalFiles -gt 0 -and $totalFailed -gt 0) - { - $err = "$totalFailed of $totalFiles deployments failed." - Throw $err - } - } - else - { - Write-Output "[Warning] $Directory not found. nothing to deploy" - } -} - -function SmartDeployment($fullDeploymentFlag, $remoteShaTable, $path, $parameterFile, $templateObject) { - try { - $skip = $false - $isSuccess = $null - if (!$fullDeploymentFlag) { - $existingSha = $global:localCsvTablefinal[$path] - $remoteSha = $remoteShaTable[$path] - $skip = (($existingSha) -and ($existingSha -eq $remoteSha)) - if ($skip -and $parameterFile) { - $existingShaForParameterFile = $global:localCsvTablefinal[$parameterFile] - $remoteShaForParameterFile = $remoteShaTable[$parameterFile] - $skip = (($existingShaForParameterFile) -and ($existingShaForParameterFile -eq $remoteShaForParameterFile)) - } - } - if (!$skip) { - $deploymentName = GenerateDeploymentName - $isSuccess = AttemptDeployment $path $parameterFile $deploymentName $templateObject - } - return @{ - skip = $skip - isSuccess = $isSuccess - } - } - catch { - Write-Host "[Error] An error occurred while trying to deploy file $path. Exception details: $_" - Write-Host $_.ScriptStackTrace - } -} - -function TryGetCsvFile { - if (Test-Path $csvPath) { - $global:localCsvTablefinal = ReadCsvToTable - Remove-Item -Path $csvPath - git add $csvPath - git commit -m "Removed tracking file and moved to new sentinel created branch" - git push origin $branchName - } - - $relativeCsvPath = RelativePathWithBackslash $csvPath - $resourceBranchExists = git ls-remote --heads "https://github.com/$githubRepository" $newResourceBranch | wc -l - - if ($resourceBranchExists -eq 1) { - git fetch > $null - git checkout $newResourceBranch - - if (Test-Path $relativeCsvPath) { - $global:localCsvTablefinal = ReadCsvToTable - } - git checkout $branchName - } -} - -function main() { - git config --global user.email "donotreply@microsoft.com" - git config --global user.name "Sentinel" - - if ($CloudEnv -ne 'AzureCloud') - { - Write-Output "Attempting Sign In to Azure Cloud" - ConnectAzCloud - } - - TryGetCsvFile - LoadDeploymentConfig - $tree = GetGithubTree - $remoteShaTable = GetCommitShaTable $tree - - $existingConfigSha = $global:localCsvTablefinal[$configPath] - $remoteConfigSha = $remoteShaTable[$configPath] - $modifiedConfig = ($existingConfigSha -xor $remoteConfigSha) -or ($existingConfigSha -and $remoteConfigSha -and ($existingConfigSha -ne $remoteConfigSha)) - - if ($remoteConfigSha) { - $global:updatedCsvTable[$configPath] = $remoteConfigSha - } - - $fullDeploymentFlag = $modifiedConfig -or ($smartDeployment -eq "false") - Deployment $fullDeploymentFlag $remoteShaTable $tree -} - -main \ No newline at end of file