From a573decf7a0b654a5632dc12c1b9019ce5c6b281 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 25 Feb 2026 14:30:59 -0800 Subject: [PATCH 1/6] verify release hashes this is a throw away script but I dont want customers to hit this --- .vsts-dnup-pr.yml | 106 ++++++ .../releases/Verify-ReleaseHashes.ps1 | 307 ++++++++++++++++++ 2 files changed, 413 insertions(+) create mode 100644 src/Installer/releases/Verify-ReleaseHashes.ps1 diff --git a/.vsts-dnup-pr.yml b/.vsts-dnup-pr.yml index dc0a73026a5b..625c86b55068 100644 --- a/.vsts-dnup-pr.yml +++ b/.vsts-dnup-pr.yml @@ -8,6 +8,7 @@ pr: - dnup - release/dnup - release/dotnetup + - nagilson-dnup-executables parameters: - name: enableArm64Job @@ -65,4 +66,109 @@ stages: emoji: 💪 helixTargetQueue: osx.13.arm64.open +- stage: executables + displayName: 🏗️ Build dotnetup executables + dependsOn: [] + jobs: + ### Build a single platform executable for PR validation ### + - template: /eng/pipelines/templates/jobs/dotnetup/dotnetup-executables.yml@self + parameters: + rid: linux-x64 + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals build.ubuntu.2204.amd64.open + os: linux + emoji: 🐧 + +- stage: hash_verification + displayName: 🔐 Verify release manifest hashes + dependsOn: [] + jobs: + - template: /eng/common/templates/job/job.yml + parameters: + displayName: '🔐 Verify 7.0 release hashes' + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64.open + os: windows + helixRepo: dotnet/sdk + timeoutInMinutes: 360 + enableTelemetry: true + steps: + - powershell: | + New-Item -ItemType Directory -Force -Path '$(Build.SourcesDirectory)/src/Installer/releases/7.0' | Out-Null + try { & '$(Build.SourcesDirectory)/src/Installer/releases/Verify-ReleaseHashes.ps1' -ChannelVersion '7.0' -OutputDir '$(Build.SourcesDirectory)/src/Installer/releases/7.0' } catch { Write-Host $_ } + Write-Host "Script exited with code: $LASTEXITCODE" + displayName: '🔐 Verify 7.0 hashes' + - task: PublishPipelineArtifact@1 + displayName: '📦 Publish 7.0 results' + condition: always() + inputs: + artifactName: 'hash-verification-7.0' + - template: /eng/common/templates/job/job.yml + parameters: + displayName: '🔐 Verify 8.0 release hashes' + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64.open + os: windows + helixRepo: dotnet/sdk + timeoutInMinutes: 360 + enableTelemetry: true + steps: + - powershell: | + New-Item -ItemType Directory -Force -Path '$(Build.SourcesDirectory)/src/Installer/releases/8.0' | Out-Null + try { & '$(Build.SourcesDirectory)/src/Installer/releases/Verify-ReleaseHashes.ps1' -ChannelVersion '8.0' -OutputDir '$(Build.SourcesDirectory)/src/Installer/releases/8.0' } catch { Write-Host $_ } + Write-Host "Script exited with code: $LASTEXITCODE" + displayName: '🔐 Verify 8.0 hashes' + - task: PublishPipelineArtifact@1 + displayName: '📦 Publish 8.0 results' + condition: always() + inputs: + targetPath: '$(Build.SourcesDirectory)/src/Installer/releases/8.0' + artifactName: 'hash-verification-8.0' + - template: /eng/common/templates/job/job.yml + parameters: + displayName: '🔐 Verify 9.0 release hashes' + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64.open + os: windows + helixRepo: dotnet/sdk + timeoutInMinutes: 360 + enableTelemetry: true + steps: + - powershell: | + New-Item -ItemType Directory -Force -Path '$(Build.SourcesDirectory)/src/Installer/releases/9.0' | Out-Null + try { & '$(Build.SourcesDirectory)/src/Installer/releases/Verify-ReleaseHashes.ps1' -ChannelVersion '9.0' -OutputDir '$(Build.SourcesDirectory)/src/Installer/releases/9.0' } catch { Write-Host $_ } + Write-Host "Script exited with code: $LASTEXITCODE" + displayName: '🔐 Verify 9.0 hashes' + - task: PublishPipelineArtifact@1 + displayName: '📦 Publish 9.0 results' + condition: always() + inputs: + targetPath: '$(Build.SourcesDirectory)/src/Installer/releases/9.0' + artifactName: 'hash-verification-9.0' + - template: /eng/common/templates/job/job.yml + parameters: + displayName: '🔐 Verify 10.0 release hashes' + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64.open + os: windows + helixRepo: dotnet/sdk + timeoutInMinutes: 360 + enableTelemetry: true + steps: + - powershell: | + New-Item -ItemType Directory -Force -Path '$(Build.SourcesDirectory)/src/Installer/releases/10.0' | Out-Null + try { & '$(Build.SourcesDirectory)/src/Installer/releases/Verify-ReleaseHashes.ps1' -ChannelVersion '10.0' -OutputDir '$(Build.SourcesDirectory)/src/Installer/releases/10.0' } catch { Write-Host $_ } + Write-Host "Script exited with code: $LASTEXITCODE" + displayName: '🔐 Verify 10.0 hashes' + - task: PublishPipelineArtifact@1 + displayName: '📦 Publish 10.0 results' + condition: always() + inputs: + targetPath: '$(Build.SourcesDirectory)/src/Installer/releases/10.0' + artifactName: 'hash-verification-10.0' diff --git a/src/Installer/releases/Verify-ReleaseHashes.ps1 b/src/Installer/releases/Verify-ReleaseHashes.ps1 new file mode 100644 index 000000000000..a07741af076b --- /dev/null +++ b/src/Installer/releases/Verify-ReleaseHashes.ps1 @@ -0,0 +1,307 @@ +<# +.SYNOPSIS + Downloads and verifies SHA-512 hashes for .NET release archives (.zip and .tar.gz). +.DESCRIPTION + Fetches the releases.json manifest for a given .NET channel version, + downloads all .zip and .tar.gz files, computes their SHA-512 hashes, + and compares against the manifest. Valid files are deleted; mismatched + files are kept for inspection. + + Designed to run on CI or locally. Exit code 1 if any mismatches found. +.PARAMETER ChannelVersion + The .NET channel version to verify (e.g., "7.0", "8.0", "9.0", "10.0"). +.PARAMETER OutputDir + Directory to download files into. Defaults to a subdirectory named after the channel. +.PARAMETER Components + Which component types to verify. Comma-separated list. + Valid values: sdk, runtime, aspnetcore, windowsdesktop, all + Defaults to "all". +.PARAMETER ReleaseVersion + Optional. If specified, only verify files for this specific release version + (e.g., "7.0.20"). Otherwise verifies all releases in the channel. +.PARAMETER DryRun + If set, lists files that would be downloaded without actually downloading them. +.EXAMPLE + .\Verify-ReleaseHashes.ps1 -ChannelVersion 7.0 + .\Verify-ReleaseHashes.ps1 -ChannelVersion 7.0 -Components runtime,aspnetcore + .\Verify-ReleaseHashes.ps1 -ChannelVersion 7.0 -Components runtime -ReleaseVersion 7.0.20 + .\Verify-ReleaseHashes.ps1 -ChannelVersion 8.0 -DryRun +#> +param( + [Parameter(Mandatory = $true)] + [string]$ChannelVersion, + + [string]$OutputDir = $null, + + [string]$Components = "all", + + [string]$ReleaseVersion = $null, + + [switch]$DryRun +) + +$ErrorActionPreference = 'Stop' + +if (-not $OutputDir) { + $OutputDir = Join-Path $PSScriptRoot $ChannelVersion +} + +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null + +# Parse component filter +$componentFilter = if ($Components -eq 'all') { + @('sdk', 'runtime', 'aspnetcore', 'windowsdesktop') +} else { + $Components -split ',' | ForEach-Object { $_.Trim().ToLowerInvariant() } +} + +$manifestUrl = "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/$ChannelVersion/releases.json" +Write-Host "Fetching manifest: $manifestUrl" -ForegroundColor Cyan + +try { + $manifestJson = Invoke-RestMethod -Uri $manifestUrl -UseBasicParsing +} +catch { + Write-Error "Failed to fetch manifest for $ChannelVersion : $_" + return +} + +# Collect all files to verify: (url, hash, name, releaseVersion, component) +$filesToVerify = [System.Collections.Generic.List[hashtable]]::new() + +# Helper function to extract files from a component object +function Add-ComponentFiles { + param($componentData, $sectionName, $releaseVer, $fileList) + + if ($null -eq $componentData) { return } + + # Handle both single object and array + $items = @($componentData) + + foreach ($component in $items) { + if ($null -eq $component -or $null -eq $component.files) { continue } + foreach ($file in $component.files) { + $name = $file.name + if (-not $name) { continue } + if ($name -notmatch '\.(zip|tar\.gz)$') { continue } + if (-not $file.url -or -not $file.hash) { continue } + + $fileList.Add(@{ + Url = $file.url + ExpectedHash = $file.hash + FileName = $name + ReleaseVersion = $releaseVer + Component = $sectionName + Rid = $file.rid + }) + } + } +} + +foreach ($release in $manifestJson.releases) { + $relVer = $release.'release-version' + + # Filter by specific release version if requested + if ($ReleaseVersion -and $relVer -ne $ReleaseVersion) { continue } + + if ($componentFilter -contains 'sdk') { + Add-ComponentFiles -componentData $release.sdk -sectionName 'sdk' -releaseVer $relVer -fileList $filesToVerify + if ($release.sdks) { + Add-ComponentFiles -componentData $release.sdks -sectionName 'sdks' -releaseVer $relVer -fileList $filesToVerify + } + } + if ($componentFilter -contains 'runtime') { + Add-ComponentFiles -componentData $release.runtime -sectionName 'runtime' -releaseVer $relVer -fileList $filesToVerify + } + if ($componentFilter -contains 'aspnetcore') { + Add-ComponentFiles -componentData $release.'aspnetcore-runtime' -sectionName 'aspnetcore-runtime' -releaseVer $relVer -fileList $filesToVerify + } + if ($componentFilter -contains 'windowsdesktop') { + Add-ComponentFiles -componentData $release.'windowsdesktop' -sectionName 'windowsdesktop' -releaseVer $relVer -fileList $filesToVerify + } +} + +$totalFiles = $filesToVerify.Count +Write-Host "Found $totalFiles .zip/.tar.gz files to verify across $ChannelVersion releases." -ForegroundColor Green +Write-Host "Components: $($componentFilter -join ', ')" -ForegroundColor Green +if ($ReleaseVersion) { Write-Host "Filtered to release: $ReleaseVersion" -ForegroundColor Green } + +if ($DryRun) { + Write-Host "" + Write-Host "DRY RUN - Files that would be downloaded:" -ForegroundColor Yellow + Write-Host "==========================================" -ForegroundColor Yellow + $grouped = $filesToVerify | Group-Object { $_.ReleaseVersion } + foreach ($group in $grouped | Sort-Object Name) { + Write-Host " Release $($group.Name): $($group.Count) files" -ForegroundColor Cyan + foreach ($entry in $group.Group) { + Write-Host " $($entry.Component)/$($entry.Rid)/$($entry.FileName)" -ForegroundColor White + } + } + Write-Host "" + Write-Host "Total: $totalFiles files" -ForegroundColor Green + exit 0 +} + +# Results tracking +$allResults = @() + +# Create a single HttpClient for all downloads (no automatic decompression - critical for hash matching) +Add-Type -AssemblyName System.Net.Http +$handler = New-Object System.Net.Http.HttpClientHandler +$client = New-Object System.Net.Http.HttpClient($handler) +$client.Timeout = [timespan]::FromMinutes(15) + +$batchIndex = 0 + +foreach ($entry in $filesToVerify) { + $batchIndex++ + $url = $entry.Url + $expectedHash = $entry.ExpectedHash + $fileName = $entry.FileName + $releaseVersion = $entry.ReleaseVersion + $component = $entry.Component + $rid = $entry.Rid + + # Build a unique local filename to avoid collisions + $localName = "${releaseVersion}_${component}_${rid}_${fileName}" -replace '[<>:"/\\|?*]', '_' + $localPath = Join-Path $OutputDir $localName + + $result = @{ + FileName = $fileName + ReleaseVersion = $releaseVersion + Component = $component + Rid = $rid + Url = $url + ExpectedHash = $expectedHash + ActualHash = '' + Status = 'Unknown' + LocalPath = $localPath + Error = '' + } + + Write-Host "[$batchIndex/$totalFiles] Downloading: $fileName ($releaseVersion $component $rid)..." -NoNewline + + try { + $response = $client.GetAsync($url).GetAwaiter().GetResult() + if (-not $response.IsSuccessStatusCode) { + $result.Status = 'DownloadFailed' + $result.Error = "HTTP $($response.StatusCode)" + Write-Host " DOWNLOAD FAILED ($($response.StatusCode))" -ForegroundColor Red + $allResults += $result + continue + } + + $fs = [System.IO.File]::Create($localPath) + try { + $response.Content.CopyToAsync($fs).GetAwaiter().GetResult() + } + finally { + $fs.Close() + $fs.Dispose() + } + + # Compute SHA-512 + $sha512 = [System.Security.Cryptography.SHA512]::Create() + $fileStream = [System.IO.File]::OpenRead($localPath) + try { + $hashBytes = $sha512.ComputeHash($fileStream) + } + finally { + $fileStream.Close() + $fileStream.Dispose() + $sha512.Dispose() + } + + $actualHash = [BitConverter]::ToString($hashBytes).Replace('-', '').ToLowerInvariant() + $result.ActualHash = $actualHash + + if ($actualHash -eq $expectedHash.ToLowerInvariant()) { + $result.Status = 'Valid' + Remove-Item -Path $localPath -Force -ErrorAction SilentlyContinue + Write-Host " OK" -ForegroundColor Green + } + else { + $result.Status = 'MISMATCH' + Write-Host " MISMATCH!" -ForegroundColor Red + } + } + catch { + $result.Status = 'Error' + $result.Error = $_.Exception.Message + Write-Host " ERROR: $($_.Exception.Message)" -ForegroundColor Yellow + } + + $allResults += $result +} + +$client.Dispose() +$handler.Dispose() + +# Summary +$valid = @($allResults | Where-Object { $_.Status -eq 'Valid' }) +$mismatched = @($allResults | Where-Object { $_.Status -eq 'MISMATCH' }) +$failed = @($allResults | Where-Object { $_.Status -notin @('Valid', 'MISMATCH') }) + +Write-Host "" +Write-Host "============================================" -ForegroundColor Cyan +Write-Host " VERIFICATION RESULTS FOR $ChannelVersion" -ForegroundColor Cyan +Write-Host "============================================" -ForegroundColor Cyan +Write-Host " Total files checked: $($allResults.Count)" -ForegroundColor White +Write-Host " Valid (hash match): $($valid.Count)" -ForegroundColor Green +Write-Host " MISMATCHED: $($mismatched.Count)" -ForegroundColor Red +Write-Host " Failed/Error: $($failed.Count)" -ForegroundColor Yellow +Write-Host "" + +if ($mismatched.Count -gt 0) { + Write-Host "MISMATCHED FILES (kept for inspection):" -ForegroundColor Red + Write-Host "----------------------------------------" -ForegroundColor Red + foreach ($m in $mismatched) { + Write-Host " Release: $($m.ReleaseVersion)" -ForegroundColor White + Write-Host " File: $($m.FileName)" -ForegroundColor White + Write-Host " RID: $($m.Rid)" -ForegroundColor White + Write-Host " Component:$($m.Component)" -ForegroundColor White + Write-Host " Expected: $($m.ExpectedHash)" -ForegroundColor Yellow + Write-Host " Actual: $($m.ActualHash)" -ForegroundColor Red + Write-Host " Kept at: $($m.LocalPath)" -ForegroundColor Cyan + Write-Host "" + } +} + +if ($failed.Count -gt 0) { + Write-Host "FAILED DOWNLOADS:" -ForegroundColor Yellow + Write-Host "------------------" -ForegroundColor Yellow + foreach ($f in $failed) { + Write-Host " $($f.FileName) ($($f.ReleaseVersion) $($f.Component) $($f.Rid)): $($f.Status) - $($f.Error)" -ForegroundColor Yellow + } + Write-Host "" +} + +# Export results to CSV for later reference +$csvPath = Join-Path $OutputDir "verification-results-$ChannelVersion.csv" +$allResults | ForEach-Object { + [PSCustomObject]@{ + ReleaseVersion = $_.ReleaseVersion + Component = $_.Component + Rid = $_.Rid + FileName = $_.FileName + Status = $_.Status + ExpectedHash = $_.ExpectedHash + ActualHash = $_.ActualHash + Error = $_.Error + LocalPath = $_.LocalPath + Url = $_.Url + } +} | Export-Csv -Path $csvPath -NoTypeInformation -Encoding UTF8 + +Write-Host "Full results exported to: $csvPath" -ForegroundColor Cyan +Write-Host "" + +# Exit with code 1 if any mismatches found (useful for CI) +if ($mismatched.Count -gt 0) { + Write-Host "FAILURE: $($mismatched.Count) hash mismatch(es) detected." -ForegroundColor Red + exit 1 +} +else { + Write-Host "SUCCESS: All $($valid.Count) files verified." -ForegroundColor Green + exit 0 +} From 5081096b01ad845fa242a09b0aff8ecacb999ea7 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 25 Feb 2026 14:31:09 -0800 Subject: [PATCH 2/6] fix target path --- .vsts-dnup-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.vsts-dnup-pr.yml b/.vsts-dnup-pr.yml index 625c86b55068..285e2f49a469 100644 --- a/.vsts-dnup-pr.yml +++ b/.vsts-dnup-pr.yml @@ -104,6 +104,7 @@ stages: displayName: '📦 Publish 7.0 results' condition: always() inputs: + targetPath: '$(Build.SourcesDirectory)/src/Installer/releases/7.0' artifactName: 'hash-verification-7.0' - template: /eng/common/templates/job/job.yml parameters: From 35613e0d18ade5123e939cea7488ad9e04c79b4c Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 25 Feb 2026 14:37:14 -0800 Subject: [PATCH 3/6] Remove executables stage (template not on release/dnup) --- .vsts-dnup-pr.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.vsts-dnup-pr.yml b/.vsts-dnup-pr.yml index 285e2f49a469..dd5b0b707786 100644 --- a/.vsts-dnup-pr.yml +++ b/.vsts-dnup-pr.yml @@ -8,7 +8,6 @@ pr: - dnup - release/dnup - release/dotnetup - - nagilson-dnup-executables parameters: - name: enableArm64Job @@ -66,20 +65,6 @@ stages: emoji: 💪 helixTargetQueue: osx.13.arm64.open -- stage: executables - displayName: 🏗️ Build dotnetup executables - dependsOn: [] - jobs: - ### Build a single platform executable for PR validation ### - - template: /eng/pipelines/templates/jobs/dotnetup/dotnetup-executables.yml@self - parameters: - rid: linux-x64 - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals build.ubuntu.2204.amd64.open - os: linux - emoji: 🐧 - - stage: hash_verification displayName: 🔐 Verify release manifest hashes dependsOn: [] From f52d4608c981e055ac485f6bfde3c74a9d146c3a Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 25 Feb 2026 15:30:01 -0800 Subject: [PATCH 4/6] remove ambiguity in the check output --- .vsts-dnup-pr.yml | 44 +++++++ .../releases/Verify-ReleaseHashes.ps1 | 110 ++++++++++++++---- 2 files changed, 132 insertions(+), 22 deletions(-) diff --git a/.vsts-dnup-pr.yml b/.vsts-dnup-pr.yml index dd5b0b707786..71fbc31efe28 100644 --- a/.vsts-dnup-pr.yml +++ b/.vsts-dnup-pr.yml @@ -69,6 +69,50 @@ stages: displayName: 🔐 Verify release manifest hashes dependsOn: [] jobs: + - template: /eng/common/templates/job/job.yml + parameters: + displayName: '🔐 Verify 5.0 release hashes' + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64.open + os: windows + helixRepo: dotnet/sdk + timeoutInMinutes: 360 + enableTelemetry: true + steps: + - powershell: | + New-Item -ItemType Directory -Force -Path '$(Build.SourcesDirectory)/src/Installer/releases/5.0' | Out-Null + try { & '$(Build.SourcesDirectory)/src/Installer/releases/Verify-ReleaseHashes.ps1' -ChannelVersion '5.0' -OutputDir '$(Build.SourcesDirectory)/src/Installer/releases/5.0' } catch { Write-Host $_ } + Write-Host "Script exited with code: $LASTEXITCODE" + displayName: '🔐 Verify 5.0 hashes' + - task: PublishPipelineArtifact@1 + displayName: '📦 Publish 5.0 results' + condition: always() + inputs: + targetPath: '$(Build.SourcesDirectory)/src/Installer/releases/5.0' + artifactName: 'hash-verification-5.0' + - template: /eng/common/templates/job/job.yml + parameters: + displayName: '🔐 Verify 6.0 release hashes' + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64.open + os: windows + helixRepo: dotnet/sdk + timeoutInMinutes: 360 + enableTelemetry: true + steps: + - powershell: | + New-Item -ItemType Directory -Force -Path '$(Build.SourcesDirectory)/src/Installer/releases/6.0' | Out-Null + try { & '$(Build.SourcesDirectory)/src/Installer/releases/Verify-ReleaseHashes.ps1' -ChannelVersion '6.0' -OutputDir '$(Build.SourcesDirectory)/src/Installer/releases/6.0' } catch { Write-Host $_ } + Write-Host "Script exited with code: $LASTEXITCODE" + displayName: '🔐 Verify 6.0 hashes' + - task: PublishPipelineArtifact@1 + displayName: '📦 Publish 6.0 results' + condition: always() + inputs: + targetPath: '$(Build.SourcesDirectory)/src/Installer/releases/6.0' + artifactName: 'hash-verification-6.0' - template: /eng/common/templates/job/job.yml parameters: displayName: '🔐 Verify 7.0 release hashes' diff --git a/src/Installer/releases/Verify-ReleaseHashes.ps1 b/src/Installer/releases/Verify-ReleaseHashes.ps1 index a07741af076b..2a798c51d4e2 100644 --- a/src/Installer/releases/Verify-ReleaseHashes.ps1 +++ b/src/Installer/releases/Verify-ReleaseHashes.ps1 @@ -63,7 +63,7 @@ try { } catch { Write-Error "Failed to fetch manifest for $ChannelVersion : $_" - return + exit 1 } # Collect all files to verify: (url, hash, name, releaseVersion, component) @@ -142,19 +142,29 @@ if ($DryRun) { exit 0 } -# Results tracking -$allResults = @() +# Results tracking (thread-safe) +$allResults = [System.Collections.Concurrent.ConcurrentBag[hashtable]]::new() # Create a single HttpClient for all downloads (no automatic decompression - critical for hash matching) Add-Type -AssemblyName System.Net.Http $handler = New-Object System.Net.Http.HttpClientHandler +$handler.MaxConnectionsPerServer = 32 $client = New-Object System.Net.Http.HttpClient($handler) $client.Timeout = [timespan]::FromMinutes(15) -$batchIndex = 0 +# Parallel download + verify using runspaces +$maxParallel = 16 +$completedCount = [ref]0 +$printLock = [object]::new() + +$runspacePool = [runspacefactory]::CreateRunspacePool(1, $maxParallel) +$runspacePool.Open() + +$jobs = [System.Collections.Generic.List[object]]::new() + +$scriptBlock = { + param($entry, $outputDir, $client, $totalFiles, $completedCount, $printLock, $allResults) -foreach ($entry in $filesToVerify) { - $batchIndex++ $url = $entry.Url $expectedHash = $entry.ExpectedHash $fileName = $entry.FileName @@ -162,9 +172,8 @@ foreach ($entry in $filesToVerify) { $component = $entry.Component $rid = $entry.Rid - # Build a unique local filename to avoid collisions $localName = "${releaseVersion}_${component}_${rid}_${fileName}" -replace '[<>:"/\\|?*]', '_' - $localPath = Join-Path $OutputDir $localName + $localPath = [System.IO.Path]::Combine($outputDir, $localName) $result = @{ FileName = $fileName @@ -179,16 +188,20 @@ foreach ($entry in $filesToVerify) { Error = '' } - Write-Host "[$batchIndex/$totalFiles] Downloading: $fileName ($releaseVersion $component $rid)..." -NoNewline - try { $response = $client.GetAsync($url).GetAwaiter().GetResult() if (-not $response.IsSuccessStatusCode) { $result.Status = 'DownloadFailed' $result.Error = "HTTP $($response.StatusCode)" - Write-Host " DOWNLOAD FAILED ($($response.StatusCode))" -ForegroundColor Red - $allResults += $result - continue + $idx = [System.Threading.Interlocked]::Increment($completedCount) + lock ($printLock) { + [Console]::ForegroundColor = 'Red' + [Console]::WriteLine("[$idx/$totalFiles] $url ... DOWNLOAD FAILED ($($response.StatusCode))") + [Console]::ResetColor() + } + $allResults.Add($result) + $response.Dispose() + return } $fs = [System.IO.File]::Create($localPath) @@ -199,6 +212,7 @@ foreach ($entry in $filesToVerify) { $fs.Close() $fs.Dispose() } + $response.Dispose() # Compute SHA-512 $sha512 = [System.Security.Cryptography.SHA512]::Create() @@ -212,31 +226,77 @@ foreach ($entry in $filesToVerify) { $sha512.Dispose() } - $actualHash = [BitConverter]::ToString($hashBytes).Replace('-', '').ToLowerInvariant() + $actualHash = [System.BitConverter]::ToString($hashBytes).Replace('-', '').ToLowerInvariant() $result.ActualHash = $actualHash if ($actualHash -eq $expectedHash.ToLowerInvariant()) { $result.Status = 'Valid' - Remove-Item -Path $localPath -Force -ErrorAction SilentlyContinue - Write-Host " OK" -ForegroundColor Green + [System.IO.File]::Delete($localPath) + $idx = [System.Threading.Interlocked]::Increment($completedCount) + lock ($printLock) { + [Console]::ForegroundColor = 'Green' + [Console]::WriteLine("[$idx/$totalFiles] $url ... OK") + [Console]::ResetColor() + } } else { $result.Status = 'MISMATCH' - Write-Host " MISMATCH!" -ForegroundColor Red + $idx = [System.Threading.Interlocked]::Increment($completedCount) + lock ($printLock) { + [Console]::ForegroundColor = 'Red' + [Console]::WriteLine("[$idx/$totalFiles] $url ... MISMATCH!") + [Console]::ResetColor() + } } } catch { $result.Status = 'Error' $result.Error = $_.Exception.Message - Write-Host " ERROR: $($_.Exception.Message)" -ForegroundColor Yellow + $idx = [System.Threading.Interlocked]::Increment($completedCount) + lock ($printLock) { + [Console]::ForegroundColor = 'Yellow' + [Console]::WriteLine("[$idx/$totalFiles] $url ... ERROR: $($_.Exception.Message)") + [Console]::ResetColor() + } } - $allResults += $result + $allResults.Add($result) } +Write-Host "Starting parallel verification with $maxParallel concurrent downloads..." -ForegroundColor Cyan + +foreach ($entry in $filesToVerify) { + $ps = [powershell]::Create() + $ps.RunspacePool = $runspacePool + [void]$ps.AddScript($scriptBlock) + [void]$ps.AddArgument($entry) + [void]$ps.AddArgument($OutputDir) + [void]$ps.AddArgument($client) + [void]$ps.AddArgument($totalFiles) + [void]$ps.AddArgument($completedCount) + [void]$ps.AddArgument($printLock) + [void]$ps.AddArgument($allResults) + + $jobs.Add(@{ + PowerShell = $ps + Handle = $ps.BeginInvoke() + }) +} + +# Wait for all jobs to complete +foreach ($job in $jobs) { + $job.PowerShell.EndInvoke($job.Handle) + $job.PowerShell.Dispose() +} + +$runspacePool.Close() +$runspacePool.Dispose() $client.Dispose() $handler.Dispose() +# Convert ConcurrentBag to array for downstream processing +$allResults = @($allResults.ToArray()) + # Summary $valid = @($allResults | Where-Object { $_.Status -eq 'Valid' }) $mismatched = @($allResults | Where-Object { $_.Status -eq 'MISMATCH' }) @@ -258,6 +318,7 @@ if ($mismatched.Count -gt 0) { foreach ($m in $mismatched) { Write-Host " Release: $($m.ReleaseVersion)" -ForegroundColor White Write-Host " File: $($m.FileName)" -ForegroundColor White + Write-Host " URL: $($m.Url)" -ForegroundColor White Write-Host " RID: $($m.Rid)" -ForegroundColor White Write-Host " Component:$($m.Component)" -ForegroundColor White Write-Host " Expected: $($m.ExpectedHash)" -ForegroundColor Yellow @@ -296,9 +357,14 @@ $allResults | ForEach-Object { Write-Host "Full results exported to: $csvPath" -ForegroundColor Cyan Write-Host "" -# Exit with code 1 if any mismatches found (useful for CI) -if ($mismatched.Count -gt 0) { - Write-Host "FAILURE: $($mismatched.Count) hash mismatch(es) detected." -ForegroundColor Red +# Exit with code 1 if any mismatches or failures found (useful for CI) +if ($mismatched.Count -gt 0 -or $failed.Count -gt 0) { + if ($mismatched.Count -gt 0) { + Write-Host "FAILURE: $($mismatched.Count) hash mismatch(es) detected." -ForegroundColor Red + } + if ($failed.Count -gt 0) { + Write-Host "FAILURE: $($failed.Count) file(s) failed to download or verify." -ForegroundColor Red + } exit 1 } else { From 219487394afdf792c7946a78a936d5453cba9dcb Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Wed, 25 Feb 2026 15:30:44 -0800 Subject: [PATCH 5/6] concurrency --- .../releases/Verify-ReleaseHashes.ps1 | 233 ++++++++---------- 1 file changed, 104 insertions(+), 129 deletions(-) diff --git a/src/Installer/releases/Verify-ReleaseHashes.ps1 b/src/Installer/releases/Verify-ReleaseHashes.ps1 index 2a798c51d4e2..51e39497ece7 100644 --- a/src/Installer/releases/Verify-ReleaseHashes.ps1 +++ b/src/Installer/releases/Verify-ReleaseHashes.ps1 @@ -142,8 +142,8 @@ if ($DryRun) { exit 0 } -# Results tracking (thread-safe) -$allResults = [System.Collections.Concurrent.ConcurrentBag[hashtable]]::new() +# Results tracking +$allResults = @() # Create a single HttpClient for all downloads (no automatic decompression - critical for hash matching) Add-Type -AssemblyName System.Net.Http @@ -152,151 +152,126 @@ $handler.MaxConnectionsPerServer = 32 $client = New-Object System.Net.Http.HttpClient($handler) $client.Timeout = [timespan]::FromMinutes(15) -# Parallel download + verify using runspaces -$maxParallel = 16 -$completedCount = [ref]0 -$printLock = [object]::new() - -$runspacePool = [runspacefactory]::CreateRunspacePool(1, $maxParallel) -$runspacePool.Open() - -$jobs = [System.Collections.Generic.List[object]]::new() - -$scriptBlock = { - param($entry, $outputDir, $client, $totalFiles, $completedCount, $printLock, $allResults) - - $url = $entry.Url - $expectedHash = $entry.ExpectedHash - $fileName = $entry.FileName - $releaseVersion = $entry.ReleaseVersion - $component = $entry.Component - $rid = $entry.Rid - - $localName = "${releaseVersion}_${component}_${rid}_${fileName}" -replace '[<>:"/\\|?*]', '_' - $localPath = [System.IO.Path]::Combine($outputDir, $localName) - - $result = @{ - FileName = $fileName - ReleaseVersion = $releaseVersion - Component = $component - Rid = $rid - Url = $url - ExpectedHash = $expectedHash - ActualHash = '' - Status = 'Unknown' - LocalPath = $localPath - Error = '' +# Process files in parallel batches +$batchSize = 16 +$completedCount = 0 + +Write-Host "Starting parallel verification ($batchSize concurrent downloads)..." -ForegroundColor Cyan + +for ($batchStart = 0; $batchStart -lt $totalFiles; $batchStart += $batchSize) { + $batchEnd = [Math]::Min($batchStart + $batchSize, $totalFiles) + $batch = $filesToVerify.GetRange($batchStart, $batchEnd - $batchStart) + + # Start all downloads in this batch simultaneously + $pendingDownloads = @() + foreach ($entry in $batch) { + $localName = "$($entry.ReleaseVersion)_$($entry.Component)_$($entry.Rid)_$($entry.FileName)" -replace '[<>:"/\\|?*]', '_' + $localPath = Join-Path $OutputDir $localName + $pendingDownloads += @{ + Entry = $entry + LocalPath = $localPath + Task = $client.GetAsync($entry.Url) + } } + # Wait for all downloads in this batch to complete try { - $response = $client.GetAsync($url).GetAwaiter().GetResult() - if (-not $response.IsSuccessStatusCode) { - $result.Status = 'DownloadFailed' - $result.Error = "HTTP $($response.StatusCode)" - $idx = [System.Threading.Interlocked]::Increment($completedCount) - lock ($printLock) { - [Console]::ForegroundColor = 'Red' - [Console]::WriteLine("[$idx/$totalFiles] $url ... DOWNLOAD FAILED ($($response.StatusCode))") - [Console]::ResetColor() - } - $allResults.Add($result) - $response.Dispose() - return - } + [System.Threading.Tasks.Task]::WaitAll(@($pendingDownloads | ForEach-Object { $_.Task })) + } + catch { + # Individual failures are handled below; WaitAll throws AggregateException + } - $fs = [System.IO.File]::Create($localPath) - try { - $response.Content.CopyToAsync($fs).GetAwaiter().GetResult() + # Process each result + foreach ($dl in $pendingDownloads) { + $completedCount++ + $entry = $dl.Entry + $localPath = $dl.LocalPath + $url = $entry.Url + + $result = @{ + FileName = $entry.FileName + ReleaseVersion = $entry.ReleaseVersion + Component = $entry.Component + Rid = $entry.Rid + Url = $url + ExpectedHash = $entry.ExpectedHash + ActualHash = '' + Status = 'Unknown' + LocalPath = $localPath + Error = '' } - finally { - $fs.Close() - $fs.Dispose() - } - $response.Dispose() - # Compute SHA-512 - $sha512 = [System.Security.Cryptography.SHA512]::Create() - $fileStream = [System.IO.File]::OpenRead($localPath) try { - $hashBytes = $sha512.ComputeHash($fileStream) - } - finally { - $fileStream.Close() - $fileStream.Dispose() - $sha512.Dispose() - } + if ($dl.Task.IsFaulted) { + $result.Status = 'Error' + $result.Error = $dl.Task.Exception.InnerException.Message + Write-Host "[$completedCount/$totalFiles] $url ... ERROR: $($result.Error)" -ForegroundColor Yellow + $allResults += $result + continue + } - $actualHash = [System.BitConverter]::ToString($hashBytes).Replace('-', '').ToLowerInvariant() - $result.ActualHash = $actualHash - - if ($actualHash -eq $expectedHash.ToLowerInvariant()) { - $result.Status = 'Valid' - [System.IO.File]::Delete($localPath) - $idx = [System.Threading.Interlocked]::Increment($completedCount) - lock ($printLock) { - [Console]::ForegroundColor = 'Green' - [Console]::WriteLine("[$idx/$totalFiles] $url ... OK") - [Console]::ResetColor() + $response = $dl.Task.Result + try { + if (-not $response.IsSuccessStatusCode) { + $result.Status = 'DownloadFailed' + $result.Error = "HTTP $($response.StatusCode)" + Write-Host "[$completedCount/$totalFiles] $url ... DOWNLOAD FAILED ($($response.StatusCode))" -ForegroundColor Red + $allResults += $result + continue + } + + $fs = [System.IO.File]::Create($localPath) + try { + $response.Content.CopyToAsync($fs).GetAwaiter().GetResult() + } + finally { + $fs.Close() + $fs.Dispose() + } } - } - else { - $result.Status = 'MISMATCH' - $idx = [System.Threading.Interlocked]::Increment($completedCount) - lock ($printLock) { - [Console]::ForegroundColor = 'Red' - [Console]::WriteLine("[$idx/$totalFiles] $url ... MISMATCH!") - [Console]::ResetColor() + finally { + $response.Dispose() } - } - } - catch { - $result.Status = 'Error' - $result.Error = $_.Exception.Message - $idx = [System.Threading.Interlocked]::Increment($completedCount) - lock ($printLock) { - [Console]::ForegroundColor = 'Yellow' - [Console]::WriteLine("[$idx/$totalFiles] $url ... ERROR: $($_.Exception.Message)") - [Console]::ResetColor() - } - } - $allResults.Add($result) -} + # Compute SHA-512 + $sha512 = [System.Security.Cryptography.SHA512]::Create() + $fileStream = [System.IO.File]::OpenRead($localPath) + try { + $hashBytes = $sha512.ComputeHash($fileStream) + } + finally { + $fileStream.Close() + $fileStream.Dispose() + $sha512.Dispose() + } -Write-Host "Starting parallel verification with $maxParallel concurrent downloads..." -ForegroundColor Cyan - -foreach ($entry in $filesToVerify) { - $ps = [powershell]::Create() - $ps.RunspacePool = $runspacePool - [void]$ps.AddScript($scriptBlock) - [void]$ps.AddArgument($entry) - [void]$ps.AddArgument($OutputDir) - [void]$ps.AddArgument($client) - [void]$ps.AddArgument($totalFiles) - [void]$ps.AddArgument($completedCount) - [void]$ps.AddArgument($printLock) - [void]$ps.AddArgument($allResults) - - $jobs.Add(@{ - PowerShell = $ps - Handle = $ps.BeginInvoke() - }) -} + $actualHash = [BitConverter]::ToString($hashBytes).Replace('-', '').ToLowerInvariant() + $result.ActualHash = $actualHash -# Wait for all jobs to complete -foreach ($job in $jobs) { - $job.PowerShell.EndInvoke($job.Handle) - $job.PowerShell.Dispose() + if ($actualHash -eq $entry.ExpectedHash.ToLowerInvariant()) { + $result.Status = 'Valid' + Remove-Item -Path $localPath -Force -ErrorAction SilentlyContinue + Write-Host "[$completedCount/$totalFiles] $url ... OK" -ForegroundColor Green + } + else { + $result.Status = 'MISMATCH' + Write-Host "[$completedCount/$totalFiles] $url ... MISMATCH!" -ForegroundColor Red + } + } + catch { + $result.Status = 'Error' + $result.Error = $_.Exception.Message + Write-Host "[$completedCount/$totalFiles] $url ... ERROR: $($_.Exception.Message)" -ForegroundColor Yellow + } + + $allResults += $result + } } -$runspacePool.Close() -$runspacePool.Dispose() $client.Dispose() $handler.Dispose() -# Convert ConcurrentBag to array for downstream processing -$allResults = @($allResults.ToArray()) - # Summary $valid = @($allResults | Where-Object { $_.Status -eq 'Valid' }) $mismatched = @($allResults | Where-Object { $_.Status -eq 'MISMATCH' }) From 69bb511ec84fc3d0756cac2d282c3f97b62366b1 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 26 Feb 2026 09:38:29 -0800 Subject: [PATCH 6/6] scan exe and pkg next --- src/Installer/releases/Verify-ReleaseHashes.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Installer/releases/Verify-ReleaseHashes.ps1 b/src/Installer/releases/Verify-ReleaseHashes.ps1 index 51e39497ece7..a08cc96227d6 100644 --- a/src/Installer/releases/Verify-ReleaseHashes.ps1 +++ b/src/Installer/releases/Verify-ReleaseHashes.ps1 @@ -1,9 +1,9 @@ <# .SYNOPSIS - Downloads and verifies SHA-512 hashes for .NET release archives (.zip and .tar.gz). + Downloads and verifies SHA-512 hashes for .NET release installers (.exe and .pkg). .DESCRIPTION Fetches the releases.json manifest for a given .NET channel version, - downloads all .zip and .tar.gz files, computes their SHA-512 hashes, + downloads all .exe and .pkg files, computes their SHA-512 hashes, and compares against the manifest. Valid files are deleted; mismatched files are kept for inspection. @@ -83,7 +83,7 @@ function Add-ComponentFiles { foreach ($file in $component.files) { $name = $file.name if (-not $name) { continue } - if ($name -notmatch '\.(zip|tar\.gz)$') { continue } + if ($name -notmatch '\.(exe|pkg)$') { continue } if (-not $file.url -or -not $file.hash) { continue } $fileList.Add(@{ @@ -122,7 +122,7 @@ foreach ($release in $manifestJson.releases) { } $totalFiles = $filesToVerify.Count -Write-Host "Found $totalFiles .zip/.tar.gz files to verify across $ChannelVersion releases." -ForegroundColor Green +Write-Host "Found $totalFiles .exe/.pkg files to verify across $ChannelVersion releases." -ForegroundColor Green Write-Host "Components: $($componentFilter -join ', ')" -ForegroundColor Green if ($ReleaseVersion) { Write-Host "Filtered to release: $ReleaseVersion" -ForegroundColor Green }