Skip to content

Commit 08c285e

Browse files
authored
Re-apply internal staging versions of .NET in sync-internal-release command (#6699)
1 parent 71f7f31 commit 08c285e

38 files changed

+781
-255
lines changed

eng/update-dependencies/BaseCommand.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,12 @@
33

44
using System.CommandLine;
55
using System.CommandLine.NamingConventionBinder;
6-
using System.Threading.Tasks;
76
using Microsoft.Extensions.DependencyInjection;
87
using Microsoft.Extensions.Hosting;
98

109
namespace Dotnet.Docker;
1110

12-
public abstract class BaseCommand<TOptions>() where TOptions : IOptions
11+
public abstract class BaseCommand<TOptions>() : ICommand<TOptions> where TOptions : IOptions
1312
{
1413
public abstract Task<int> ExecuteAsync(TOptions options);
1514

@@ -34,7 +33,7 @@ public static Command Create(string name, string description)
3433
private static BindingHandler Handler =>
3534
CommandHandler.Create<TOptions, IHost>(async (options, host) =>
3635
{
37-
var thisCommand = host.Services.GetRequiredService<BaseCommand<TOptions>>();
36+
var thisCommand = host.Services.GetRequiredService<ICommand<TOptions>>();
3837
return await thisCommand.ExecuteAsync(options);
3938
});
4039
}

eng/update-dependencies/BaseUrlUpdater.cs

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,48 +17,44 @@ internal class BaseUrlUpdater : FileRegexUpdater
1717
{
1818
private const string BaseUrlGroupName = "BaseUrlValue";
1919
private readonly SpecificCommandOptions _options;
20-
private readonly JObject _manifestVariables;
20+
private readonly ManifestVariables _manifestVariables;
2121
private readonly string _manifestVariableName;
2222

2323
/// <summary>
2424
/// Creates a new <see cref="IDependencyUpdater"/> for updating base URLs.
2525
/// If the base URL variable cannot be found in the manifest, the updater
2626
/// won't do anything.
2727
/// </summary>
28-
public static IDependencyUpdater Create(string repoRoot, SpecificCommandOptions options)
28+
public static IDependencyUpdater Create(ManifestVariables manifestVariables, SpecificCommandOptions options)
2929
{
30-
// Load manifest and extract variables once so the constructor doesn't duplicate this logic
31-
var manifest = ManifestHelper.LoadManifest(SpecificCommand.VersionsFilename);
32-
var manifestVariables = (JObject?)manifest["variables"];
33-
3430
if (manifestVariables is null)
3531
{
3632
Trace.TraceWarning("BaseUrlUpdater: manifest variables missing - skipping base URL update.");
3733
return new EmptyDependencyUpdater();
3834
}
3935

36+
var upstreamBranch = manifestVariables.GetValue("branch");
4037
string baseUrlVarName = ManifestHelper.GetBaseUrlVariableName(
41-
options.DockerfileVersion,
42-
options.SourceBranch,
43-
options.VersionSourceName,
44-
options.IsSdkOnly);
38+
dockerfileVersion: options.DockerfileVersion,
39+
branch: upstreamBranch,
40+
versionSourceName: options.VersionSourceName,
41+
sdkOnlyRelease: options.IsSdkOnly);
4542

46-
if (!manifestVariables.ContainsKey(baseUrlVarName))
43+
if (!manifestVariables.HasValue(baseUrlVarName))
4744
{
4845
Trace.TraceWarning($"BaseUrlUpdater: variable '{baseUrlVarName}' not found - skipping base URL update.");
4946
return new EmptyDependencyUpdater();
5047
}
5148

52-
return new BaseUrlUpdater(repoRoot, options, manifestVariables, baseUrlVarName);
49+
return new BaseUrlUpdater(options, manifestVariables, baseUrlVarName);
5350
}
5451

5552
private BaseUrlUpdater(
56-
string repoRoot,
5753
SpecificCommandOptions options,
58-
JObject manifestVariables,
54+
ManifestVariables manifestVariables,
5955
string manifestVariableName)
6056
{
61-
Path = System.IO.Path.Combine(repoRoot, SpecificCommand.VersionsFilename);
57+
Path = options.GetManifestVersionsFilePath();
6258
VersionGroupName = BaseUrlGroupName;
6359
_options = options;
6460
_manifestVariables = manifestVariables;
@@ -72,7 +68,7 @@ protected override string TryGetDesiredValue(IEnumerable<IDependencyInfo> depend
7268
usedDependencyInfos = Enumerable.Empty<IDependencyInfo>();
7369

7470
string baseUrlVersionVarName = _manifestVariableName;
75-
string unresolvedBaseUrl = _manifestVariables[baseUrlVersionVarName]?.ToString() ??
71+
string unresolvedBaseUrl = _manifestVariables.Variables[baseUrlVersionVarName]?.ToString() ??
7672
throw new InvalidOperationException($"Variable with name '{baseUrlVersionVarName}' is missing.");
7773

7874
if (_options.IsInternal)

eng/update-dependencies/CreatePullRequestOptions.cs

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,33 +8,43 @@ namespace Dotnet.Docker;
88

99
public abstract record CreatePullRequestOptions
1010
{
11-
private string? _targetBranch = null;
11+
private string _azdoOrganizationUrl = "";
12+
13+
/// <summary>
14+
/// The root of the dotnet-docker repo to run against.
15+
/// </summary>
16+
public string RepoRoot { get; init; } = Directory.GetCurrentDirectory();
1217

1318
public string User { get; init; } = "";
1419
public string Email { get; init; } = "";
1520
public string Password { get; init; } = "";
16-
public string AzdoOrganization { get; init; } = "";
21+
public string AzdoOrganization
22+
{
23+
get => _azdoOrganizationUrl;
24+
init => _azdoOrganizationUrl = value.TrimEnd('/');
25+
}
1726
public string AzdoProject { get; init; } = "";
1827
public string AzdoRepo { get; init; } = "";
1928
public string VersionSourceName { get; init; } = "";
20-
public string SourceBranch { get; init; } = "nightly";
21-
public string TargetBranch
22-
{
23-
get => _targetBranch ?? SourceBranch;
24-
init => _targetBranch = value;
25-
}
29+
public string SourceBranch { get; init; } = "";
30+
public string TargetBranch { get; init; } = "nightly";
2631

2732
public static List<Option> Options =>
2833
[
34+
new Option<string>("--repo-root") { Description = "The root of the dotnet-docker repo to run against (defaults to current working directory)" },
2935
new Option<string>("--user") { Description = "GitHub or AzDO user used to make PR (if not specified, a PR will not be created)" },
3036
new Option<string>("--email") { Description = "GitHub or AzDO email used to make PR (if not specified, a PR will not be created)" },
3137
new Option<string>("--password") { Description = "GitHub or AzDO password used to make PR (if not specified, a PR will not be created)" },
32-
new Option<string>("--azdo-organization") { Description = "Name of the AzDO organization" },
38+
new Option<string>("--azdo-organization")
39+
{
40+
Description = "URL of the AzDO organization (like https://dev.azure.com/<orgname>), with or without a trailing slash."
41+
+ " The Azure Pipelines variable 'System.CollectionUri' provides this value.",
42+
},
3343
new Option<string>("--azdo-project") { Description = "Name of the AzDO project" },
3444
new Option<string>("--azdo-repo") { Description = "Name of the AzDO repo" },
3545
new Option<string>("--version-source-name") { Description = "The name of the source from which the version information was acquired." },
36-
new Option<string>("--source-branch") { Description = "Branch where the Dockerfiles are hosted" },
37-
new Option<string>("--target-branch") { Description = "Target branch of the generated PR (defaults to value of source-branch)" },
46+
new Option<string>("--source-branch") { Description = "If synchronizing multiple branches, the branch to pull updates from" },
47+
new Option<string>("--target-branch") { Description = "Pull request will be submitted targeting this branch" },
3848
];
3949

4050
public static List<Argument> Arguments => [];
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
namespace Dotnet.Docker;
5+
6+
internal static class CreatePullRequestOptionsExtensions
7+
{
8+
public static string GetManifestVersionsFilePath(this CreatePullRequestOptions options) =>
9+
Path.Combine(options.RepoRoot, "manifest.versions.json");
10+
11+
public static string GetAzdoRepoUrl(this CreatePullRequestOptions options)
12+
{
13+
// Validate that we have all the required pieces to construct the repo URL.
14+
ArgumentException.ThrowIfNullOrWhiteSpace(options.AzdoOrganization);
15+
ArgumentException.ThrowIfNullOrWhiteSpace(options.AzdoProject);
16+
ArgumentException.ThrowIfNullOrWhiteSpace(options.AzdoRepo);
17+
ArgumentException.ThrowIfNullOrWhiteSpace(options.TargetBranch);
18+
ArgumentException.ThrowIfNullOrWhiteSpace(options.SourceBranch);
19+
20+
// AzdoOrganization is a URL like https://dev.azure.com/<org>
21+
// A valid Azure DevOps repository URL is formatted like https://dev.azure.com/<org>/<project>/_git/<repo>
22+
return $"{options.AzdoOrganization}/{options.AzdoProject}/_git/{options.AzdoRepo}";
23+
}
24+
}

eng/update-dependencies/DependencyInjectionExtensions.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ public static void AddCommand<TCommand, TOptions>(
1717
where TCommand : BaseCommand<TOptions>
1818
where TOptions : IOptions
1919
{
20-
serviceCollection.AddSingleton<BaseCommand<TOptions>, TCommand>();
20+
serviceCollection.AddSingleton<ICommand<TOptions>, TCommand>();
2121
}
2222

2323
/// <summary>

eng/update-dependencies/DockerfileShaUpdater.cs

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,17 @@ public class DockerfileShaUpdater : FileRegexUpdater
4040
private readonly SpecificCommandOptions _options;
4141
private readonly string _versions;
4242
private readonly Dictionary<string, string> _urls;
43-
private readonly Lazy<JObject> _manifestVariables;
43+
private readonly ManifestVariables _manifestVariables;
4444

4545
public DockerfileShaUpdater(
46-
string productName, string dockerfileVersion, string? buildVersion, string arch, string os, string versions, SpecificCommandOptions options)
46+
string productName,
47+
string dockerfileVersion,
48+
string? buildVersion,
49+
string arch,
50+
string os,
51+
string versions,
52+
SpecificCommandOptions options,
53+
ManifestVariables manifestVariables)
4754
{
4855
_productName = productName;
4956
_dockerfileVersion = new Version(dockerfileVersion);
@@ -52,6 +59,7 @@ public DockerfileShaUpdater(
5259
_os = os;
5360
_versions = versions;
5461
_options = options;
62+
_manifestVariables = manifestVariables;
5563

5664
// Maps a product name to a set of one or more candidate URLs referencing the associated artifact. The order of the URLs
5765
// should be in priority order with each subsequent URL being the fallback.
@@ -69,18 +77,6 @@ public DockerfileShaUpdater(
6977
{ "sdk", "$DOTNET_BASE_URL/Sdk/$VERSION_DIR/dotnet-sdk-$VERSION_FILE-$OS-$ARCH.$ARCHIVE_EXT" },
7078
};
7179

72-
_manifestVariables = new Lazy<JObject>(
73-
() =>
74-
{
75-
const string VariablesProperty = "variables";
76-
JToken? variables = ManifestHelper.LoadManifest(SpecificCommand.VersionsFilename)[VariablesProperty];
77-
if (variables is null)
78-
{
79-
throw new InvalidOperationException($"'{VariablesProperty}' property missing in '{SpecificCommand.VersionsFilename}'");
80-
}
81-
return (JObject)variables;
82-
});
83-
8480
if (!string.IsNullOrEmpty(_options.InternalAccessToken))
8581
{
8682
s_httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(
@@ -91,9 +87,12 @@ public DockerfileShaUpdater(
9187
}
9288

9389
public static IEnumerable<IDependencyUpdater> CreateUpdaters(
94-
string productName, string dockerfileVersion, string repoRoot, SpecificCommandOptions options)
90+
string productName,
91+
string dockerfileVersion,
92+
SpecificCommandOptions options,
93+
ManifestVariables variables)
9594
{
96-
string versionsPath = System.IO.Path.Combine(repoRoot, SpecificCommand.VersionsFilename);
95+
string versionsPath = options.GetManifestVersionsFilePath();
9796
string versions = File.ReadAllText(versionsPath);
9897

9998
// The format of the sha variable name is '<productName>|<dockerfileVersion>|<os>|<arch>|sha'.
@@ -116,7 +115,8 @@ public static IEnumerable<IDependencyUpdater> CreateUpdaters(
116115
GetArch(parts),
117116
GetOs(parts),
118117
versions,
119-
options)
118+
options,
119+
variables)
120120
{
121121
Path = versionsPath,
122122
VersionGroupName = ShaValueGroupName
@@ -137,7 +137,7 @@ public static IEnumerable<IDependencyUpdater> CreateUpdaters(
137137
{
138138
usedBuildInfos = [dependencyBuildInfos.First(info => info.SimpleName == _productName)];
139139

140-
string baseUrl = ManifestHelper.GetBaseUrl(_manifestVariables.Value, _options);
140+
string baseUrl = ManifestHelper.GetBaseUrl(_manifestVariables.Variables, _options);
141141
// Remove Aspire Dashboard case once https://github.com/dotnet/aspire/issues/2035 is fixed.
142142
string archiveExt = _os.Contains("win") || _productName.Contains("aspire-dashboard") ? "zip" : "tar.gz";
143143
string versionDir = _buildVersion ?? "";
@@ -275,7 +275,7 @@ private static string GetArch(string[] variableParts)
275275
// corresponding build in the daily build location, for example, will not be signed due. So when we're targeting
276276
// the daily build location, we wouldn't use the release checksums file and instead use the other means of
277277
// retrieving the checksums.
278-
string baseUrl = ManifestHelper.GetBaseUrl(_manifestVariables.Value, _options);
278+
string baseUrl = ManifestHelper.GetBaseUrl(_manifestVariables.Variables, _options);
279279
if (baseUrl != ReleaseDotnetBaseCdnUrl)
280280
{
281281
return null;

eng/update-dependencies/FromStagingPipelineCommand.cs

Lines changed: 9 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
using Dotnet.Docker.Sync;
45
using Microsoft.Extensions.Logging;
56

67
namespace Dotnet.Docker;
78

89
internal partial class FromStagingPipelineCommand(
910
ILogger<FromStagingPipelineCommand> logger,
10-
PipelineArtifactProvider pipelineArtifactProvider)
11+
PipelineArtifactProvider pipelineArtifactProvider,
12+
IInternalVersionsService internalVersionsService)
1113
: BaseCommand<FromStagingPipelineOptions>
1214
{
1315
private readonly ILogger<FromStagingPipelineCommand> _logger = logger;
1416
private readonly PipelineArtifactProvider _pipelineArtifactProvider = pipelineArtifactProvider;
17+
private readonly IInternalVersionsService _internalVersionsService = internalVersionsService;
1518

1619
public override async Task<int> ExecuteAsync(FromStagingPipelineOptions options)
1720
{
@@ -44,7 +47,10 @@ public override async Task<int> ExecuteAsync(FromStagingPipelineOptions options)
4447
string dockerfileVersion = VersionHelper.ResolveMajorMinorVersion(releaseConfig.RuntimeBuild).ToString();
4548

4649
// Record pipeline run ID for this dockerfileVersion, for later use by sync-internal-release command
47-
RecordInternalVersion(dockerfileVersion, options.StagingPipelineRunId.ToString());
50+
_internalVersionsService.RecordInternalStagingBuild(
51+
options.RepoRoot,
52+
dockerfileVersion,
53+
options.StagingPipelineRunId);
4854

4955
var productVersions = (options.Internal, releaseConfig.SdkOnly) switch
5056
{
@@ -90,6 +96,7 @@ public override async Task<int> ExecuteAsync(FromStagingPipelineOptions options)
9096
var updateDependencies = new SpecificCommand();
9197
var updateDependenciesOptions = new SpecificCommandOptions()
9298
{
99+
RepoRoot = options.RepoRoot,
93100
DockerfileVersion = dockerfileVersion.ToString(),
94101
ProductVersions = productVersions,
95102

@@ -109,59 +116,6 @@ public override async Task<int> ExecuteAsync(FromStagingPipelineOptions options)
109116
return await updateDependencies.ExecuteAsync(updateDependenciesOptions);
110117
}
111118

112-
/// <summary>
113-
/// Records the staging pipeline run ID in an easy to parse format. This
114-
/// can be used by the sync-internal-release pipeline to record and
115-
/// re-apply the same staging builds after resetting the state of the repo
116-
/// to match the public release branch.
117-
/// </summary>
118-
/// <remarks>
119-
/// This will only store one staging pipeline run ID per dockerfileVersion
120-
/// </remarks>
121-
/// <param name="dockerfileVersion">major-minor version</param>
122-
/// <param name="stagingPipelineRunId">the build ID of the staging pipeline run</param>
123-
private void RecordInternalVersion(string dockerfileVersion, string stagingPipelineRunId)
124-
{
125-
const string InternalVersionsFile = "internal-versions.txt";
126-
127-
// Internal versions file should have one line per dockerfileVersion
128-
// Each line should be formatted as: <dockerfileVersion>=<stagingPipelineRunId>
129-
//
130-
// The preferable way to do this would be to record the version in
131-
// manifest.versions.json, however that would require one of the following:
132-
// 1) round-trip serialization, which would remove any whitespace/blank lines - which are
133-
// important for keeping the file readable and reducing git merge conflicts
134-
// 2) lots of regex JSON manipulation which is error-prone and harder to maintain
135-
//
136-
// So for now, the separate file and format is a compromise.
137-
138-
var versionsFilePath = Path.GetFullPath(SpecificCommand.VersionsFilename);
139-
var versionsFileDir = Path.GetDirectoryName(versionsFilePath) ?? "";
140-
var internalVersionFile = Path.Combine(versionsFileDir, InternalVersionsFile);
141-
Dictionary<string, string> versions = [];
142-
143-
_logger.LogInformation(
144-
"Recording staging pipeline build ID in {internalVersionFile}",
145-
internalVersionFile);
146-
147-
try
148-
{
149-
// File already exists - read existing versions
150-
versions = File.ReadAllLines(internalVersionFile)
151-
.Select(line => line.Split('=', 2))
152-
.Where(parts => parts.Length == 2)
153-
.ToDictionary(parts => parts[0], parts => parts[1]);
154-
}
155-
catch (FileNotFoundException)
156-
{
157-
// File doesn't exist - it will be created
158-
}
159-
160-
versions[dockerfileVersion] = stagingPipelineRunId;
161-
var versionLines = versions.Select(kv => $"{kv.Key}={kv.Value}");
162-
File.WriteAllLines(internalVersionFile, versionLines);
163-
}
164-
165119
/// <summary>
166120
/// Formats a storage account URL has a specific format:
167121
/// - Starts with "https://"

0 commit comments

Comments
 (0)