From c5e3890c56446af6f8af47041609969c08be6b50 Mon Sep 17 00:00:00 2001 From: webwarrior-ws Date: Tue, 4 Nov 2025 11:13:16 +0100 Subject: [PATCH 1/3] FileConventions(Test),scripts: add globalEnv arg To functions used to detect inconsistent versions in GithubCI. This change is needed to be able to take .env file into account when searching for vars referenced in Github workflow files. --- scripts/inconsistentVersionsInGitHubCI.fsx | 2 +- .../FileConventions.Test.fs | 29 ++++++++++++------- src/FileConventions/Library.fs | 15 +++++++--- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/scripts/inconsistentVersionsInGitHubCI.fsx b/scripts/inconsistentVersionsInGitHubCI.fsx index dd61a939..fa4704b5 100755 --- a/scripts/inconsistentVersionsInGitHubCI.fsx +++ b/scripts/inconsistentVersionsInGitHubCI.fsx @@ -33,7 +33,7 @@ let targetDir = |> fst let inconsistentVersionsInGitHubCI = - FileConventions.DetectInconsistentVersionsInGitHubCI targetDir + FileConventions.DetectInconsistentVersionsInGitHubCI targetDir Map.empty if inconsistentVersionsInGitHubCI then failwith diff --git a/src/FileConventions.Test/FileConventions.Test.fs b/src/FileConventions.Test/FileConventions.Test.fs index 5dc86423..71a71fcd 100644 --- a/src/FileConventions.Test/FileConventions.Test.fs +++ b/src/FileConventions.Test/FileConventions.Test.fs @@ -289,7 +289,7 @@ let DetectInconsistentVersionsInGitHubCIWorkflow1() = )) Assert.That( - DetectInconsistentVersionsInGitHubCIWorkflow fileInfo, + DetectInconsistentVersionsInGitHubCIWorkflow fileInfo Map.empty, Is.EqualTo false ) @@ -307,7 +307,7 @@ let DetectInconsistentVersionsInGitHubCIWorkflow2() = )) Assert.That( - DetectInconsistentVersionsInGitHubCIWorkflow fileInfo, + DetectInconsistentVersionsInGitHubCIWorkflow fileInfo Map.empty, Is.EqualTo true ) @@ -325,7 +325,7 @@ let DetectInconsistentVersionsInGitHubCIWorkflow3() = )) Assert.That( - DetectInconsistentVersionsInGitHubCIWorkflow fileInfo, + DetectInconsistentVersionsInGitHubCIWorkflow fileInfo Map.empty, Is.EqualTo true ) @@ -343,7 +343,7 @@ let DetectInconsistentVersionsInGitHubCIWorkflow4() = )) Assert.That( - DetectInconsistentVersionsInGitHubCIWorkflow fileInfo, + DetectInconsistentVersionsInGitHubCIWorkflow fileInfo Map.empty, Is.EqualTo false ) @@ -370,7 +370,7 @@ let DetectInconsistentVersionsInGitHubCIWorkflow5() = }) Assert.That( - DetectInconsistentVersionsInGitHubCIWorkflow fileInfo, + DetectInconsistentVersionsInGitHubCIWorkflow fileInfo Map.empty, Is.EqualTo true ) @@ -388,7 +388,7 @@ let DetectInconsistentVersionsInGitHubCIWorkflow6() = )) Assert.That( - DetectInconsistentVersionsInGitHubCIWorkflow fileInfo, + DetectInconsistentVersionsInGitHubCIWorkflow fileInfo Map.empty, Is.EqualTo true ) @@ -406,7 +406,7 @@ let DetectInconsistentVersionsInGitHubCIWorkflow7() = )) Assert.That( - DetectInconsistentVersionsInGitHubCIWorkflow fileInfo, + DetectInconsistentVersionsInGitHubCIWorkflow fileInfo Map.empty, Is.EqualTo true ) @@ -418,7 +418,10 @@ let DetectInconsistentVersionsInGitHubCI1() = Path.Combine(dummyFilesDirectory.FullName, "DummyWorkflows") ) - Assert.That(DetectInconsistentVersionsInGitHubCI fileInfo, Is.EqualTo true) + Assert.That( + DetectInconsistentVersionsInGitHubCI fileInfo Map.empty, + Is.EqualTo true + ) [] let DetectInconsistentVersionsInGitHubCI2() = @@ -427,7 +430,10 @@ let DetectInconsistentVersionsInGitHubCI2() = Path.Combine(dummyFilesDirectory.FullName, "DummyWorkflowsWithEnv") ) - Assert.That(DetectInconsistentVersionsInGitHubCI fileInfo, Is.EqualTo true) + Assert.That( + DetectInconsistentVersionsInGitHubCI fileInfo Map.empty, + Is.EqualTo true + ) [] let DetectInconsistentVersionsInGitHubCI3() = @@ -439,7 +445,10 @@ let DetectInconsistentVersionsInGitHubCI3() = ) ) - Assert.That(DetectInconsistentVersionsInGitHubCI fileInfo, Is.EqualTo true) + Assert.That( + DetectInconsistentVersionsInGitHubCI fileInfo Map.empty, + Is.EqualTo true + ) [] let DetectInconsistentVersionsInNugetRefsInFSharpScripts1() = diff --git a/src/FileConventions/Library.fs b/src/FileConventions/Library.fs index d33a17c3..7e7f47a7 100644 --- a/src/FileConventions/Library.fs +++ b/src/FileConventions/Library.fs @@ -362,6 +362,7 @@ let private GetVersionsMapFromFiles let private DetectInconsistentVersionsInYamlFiles (fileInfos: seq) (extractVersionsFunction: YamlNode -> seq) + (_globalEnv: Map) = let envVarRegex = Regex(@"\s*\$\{\{\s*([^\s\}]+)\s*\}\}\s*", RegexOptions.Compiled) @@ -426,7 +427,10 @@ let private DetectInconsistentVersionsInYamlFiles |> Seq.map(fun item -> Seq.length item.Value > 1) |> Seq.contains true -let DetectInconsistentVersionsInGitHubCIWorkflow(fileInfos: seq) = +let DetectInconsistentVersionsInGitHubCIWorkflow + (fileInfos: seq) + (globalEnv: Map) + = fileInfos |> Seq.iter(fun fileInfo -> assert (fileInfo.FullName.EndsWith ".yml")) @@ -462,15 +466,18 @@ let DetectInconsistentVersionsInGitHubCIWorkflow(fileInfos: seq) = ) | _ -> Seq.empty - DetectInconsistentVersionsInYamlFiles fileInfos extractVersions + DetectInconsistentVersionsInYamlFiles fileInfos extractVersions globalEnv -let DetectInconsistentVersionsInGitHubCI(dir: DirectoryInfo) = +let DetectInconsistentVersionsInGitHubCI + (dir: DirectoryInfo) + (globalEnv: Map) + = let ymlFiles = dir.GetFiles("*.yml", SearchOption.AllDirectories) if Seq.isEmpty ymlFiles then false else - DetectInconsistentVersionsInGitHubCIWorkflow ymlFiles + DetectInconsistentVersionsInGitHubCIWorkflow ymlFiles globalEnv let GetVersionsMapForNugetRefsInFSharpScripts(fileInfos: seq) = fileInfos From 1817c313f14c19329c51c663ca8d431c1107fa73 Mon Sep 17 00:00:00 2001 From: webwarrior-ws Date: Tue, 4 Nov 2025 10:28:26 +0100 Subject: [PATCH 2/3] FileConventions.Test: test for new functionality Added test for detecting inconsistent versions in Github workflow files with values in global environment dictionary. Contains failing test. --- ...DummyCIWithSameSetupPulumiVersionInEnv.yml | 25 +++++++++++++++++++ .../FileConventions.Test.fs | 19 ++++++++++++++ .../FileConventions.Test.fsproj | 1 + 3 files changed, 45 insertions(+) create mode 100644 src/FileConventions.Test/DummyFiles/DummyCIWithSameSetupPulumiVersionInEnv.yml diff --git a/src/FileConventions.Test/DummyFiles/DummyCIWithSameSetupPulumiVersionInEnv.yml b/src/FileConventions.Test/DummyFiles/DummyCIWithSameSetupPulumiVersionInEnv.yml new file mode 100644 index 00000000..490d7888 --- /dev/null +++ b/src/FileConventions.Test/DummyFiles/DummyCIWithSameSetupPulumiVersionInEnv.yml @@ -0,0 +1,25 @@ +name: CI + +on: [push, pull_request] + +jobs: + jobA: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - name: Setup Pulumi CLI + uses: pulumi/setup-pulumi@v2.0.0 + with: + pulumi-version: ${{ env.PULUMI_VERSION }} + - name: Print "Hello World!" + run: echo "Hello World!" + jobB: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - name: Setup Pulumi CLI + uses: pulumi/setup-pulumi@v2.0.0 + with: + pulumi-version: 3.40.0 + - name: Print "Hello World!" + run: echo "Hello World!" diff --git a/src/FileConventions.Test/FileConventions.Test.fs b/src/FileConventions.Test/FileConventions.Test.fs index 71a71fcd..ee1b81ba 100644 --- a/src/FileConventions.Test/FileConventions.Test.fs +++ b/src/FileConventions.Test/FileConventions.Test.fs @@ -410,6 +410,25 @@ let DetectInconsistentVersionsInGitHubCIWorkflow7() = Is.EqualTo true ) +[] +let DetectInconsistentVersionsInGitHubCIWorkflow8() = + let fileInfo = + (Seq.singleton( + FileInfo( + Path.Combine( + dummyFilesDirectory.FullName, + "DummyCIWithSameSetupPulumiVersionInEnv.yml" + ) + ) + )) + + let envDict = Map.ofList [ "PULUMI_VERSION", "3.40.0" ] + + Assert.That( + DetectInconsistentVersionsInGitHubCIWorkflow fileInfo envDict, + Is.EqualTo false + ) + [] let DetectInconsistentVersionsInGitHubCI1() = diff --git a/src/FileConventions.Test/FileConventions.Test.fsproj b/src/FileConventions.Test/FileConventions.Test.fsproj index 88064c27..276a60bb 100644 --- a/src/FileConventions.Test/FileConventions.Test.fsproj +++ b/src/FileConventions.Test/FileConventions.Test.fsproj @@ -48,5 +48,6 @@ + From d10db5ffd5fe7a2df843fd43c1b96c1a056b629a Mon Sep 17 00:00:00 2001 From: webwarrior-ws Date: Tue, 4 Nov 2025 11:13:31 +0100 Subject: [PATCH 3/3] FileConventions(Test),scripts: acknowledge .env Take .env file into account in inconsistentVersionsInGitHubCI.fsx script when searching for vars referenced in Github workflow files, when .env file is present in the repository root. This is needed because workflows in pulumi-deploy now load vars from .env file. --- scripts/inconsistentVersionsInGitHubCI.fsx | 25 +++++++++- src/FileConventions/Library.fs | 53 +++++++++++++--------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/scripts/inconsistentVersionsInGitHubCI.fsx b/scripts/inconsistentVersionsInGitHubCI.fsx index fa4704b5..b4f6af4a 100755 --- a/scripts/inconsistentVersionsInGitHubCI.fsx +++ b/scripts/inconsistentVersionsInGitHubCI.fsx @@ -33,7 +33,30 @@ let targetDir = |> fst let inconsistentVersionsInGitHubCI = - FileConventions.DetectInconsistentVersionsInGitHubCI targetDir Map.empty + let repositoryRootDir = targetDir.Parent.Parent + + let globalEnv = + let dotEnvFile = + Path.Join(repositoryRootDir.FullName, ".env") |> FileInfo + + if dotEnvFile.Exists then + let lines = File.ReadAllLines dotEnvFile.FullName + + lines + |> Seq.filter(fun line -> + not( + System.String.IsNullOrWhiteSpace line || line.StartsWith '#' + ) + ) + |> Seq.map(fun line -> + let [| key; value |] = line.Split('=', count = 2) + key.Trim(), value.Trim() + ) + |> Map.ofSeq + else + Map.empty + + FileConventions.DetectInconsistentVersionsInGitHubCI targetDir globalEnv if inconsistentVersionsInGitHubCI then failwith diff --git a/src/FileConventions/Library.fs b/src/FileConventions/Library.fs index 7e7f47a7..774a785f 100644 --- a/src/FileConventions/Library.fs +++ b/src/FileConventions/Library.fs @@ -362,7 +362,7 @@ let private GetVersionsMapFromFiles let private DetectInconsistentVersionsInYamlFiles (fileInfos: seq) (extractVersionsFunction: YamlNode -> seq) - (_globalEnv: Map) + (globalEnv: Map) = let envVarRegex = Regex(@"\s*\$\{\{\s*([^\s\}]+)\s*\}\}\s*", RegexOptions.Compiled) @@ -383,6 +383,24 @@ let private DetectInconsistentVersionsInYamlFiles let matches = Seq.collect extractVersionsFunction yamlDoc.AllNodes + let yamlDict = yamlDoc :?> YamlMappingNode + + let localEnv = + match yamlDict.Children.TryGetValue "env" with + | true, (:? YamlMappingNode as node) -> node + | _ -> YamlMappingNode() + + let envDict = + localEnv.Children + |> Seq.fold + (fun acc pair -> + acc + |> Map.add + (pair.Key :?> YamlScalarNode).Value + (pair.Value :?> YamlScalarNode).Value + ) + globalEnv + matches |> Seq.fold (fun acc (key, value) -> @@ -390,26 +408,19 @@ let private DetectInconsistentVersionsInYamlFiles let variableRegexMatch = envVarRegex.Match value if variableRegexMatch.Success then - let yamlDict = yamlDoc :?> YamlMappingNode - - match yamlDict.Children.TryGetValue "env" with - | true, (:? YamlMappingNode as envDict) -> - let referenceString = - variableRegexMatch.Groups.[1].Value - - let envVarName = - if referenceString.StartsWith "env." then - referenceString.[4..] - else - referenceString - - match - envDict.Children.TryGetValue envVarName - with - | true, envVarValue -> - (envVarValue :?> YamlScalarNode).Value - | false, _ -> value - | _ -> value + let referenceString = + variableRegexMatch.Groups.[1].Value + + let envVarName = + if referenceString.StartsWith "env." then + referenceString.[4..] + else + referenceString + + match envDict.TryGetValue envVarName with + | true, envVarValue -> envVarValue + | false, _ -> + failwithf "env. var %s not found" envVarName else value