diff --git a/.github/print-matrix/action.yml b/.github/print-matrix/action.yml index 3fb0b2d8..e9993400 100644 --- a/.github/print-matrix/action.yml +++ b/.github/print-matrix/action.yml @@ -69,7 +69,7 @@ runs: CACHIX_AUTH_TOKEN: ${{ inputs.cachix-auth-token }} PRECALC_MATRIX: ${{ inputs.precalc_matrix }} MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }} - run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl print_table + run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl print-table - name: Update GitHub Comment uses: marocchino/sticky-pull-request-comment@v2.9.0 diff --git a/.github/workflows/reusable-flake-checks-ci-matrix.yml b/.github/workflows/reusable-flake-checks-ci-matrix.yml index c75c61cb..a89abd78 100644 --- a/.github/workflows/reusable-flake-checks-ci-matrix.yml +++ b/.github/workflows/reusable-flake-checks-ci-matrix.yml @@ -65,7 +65,7 @@ jobs: CACHIX_CACHE: ${{ vars.CACHIX_CACHE }} CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }} - run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl shard_matrix + run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl shard-matrix outputs: gen_matrix: ${{ steps.generate-matrix.outputs.gen_matrix }} @@ -100,7 +100,7 @@ jobs: FLAKE_PRE: ${{ matrix.prefix }} FLAKE_POST: ${{ matrix.postfix }} MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }} - run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl ci_matrix + run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl ci-matrix - uses: actions/upload-artifact@v5 with: @@ -259,4 +259,4 @@ jobs: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} CACHIX_ACTIVATE_TOKEN: '${{ secrets.CACHIX_ACTIVATE_TOKEN }}' MCL_BRANCH: ${{ github.repository == 'metacraft-labs/nixos-modules' && github.sha || 'main' }} - run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl deploy_spec + run: nix run --accept-flake-config github:metacraft-labs/nixos-modules/${{ env.MCL_BRANCH }}#mcl deploy-spec diff --git a/packages/default.nix b/packages/default.nix index 394fd7ee..df8cf6d7 100644 --- a/packages/default.nix +++ b/packages/default.nix @@ -67,9 +67,7 @@ } // optionalAttrs (system == "x86_64-linux") { mcl = pkgs.callPackage ./mcl { - buildDubPackage = inputs'.dlang-nix.legacyPackages.buildDubPackage.override { - dCompiler = inputs'.dlang-nix.packages."ldc-binary-1_38_0"; - }; + dCompiler = inputs'.dlang-nix.packages."ldc-binary-1_38_0"; inherit (legacyPackages.inputs.nixpkgs) cachix nix nix-eval-jobs; }; }; diff --git a/packages/mcl/default.nix b/packages/mcl/default.nix index 998abcfd..be5225d6 100644 --- a/packages/mcl/default.nix +++ b/packages/mcl/default.nix @@ -1,6 +1,6 @@ { lib, - buildDubPackage, + dCompiler, pkgs, nix, nix-eval-jobs, @@ -43,9 +43,10 @@ let ] ); in -buildDubPackage rec { +pkgs.buildDubPackage rec { pname = "mcl"; version = "unstable"; + src = lib.fileset.toSource { root = ./.; fileset = lib.fileset.fileFilter ( @@ -59,22 +60,33 @@ buildDubPackage rec { ) ./.; }; + inherit dCompiler; + + dubLock = ./dub-lock.json; + nativeBuildInputs = [ pkgs.makeWrapper ] ++ deps; - postFixup = '' - wrapProgram $out/bin/${pname} --set PATH "${lib.makeBinPath deps}" --set LD_LIBRARY_PATH "${lib.makeLibraryPath deps}" - ''; + dubBuildType = "debug"; - dubBuildFlags = [ - "-b" - "debug" - ]; + doCheck = true; dubTestFlags = [ "--" "-e" - excludedTests + (lib.escapeShellArg excludedTests) ]; + installPhase = '' + runHook preInstall + install -Dm755 ./build/${pname} -t $out/bin/ + runHook postInstall + ''; + + postFixup = '' + wrapProgram $out/bin/${pname} \ + --set PATH "${lib.makeBinPath deps}" \ + --set LD_LIBRARY_PATH "${lib.makeLibraryPath deps}" + ''; + meta.mainProgram = pname; } diff --git a/packages/mcl/dub-lock.json b/packages/mcl/dub-lock.json new file mode 100644 index 00000000..e9674ebc --- /dev/null +++ b/packages/mcl/dub-lock.json @@ -0,0 +1,20 @@ +{ + "dependencies": { + "argparse": { + "version": "2.0.2", + "sha256": "1p3496rcndgaf68mxizfi4wrkf5frgzzfpp9frlvbjnkinizb2q9" + }, + "mir-core": { + "version": "1.7.3", + "sha256": "0yy1jg1b0i2n7xzkj5ykl1sg3qljna9nhjid4mfjz7ncybls49l9" + }, + "mir-cpuid": { + "version": "1.2.11", + "sha256": "1gxrnvvy34cfi71p8a0s904z3zp1l7k9qvliacxmhvykxr1nhrsf" + }, + "silly": { + "version": "1.1.1", + "sha256": "0fz7ib715sfk3w69i6xns5pwd4caahvfqjf32v13daxm1ms8xdzz" + } + } +} diff --git a/packages/mcl/dub.sdl b/packages/mcl/dub.sdl index 011a343f..b7d84cc0 100644 --- a/packages/mcl/dub.sdl +++ b/packages/mcl/dub.sdl @@ -14,8 +14,6 @@ buildType "unittest-debug" { buildOptions "unittests" "debugMode" "debugInfo" } -dflags "-preview=in" -dflags "-preview=shortenedMethods" dflags "-defaultlib=libphobos2.so" platform="dmd" lflags "-fuse-ld=gold" platform="dmd" dflags "-mcpu=generic" platform="ldc" @@ -23,3 +21,4 @@ dflags "-mcpu=baseline" platform="dmd" dependency "mir-cpuid" version="~>1.2.11" dependency "silly" version="~>1.1.1" +dependency "argparse" version="~>2.0.2" diff --git a/packages/mcl/dub.selections.json b/packages/mcl/dub.selections.json index 2977f10a..a032fdd6 100644 --- a/packages/mcl/dub.selections.json +++ b/packages/mcl/dub.selections.json @@ -1,7 +1,8 @@ { "fileVersion": 1, "versions": { - "mir-core": "1.7.1", + "argparse": "2.0.2", + "mir-core": "1.7.3", "mir-cpuid": "1.2.11", "silly": "1.1.1" } diff --git a/packages/mcl/src/main.d b/packages/mcl/src/main.d index 4412d7ea..b184a29b 100644 --- a/packages/mcl/src/main.d +++ b/packages/mcl/src/main.d @@ -1,59 +1,56 @@ +module mcl.main; + import std.stdio : writefln, writeln, stderr; import std.array : replace; import std.getopt : getopt; import std.logger : infof, errorf, LogLevel; - +import std.sumtype : SumType, match; +import std.string : stripRight, stripLeft; +import std.algorithm : endsWith; +import std.format : format; +import std.meta : staticMap; +import std.traits : Parameters; import mcl.utils.path : rootDir; import mcl.utils.tui : bold; -import cmds = mcl.commands; +import mcl.commands : SubCommandFunctions; -alias supportedCommands = imported!`std.traits`.AliasSeq!( - cmds.get_fstab, - cmds.deploy_spec, - cmds.ci_matrix, - cmds.print_table, - cmds.shard_matrix, - cmds.host_info, - cmds.ci, - cmds.machine, - cmds.config, -); +import argparse : Command, Description, SubCommand, NamedArgument, Default, CLI, matchCmd; -int main(string[] args) +@(Command(" ").Description(" ")) +struct UnknownCommandArgs { } +int unknown_command(UnknownCommandArgs unused) { - if (args.length < 2) - return wrongUsage("no command selected"); + stderr.writeln("Unknown command. Use --help for a list of available commands."); + return 1; +} - string cmd = args[1]; - LogLevel logLevel = LogLevel.info; - args.getopt("log-level", &logLevel); +struct MCLArgs +{ + @NamedArgument(["log-level"]) + LogLevel logLevel = cast(LogLevel)-1; - setLogLevel(logLevel); + SubCommand!( + staticMap!(Parameters, SubCommandFunctions), + Default!UnknownCommandArgs + ) cmd; +} - infof("Git root: '%s'", rootDir.bold); +alias SumTypeCase(alias func) = (Parameters!func args) => func(args); - try switch (cmd) - { - default: - return wrongUsage("unknown command: `" ~ cmd ~ "`"); +mixin CLI!MCLArgs.main!((args) +{ + LogLevel logLevel = LogLevel.info; + if (args.logLevel != cast(LogLevel)-1) + logLevel = args.logLevel; + setLogLevel(logLevel); - static foreach (command; supportedCommands) - case __traits(identifier, command): - { + int result = args.cmd.matchCmd!( + staticMap!(SumTypeCase, unknown_command, SubCommandFunctions), + ); - infof("Running %s task", cmd.bold); - command(args[2..$]); - infof("Execution Succesfull"); - return 0; - } - } - catch (Exception e) - { - errorf("Task %s failed. Error:\n%s", cmd.bold, e); - return 1; - } -} + return 0; +}); void setLogLevel(LogLevel l) { @@ -61,13 +58,3 @@ void setLogLevel(LogLevel l) globalLogLevel = l; (cast()sharedLog()).logLevel = l; } - -int wrongUsage(string error) -{ - writefln("Error: %s.", error); - writeln("Usage:\n"); - static foreach (cmd; supportedCommands) - writefln(" mcl %s", __traits(identifier, cmd)); - - return 1; -} diff --git a/packages/mcl/src/src/mcl/commands/ci.d b/packages/mcl/src/src/mcl/commands/ci.d index 72383894..8c014d9e 100644 --- a/packages/mcl/src/src/mcl/commands/ci.d +++ b/packages/mcl/src/src/mcl/commands/ci.d @@ -8,31 +8,34 @@ import std.array : array, join; import std.conv : to; import std.process : ProcessPipes; -import mcl.utils.env : optional, parseEnv; -import mcl.commands.ci_matrix: nixEvalJobs, SupportedSystem, Params, flakeAttr; +import argparse : Command, Description; + +import mcl.commands.ci_matrix: nixEvalJobs, SupportedSystem, flakeAttr, CiMatrixBaseArgs; import mcl.commands.shard_matrix: generateShardMatrix; import mcl.utils.path : rootDir, createResultDirs; import mcl.utils.process : execute; import mcl.utils.nix : nix; import mcl.utils.json : toJSON; -Params params; -export void ci(string[] args) -{ - params = parseEnv!Params; +@(Command("ci").Description("Run CI")) +struct CiArgs { + mixin CiMatrixBaseArgs!(); +} +export int ci(CiArgs args) +{ auto shardMatrix = generateShardMatrix(); foreach (shard; shardMatrix.include) { - params.flakePre = shard.prefix; - params.flakePost = shard.postfix; + args.flakePre = shard.prefix; + args.flakePost = shard.postfix; - if (params.flakePre == "") + if (args.flakePre == "") { - params.flakePre = "checks"; + args.flakePre = "checks"; } - string cachixUrl = "https://" ~ params.cachixCache ~ ".cachix.org"; + string cachixUrl = "https://" ~ args.cachixCache ~ ".cachix.org"; version (AArch64) { string arch = "aarch64"; } @@ -47,8 +50,8 @@ export void ci(string[] args) string os = "darwin"; } - auto matrix = flakeAttr(params.flakePre, arch, os, params.flakePost) - .nixEvalJobs(cachixUrl, false); + auto matrix = flakeAttr(args.flakePre, arch, os, args.flakePost) + .nixEvalJobs(cachixUrl, args, false); foreach (pkg; matrix) { @@ -65,8 +68,9 @@ export void ci(string[] args) "".writeln; auto json = parseJSON(res.stdout.byLine.join("\n").to!string); auto path = json.array[0]["outputs"]["out"].str; - execute(["cachix", "push", params.cachixCache, path], false, true).writeln; + execute(["cachix", "push", args.cachixCache, path], false, true).writeln; } } + return 0; } diff --git a/packages/mcl/src/src/mcl/commands/ci_matrix.d b/packages/mcl/src/src/mcl/commands/ci_matrix.d index 91d262fb..10ea42c6 100755 --- a/packages/mcl/src/src/mcl/commands/ci_matrix.d +++ b/packages/mcl/src/src/mcl/commands/ci_matrix.d @@ -16,13 +16,17 @@ import std.exception : enforce; import std.format : fmt = format; import std.logger : tracef, infof, errorf, warningf; -import mcl.utils.env : optional, MissingEnvVarsException, parseEnv; +import argparse : Command, Description, NamedArgument, Required, Placeholder, EnvFallback; + import mcl.utils.string : enumToString, StringRepresentation, MaxWidth, writeRecordAsTable; import mcl.utils.json : toJSON; import mcl.utils.path : rootDir, resultDir, gcRootsDir, createResultDirs; import mcl.utils.process : execute; import mcl.utils.nix : nix; +import mcl.commands.ci: CiArgs; +import mcl.commands.deploy_spec: DeploySpecArgs; + enum GitHubOS { @StringRepresentation("ubuntu-latest") ubuntuLatest, @@ -141,18 +145,84 @@ version (unittest) ]; } -immutable Params params; +mixin template CiMatrixBaseArgs() +{ + import argparse : Command, Description, NamedArgument, Required, Placeholder, EnvFallback; + + @(NamedArgument(["flake-pre"]) + .Placeholder("prefix") + ) + string flakePre = "checks"; + + @(NamedArgument(["flake-post"]) + .Placeholder("postfix") + ) + string flakePost; + + @(NamedArgument(["max-workers"]) + .Placeholder("count") + ) + int maxWorkers; -version (unittest) {} else -shared static this() + @(NamedArgument(["max-memory"]) + .Placeholder("mb") + ) + int maxMemory; + + @(NamedArgument(["initial"]) + ) + bool isInitial; + + @(NamedArgument(["cachix-cache"]) + .Placeholder("cache") + .EnvFallback("CACHIX_CACHE") + .Required() + ) + string cachixCache; + + @(NamedArgument(["cachix-auth-token"]) + .Placeholder("token") + .EnvFallback("CACHIX_AUTH_TOKEN") + .Required() + ) + string cachixAuthToken; +} + +@(Command("ci-matrix", "ci_matrix") + .Description("Print a table of the cache status of each package")) +struct CiMatrixArgs +{ + mixin CiMatrixBaseArgs!(); +} + +@(Command("print-table", "print_table") + .Description("Print a table of the cache status of each package")) +struct PrintTableArgs { - params = parseEnv!Params; + mixin CiMatrixBaseArgs!(); + + @(NamedArgument(["precalc-matrix"]) + .Placeholder("matrix") + ) + string precalcMatrix; } -export void ci_matrix(string[] args) +export int ci_matrix(CiMatrixArgs args) { createResultDirs(); - nixEvalForAllSystems().array.printTableForCacheStatus(); + nixEvalForAllSystems(args).array.printTableForCacheStatus(args); + return 0; +} + +export int print_table(PrintTableArgs args) +{ + createResultDirs(); + + getPrecalcMatrix(args) + .checkCacheStatus(args) + .printTableForCacheStatus(args); + + return 0; } string flakeAttr(string prefix, SupportedSystem system, string postfix) @@ -167,14 +237,15 @@ string flakeAttr(string prefix, string arch, string os, string postfix) return "%s.%s-%s%s".fmt(prefix, arch, os, postfix); } -Package[] checkCacheStatus(Package[] packages) +Package[] checkCacheStatus(T)(Package[] packages, auto ref T args) + if (is(T == CiMatrixArgs) || is(T == PrintTableArgs) || is(T == CiArgs) || is(T == DeploySpecArgs)) { import std.array : appender; import std.parallelism : parallel; foreach (ref pkg; packages.parallel) { - pkg = checkPackage(pkg); + pkg = checkPackage(pkg, args); struct Output { string isCached, name, storePath; } auto res = appender!string; writeRecordAsTable( @@ -186,32 +257,7 @@ Package[] checkCacheStatus(Package[] packages) return packages; } -export void print_table(string[] args) -{ - createResultDirs(); - - getPrecalcMatrix() - .checkCacheStatus() - .printTableForCacheStatus(); -} - -struct Params -{ - @optional() string flakePre; - @optional() string flakePost; - @optional() string precalcMatrix; - @optional() int maxWorkers; - @optional() int maxMemory; - @optional() bool isInitial; - string cachixCache; - string cachixAuthToken; - void setup() - { - if (this.flakePre == "") - this.flakePre = "checks"; - } -} GitHubOS systemToGHPlatform(SupportedSystem os) { @@ -308,28 +354,28 @@ unittest )); } } - -Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = true) +Package[] nixEvalJobs(T)(string flakeAttrPrefix, string cachixUrl, auto ref T args, bool doCheck = true) + if (is(T == CiMatrixArgs) || is(T == PrintTableArgs) || is(T == CiArgs) || is(T == DeploySpecArgs)) { Package[] result = []; bool hasError = false; - int maxMemoryMB = getAvailableMemoryMB(); - int maxWorkers = getNixEvalWorkerCount(); + int maxMemoryMB = getAvailableMemoryMB(args); + int maxWorkers = getNixEvalWorkerCount(args); - const args = [ + const nixArgs = [ "nix-eval-jobs", "--quiet", "--option", "warn-dirty", "false", "--check-cache-status", "--gc-roots-dir", gcRootsDir, "--workers", maxWorkers.to!string, "--max-memory-size", maxMemoryMB.to!string, "--flake", rootDir ~ "#" ~ flakeAttrPrefix ]; - const commandString = args.join(" "); + const commandString = nixArgs.join(" "); - tracef("%-(%s %)", args); + tracef("%-(%s %)", nixArgs); - auto pipes = pipeProcess(args, Redirect.stdout | Redirect.stderr); + auto pipes = pipeProcess(nixArgs, Redirect.stdout | Redirect.stderr); void logWarning(const char[] msg) { @@ -353,7 +399,7 @@ Package[] nixEvalJobs(string flakeAttrPrefix, string cachixUrl, bool doCheck = t Package pkg = json.packageFromNixEvalJobsJson( flakeAttrPrefix, cachixUrl); - if (doCheck)pkg = pkg.checkPackage(); + if (doCheck)pkg = pkg.checkPackage(args); result ~= pkg; @@ -405,16 +451,17 @@ SupportedSystem[] getSupportedSystems(string flakeRef = ".") return json.array.map!(system => getSystem(system.str)).array; } -Package[] nixEvalForAllSystems() +Package[] nixEvalForAllSystems(T)(auto ref T args) + if (is(T == CiMatrixArgs) || is(T == PrintTableArgs) || is(T == CiArgs) || is(T == DeploySpecArgs)) { - const cachixUrl = "https://" ~ params.cachixCache ~ ".cachix.org"; + const cachixUrl = "https://" ~ args.cachixCache ~ ".cachix.org"; const systems = getSupportedSystems(); infof("Evaluating flake for: %s", systems); return systems.map!(system => - flakeAttr(params.flakePre, system, params.flakePost) - .nixEvalJobs(cachixUrl) + flakeAttr(args.flakePre, system, args.flakePost) + .nixEvalJobs(cachixUrl, args) ) .reduce!((a, b) => a ~ b) .array @@ -422,21 +469,26 @@ Package[] nixEvalForAllSystems() .array; } -int getNixEvalWorkerCount() +int getNixEvalWorkerCount(T)(auto ref T args) + if (is(T == CiMatrixArgs) || is(T == PrintTableArgs) || is(T == CiArgs) || is(T == DeploySpecArgs)) { - return params.maxWorkers == 0 ? (threadsPerCPU() < 8 ? threadsPerCPU() : 8) : params.maxWorkers; + return args.maxWorkers == 0 ? (threadsPerCPU() < 8 ? threadsPerCPU() : 8) : args.maxWorkers; } @("getNixEvalWorkerCount") unittest { - assert(getNixEvalWorkerCount() == (threadsPerCPU() < 8 ? threadsPerCPU() : 8)); + CiMatrixArgs args; + args.maxWorkers = 0; + assert(getNixEvalWorkerCount(args) == (threadsPerCPU() < 8 ? threadsPerCPU() : 8)); + + args.maxWorkers = 4; + assert(getNixEvalWorkerCount(args) == 4); } -int getAvailableMemoryMB() +int getAvailableMemoryMB(T)(auto ref T args) + if (is(T == CiMatrixArgs) || is(T == PrintTableArgs) || is(T == CiArgs) || is(T == DeploySpecArgs)) { - - // free="$(< /proc/meminfo grep MemFree | tr -s ' ' | cut -d ' ' -f 2)" int free = "/proc/meminfo".readText .splitLines .find!(a => a.indexOf("MemFree") != -1) @@ -457,15 +509,20 @@ int getAvailableMemoryMB() .find!(a => a.indexOf("Shmem:") != -1) .front .split[1].to!int; - int maxMemoryMB = params.maxMemory == 0 ? ((free + cached + buffers + shmem) / 1024) - : params.maxMemory; + int maxMemoryMB = args.maxMemory == 0 ? ((free + cached + buffers + shmem) / 1024) + : args.maxMemory; return maxMemoryMB; } @("getAvailableMemoryMB") unittest { - assert(getAvailableMemoryMB() > 0); + CiMatrixArgs args; + args.maxMemory = 0; + assert(getAvailableMemoryMB(args) > 0); + + args.maxMemory = 1024; + assert(getAvailableMemoryMB(args) == 1024); } void saveCachixDeploySpec(Package[] packages) @@ -490,12 +547,13 @@ unittest assert(testPackageArray[1].output == deploySpec[0]["out"].str); } -void saveGHCIMatrix(Package[] packages) +void saveGHCIMatrix(T)(Package[] packages, auto ref T args) + if (is(T == CiMatrixArgs) || is(T == PrintTableArgs) || is(T == CiArgs) || is(T == DeploySpecArgs)) { auto matrix = JSONValue([ "include": JSONValue(packages.map!(pkg => pkg.toJSON()).array) ]); - string resPath = rootDir.buildPath(params.isInitial ? "matrix-pre.json" : "matrix-post.json"); + string resPath = rootDir.buildPath(args.isInitial ? "matrix-pre.json" : "matrix-post.json"); resPath.write(JSONValue(matrix).toString(JSONOptions.doNotEscapeSlashes)); } @@ -505,9 +563,11 @@ unittest import std.file : rmdirRecurse; createResultDirs(); - saveGHCIMatrix(cast(Package[]) testPackageArray); + CiMatrixArgs args; + args.isInitial = false; + saveGHCIMatrix(cast(Package[]) testPackageArray, args); JSONValue matrix = rootDir - .buildPath(params.isInitial ? "matrix-pre.json" : "matrix-post.json") + .buildPath(args.isInitial ? "matrix-pre.json" : "matrix-post.json") .readText .parseJSON; assert(testPackageArray[0].name == matrix["include"][0]["name"].str); @@ -633,24 +693,28 @@ unittest } -void printTableForCacheStatus(Package[] packages) +void printTableForCacheStatus(T)(Package[] packages, auto ref T args) + if (is(T == CiMatrixArgs) || is(T == PrintTableArgs) || is(T == CiArgs) || is(T == DeploySpecArgs)) { - if (params.precalcMatrix == "") - { - saveGHCIMatrix(packages); + static if (is(T == PrintTableArgs)) { + if (args.precalcMatrix == "") + { + saveGHCIMatrix(packages, args); + } } saveCachixDeploySpec(packages); - saveGHCIComment(convertNixEvalToTableSummary(packages, params.isInitial)); + saveGHCIComment(convertNixEvalToTableSummary(packages, args.isInitial)); } -Package checkPackage(Package pkg) +Package checkPackage(T)(Package pkg, auto ref T args) + if (is(T == CiMatrixArgs) || is(T == PrintTableArgs) || is(T == CiArgs) || is(T == DeploySpecArgs)) { import std.algorithm : canFind; import std.string : lineSplitter; import std.net.curl : HTTP, httpGet = get, HTTPStatusException; auto http = HTTP(); - http.addRequestHeader("Authorization", "Bearer " ~ params.cachixAuthToken); + http.addRequestHeader("Authorization", "Bearer " ~ args.cachixAuthToken); try { @@ -681,21 +745,20 @@ unittest cacheUrl: nixosCacheEndpoint ~ storePathHash ~ ".narinfo", ); + CiMatrixArgs args; + args.cachixAuthToken = ""; + assert(!testPackage.isCached); - assert(checkPackage(testPackage).isCached); + assert(checkPackage(testPackage, args).isCached); testPackage.cacheUrl = nixosCacheEndpoint ~ "nonexistent.narinfo"; - assert(!checkPackage(testPackage).isCached); + assert(!checkPackage(testPackage, args).isCached); } -Package[] getPrecalcMatrix() +Package[] getPrecalcMatrix(PrintTableArgs args) { - auto precalcMatrixStr = params.precalcMatrix == "" ? "{\"include\": []}" : params.precalcMatrix; - enforce!MissingEnvVarsException( - params.precalcMatrix != "", - "missing environment variables: %s".fmt("precalcMatrix") - ); + auto precalcMatrixStr = args.precalcMatrix == "" ? "{\"include\": []}" : args.precalcMatrix; return parseJSON(precalcMatrixStr)["include"].array.map!((pkg) { Package result = { name: pkg["name"].str, @@ -708,5 +771,4 @@ Package[] getPrecalcMatrix() output: pkg["output"].str}; return result; }).array; - - } +} diff --git a/packages/mcl/src/src/mcl/commands/config.d b/packages/mcl/src/src/mcl/commands/config.d index 4ead6f48..87917985 100755 --- a/packages/mcl/src/src/mcl/commands/config.d +++ b/packages/mcl/src/src/mcl/commands/config.d @@ -7,119 +7,172 @@ import std.range : drop, front; import std.stdio : writeln; import std.string : indexOf; -import mcl.utils.env : optional, parseEnv; +import argparse : Command, Description, SubCommand, Default, PositionalArgument, Placeholder, Optional, matchCmd; + import mcl.utils.fetch : fetchJson; import mcl.utils.log : errorAndExit; import mcl.utils.nix : nix, queryStorePath; import mcl.utils.process : execute; import mcl.utils.string : camelCaseToCapitalCase; -export void config(string[] args) { - if (args.length == 0) { - errorAndExit("Usage: mcl config [args]"); - } - if (!checkRepo()) { - errorAndExit("This command must be run from a repository containing a NixOS machine configuration"); - } +@(Command("config").Description("Manage NixOS machine configurations")) +struct ConfigArgs +{ + SubCommand!( + SysArgs, + HomeArgs, + StartVmArgs, + Default!UnknownCommandArgs + ) cmd; +} + +@(Command("sys").Description("Manage system configurations")) +struct SysArgs +{ + SubCommand!( + SysApplyArgs, + SysEditArgs, + Default!UnknownCommandArgs + ) cmd; +} + +@(Command("apply").Description("Apply system configuration")) +struct SysApplyArgs +{ + @(PositionalArgument(0).Placeholder("machine-name").Description("Name of the machine")) + string machineName; +} + +@(Command("edit").Description("Edit system configuration")) +struct SysEditArgs +{ + @(PositionalArgument(0).Placeholder("machine-name").Description("Name of the machine")) + string machineName; +} + +@(Command("home").Description("Manage home configurations")) +struct HomeArgs +{ + SubCommand!( + HomeApplyArgs, + HomeEditArgs, + Default!UnknownCommandArgs + ) cmd; +} + +@(Command("apply").Description("Apply user configuration")) +struct HomeApplyArgs +{ + @(PositionalArgument(0).Placeholder("desktop/server").Description("Type of home configuration")) + string type; +} + +@(Command("edit").Description("Edit user configuration")) +struct HomeEditArgs +{ + @(PositionalArgument(0).Placeholder("desktop/server").Description("Type of home configuration")) + string type; +} + +@(Command("start-vm").Description("Start a VM")) +struct StartVmArgs +{ + @(PositionalArgument(0).Optional().Placeholder("vm-name").Description("Name of the VM")) + string vmName = ""; +} + +@(Command(" ").Description(" ")) +struct UnknownCommandArgs { } + +int unknown_command(UnknownCommandArgs unused) +{ + errorAndExit("Unknown command. Use --help for a list of available commands."); + return 1; +} - string subcommand = args.front; - - switch (subcommand) { - case "sys": - sys(args.drop(1)); - break; - case "home": - home(args.drop(1)); - break; - case "start-vm": - startVM(args.drop(1)); - break; - default: - errorAndExit("Unknown config subcommand " ~ subcommand ~ ". Supported subcommands: sys, home, start-vm"); - break; +export int config(ConfigArgs args) +{ + if (!checkRepo()) + { + errorAndExit( + "This command must be run from a repository containing a NixOS machine configuration"); } + + return args.cmd.matchCmd!( + (SysArgs a) => sys(a), + (HomeArgs a) => home(a), + (StartVmArgs a) => startVM(a.vmName), + (UnknownCommandArgs a) => unknown_command(a) + ); } bool checkRepo() { - const string[] validRepos = ["nixos-machine-config", "infra-lido"]; - string remoteOriginUrl = execute(["git", "config", "--get", "remote.origin.url"], false); + const string[] validRepos = ["infra"]; + string remoteOriginUrl = execute([ + "git", "config", "--get", "remote.origin.url" + ], false); - foreach (string repo; validRepos) { - if (remoteOriginUrl.indexOf(repo) != -1) { + foreach (string repo; validRepos) + { + if (remoteOriginUrl.indexOf(repo) != -1) + { return true; } } return false; } -void executeCommand(string command) { +int executeCommand(string command) +{ auto exec = execute!ProcessPipes(command, true, false, Redirect.stderrToStdout); - wait(exec.pid); + return wait(exec.pid); } -void edit(string type, string path) { +int edit(string type, string path) +{ string editor = environment.get("EDITOR", "vim"); string user = environment.get("USER", "root"); writeln("Editing " ~ path ~ " configuration from: ", path); - final switch (type) { - case "system": - executeCommand(editor ~ " machines/*/" ~ path ~ "/*.nix"); - break; - case "user": - executeCommand(editor~ " users/" ~ user ~ "/gitconfig " ~ "users/" ~ user ~ "/*.nix " ~ "users/" ~ user ~ "/home-"~path~"/*.nix"); - break; + final switch (type) + { + case "system": + return executeCommand(editor ~ " machines/*/" ~ path ~ "/*.nix"); + case "user": + return executeCommand(editor ~ " users/" ~ user ~ "/gitconfig " ~ "users/" ~ user ~ "/*.nix " ~ "users/" ~ user ~ "/home-" ~ path ~ "/*.nix"); } } -void sys(string[] args) +int apply(string type, string value) { - if ((args.length < 1 || args.length > 2) && !["apply", "edit"].canFind(args.front)) - { - errorAndExit("Usage: mcl config sys apply or mcl config sys apply \n"~ - " mcl config sys edit or mcl config sys edit "); - } + writeln("Applying ", type, " configuration from: ", value); + return executeCommand("just switch-" ~ type ~ " " ~ value); +} - string machineName = args.length > 1 ? args[1] : ""; - final switch (args.front) { - case "apply": - writeln("Applying system configuration from: ", machineName); - executeCommand("just switch-system " ~ machineName); - break; - case "edit": - edit("system", machineName); - break; - } +int sys(SysArgs args) +{ + return args.cmd.matchCmd!( + (SysApplyArgs a) => apply("system", a.machineName), + (SysEditArgs a) => edit("system", a.machineName), + (UnknownCommandArgs a) => unknown_command(a) + ); } -void home(string[] args) +int home(HomeArgs args) { - if ((args.length != 2) && args.front != "apply") - { - errorAndExit("Usage: mcl config home apply \n"~ - " mcl config home edit "); - } - auto type = args[1]; - final switch (args.front) { - case "apply": - writeln("Applying home configuration from: ", type); - executeCommand("just switch-home " ~ type); - break; - case "edit": - edit("user", type); - break; - } + return args.cmd.matchCmd!( + (HomeApplyArgs a) { + writeln("Applying home configuration from: ", a.type); + return executeCommand("just switch-home " ~ a.type); + }, + (HomeEditArgs a) => "user".edit(a.type), + (UnknownCommandArgs a) => unknown_command(a) + ); } -void startVM(string[] args) +int startVM(string vmName) { - if (args.length != 1) - { - errorAndExit("Usage: mcl config start-vm "); - } - - string vmName = args.front; writeln("Starting VM: ", vmName); - executeCommand("just start-vm " ~ vmName); + return executeCommand("just start-vm " ~ vmName); } diff --git a/packages/mcl/src/src/mcl/commands/deploy_spec.d b/packages/mcl/src/src/mcl/commands/deploy_spec.d index 819fd4a2..ecdd76d7 100644 --- a/packages/mcl/src/src/mcl/commands/deploy_spec.d +++ b/packages/mcl/src/src/mcl/commands/deploy_spec.d @@ -5,15 +5,24 @@ import std.logger : infof, warningf; import std.file : exists; import std.path : buildPath; +import argparse : Command, Description; + import mcl.utils.process : spawnProcessInline; import mcl.utils.path : resultDir; import mcl.utils.cachix : cachixNixStoreUrl, DeploySpec, createMachineDeploySpec; import mcl.utils.tui : bold; import mcl.utils.json : tryDeserializeFromJsonFile, writeJsonFile; -import mcl.commands.ci_matrix : flakeAttr, params, nixEvalJobs, SupportedSystem; +import mcl.commands.ci_matrix : flakeAttr, nixEvalJobs, SupportedSystem,CiMatrixBaseArgs; + -export void deploy_spec(string[] args) +@(Command("deploy-spec", "deploy_spec") + .Description("Evaluate the Nixos machine configurations in bareMetalMachines and deploy them to cachix.")) +struct DeploySpecArgs { + mixin CiMatrixBaseArgs!(); +} + +export int deploy_spec(DeploySpecArgs args) { const deploySpecFile = resultDir.buildPath("cachix-deploy-spec.json"); @@ -22,7 +31,7 @@ export void deploy_spec(string[] args) if (!exists(deploySpecFile)) { auto nixosConfigs = flakeAttr("legacyPackages", SupportedSystem.x86_64_linux, "serverMachines") - .nixEvalJobs(params.cachixCache.cachixNixStoreUrl); + .nixEvalJobs(args.cachixCache.cachixNixStoreUrl, args); auto configsMissingFromCachix = nixosConfigs.filter!(c => !c.isCached); @@ -44,16 +53,18 @@ export void deploy_spec(string[] args) else { warningf("Reusing existing deploy spec at:\n'%s'", deploySpecFile.bold); - spec = deploySpecFile.tryDeserializeFromJsonFile!DeploySpec; + spec = deploySpecFile.tryDeserializeFromJsonFile!DeploySpec(); } infof("\n---\n%s\n---", spec); infof("%s machines will be deployed.", spec.agents.length); if (!spec.agents.length) - return; + return 1; spawnProcessInline([ "cachix", "deploy", "activate", deploySpecFile, "--async" ]); + + return 0; } diff --git a/packages/mcl/src/src/mcl/commands/get_fstab.d b/packages/mcl/src/src/mcl/commands/get_fstab.d index d063b7ed..270d1e70 100755 --- a/packages/mcl/src/src/mcl/commands/get_fstab.d +++ b/packages/mcl/src/src/mcl/commands/get_fstab.d @@ -6,47 +6,54 @@ import std.json : JSONValue; import std.format : fmt = format; import std.exception : enforce; +import argparse : Command, Description, NamedArgument, PositionalArgument, Required, Placeholder, EnvFallback; + import mcl.utils.cachix : cachixNixStoreUrl, getCachixDeploymentApiUrl; -import mcl.utils.env : optional, parseEnv; import mcl.utils.fetch : fetchJson; import mcl.utils.nix : queryStorePath, nix; import mcl.utils.string : camelCaseToCapitalCase; import mcl.utils.process : execute; -export void get_fstab(string[] args) +export int get_fstab(GetFstabArgs args) { - const params = parseEnv!Params; - const machineStorePath = getCachixDeploymentStorePath(params); + args.cachixStoreUrl = cachixNixStoreUrl(args.cachixCache); + if (!args.cachixDeployWorkspace) + args.cachixDeployWorkspace = args.cachixCache; + + const machineStorePath = getCachixDeploymentStorePath(args); const fstabStorePath = queryStorePath( machineStorePath, ["-etc", "-etc-fstab"], - params.cachixStoreUrl + args.cachixStoreUrl ); nix.build(fstabStorePath); writeln(fstabStorePath); + return 0; } -struct Params +@(Command("get-fstab", "get_fstab") + .Description("Get the store path of the fstab file for a deployment")) +struct GetFstabArgs { + @(NamedArgument(["cachix-auth-token"]).Required().Placeholder("XXX").Description("Auth Token for Cachix").EnvFallback("CACHIX_AUTH_TOKEN")) string cachixAuthToken; + @(NamedArgument(["cachix-cache"]).Required().Placeholder("cache").Description("Which Cachix cache to use").EnvFallback("CACHIX_CACHE")) string cachixCache; - @optional() string cachixStoreUrl; - @optional() string cachixDeployWorkspace; - string machineName; - uint deploymentId; - void setup() - { + @(NamedArgument(["cachix-store-url"]).Placeholder("https://...").Description("URL of the Cachix store")) + string cachixStoreUrl = ""; + @(NamedArgument(["cachix-deploy-workspace"]).Placeholder("agent-workspace").Description("Cachix workspace to deploy to")) + string cachixDeployWorkspace = ""; - cachixStoreUrl = cachixNixStoreUrl(cachixCache); - if (!cachixDeployWorkspace) - cachixDeployWorkspace = cachixCache; - } + @(PositionalArgument(0).Placeholder("machine-name").Description("Name of the machine")) + string machineName; + @(PositionalArgument(1).Placeholder("deployment-id").Description("ID of the deployment")) + uint deploymentId; } -string getCachixDeploymentStorePath(Params p) +string getCachixDeploymentStorePath(GetFstabArgs args) { - const url = getCachixDeploymentApiUrl(p.cachixDeployWorkspace, p.machineName, p.deploymentId); - const response = fetchJson(url, p.cachixAuthToken); + const url = getCachixDeploymentApiUrl(args.cachixDeployWorkspace, args.machineName, args.deploymentId); + const response = fetchJson(url, args.cachixAuthToken); return response["storePath"].get!string; } diff --git a/packages/mcl/src/src/mcl/commands/host_info.d b/packages/mcl/src/src/mcl/commands/host_info.d index 87270e7d..bbf53dc3 100644 --- a/packages/mcl/src/src/mcl/commands/host_info.d +++ b/packages/mcl/src/src/mcl/commands/host_info.d @@ -16,10 +16,11 @@ import std.json; import std.process : ProcessPipes, environment; import std.bitmanip : peek; import std.format : format; -import std.system : nativeEndian = endian;; +import std.system : nativeEndian = endian; import core.stdc.string : strlen; -import mcl.utils.env : parseEnv, optional; +import argparse : Command, Description, NamedArgument, Placeholder, EnvFallback; + import mcl.utils.json : toJSON; import mcl.utils.process : execute, isRoot; import mcl.utils.number : humanReadableSize; @@ -27,14 +28,6 @@ import mcl.utils.array : uniqIfSame; import mcl.utils.nix : Literal; import mcl.utils.coda : CodaApiClient, RowValues, CodaCell; -struct Params -{ - @optional() string codaApiToken; - void setup() - { - } -} - string[string] cpuinfo; string[string] meminfo; @@ -59,9 +52,18 @@ string[string] getProcInfo(string fileOrData, bool file = true) return r; } -export void host_info(string[] args) +@(Command("host-info", "host_info") + .Description("Get information about the host machine")) +struct HostInfoArgs { + @(NamedArgument(["coda-api-token"]) + .Placeholder("token") + .EnvFallback("CODA_API_TOKEN") + ) + string codaApiToken; +} + +export int host_info(HostInfoArgs args) { - const Params params = parseEnv!Params; const hostInfo = gatherHostInfo(); @@ -70,15 +72,17 @@ export void host_info(string[] args) .toPrettyString(JSONOptions.doNotEscapeSlashes) .writeln(); - if (!params.codaApiToken) { + if (!args.codaApiToken) { writeln("No Coda API token specified -> not uploading"); - return; + return 1; } writeln("Coda API token specified -> uploading"); - auto coda = CodaApiClient(params.codaApiToken); + auto coda = CodaApiClient(args.codaApiToken); coda.uploadHostInfo(hostInfo); + return 1; + } Info gatherHostInfo() diff --git a/packages/mcl/src/src/mcl/commands/machine.d b/packages/mcl/src/src/mcl/commands/machine.d index 008d0a86..07dcb775 100755 --- a/packages/mcl/src/src/mcl/commands/machine.d +++ b/packages/mcl/src/src/mcl/commands/machine.d @@ -1,11 +1,13 @@ module mcl.commands.machine; import std; + +import argparse : Command, Description, NamedArgument, PositionalArgument, Placeholder, SubCommand, Default, matchCmd; + import mcl.utils.log : prompt; import mcl.utils.process : execute; import mcl.utils.nix : nix, toNix, Literal, mkDefault; import mcl.utils.json : toJSON, fromJSON; -import mcl.utils.env : optional, parseEnv; import mcl.utils.array : uniqArrays; import mcl.commands.host_info: Info; @@ -173,22 +175,22 @@ string[] getGroups() return groups; } -User createUser() { +User createUser(CreateMachineArgs args) { - auto createUser = params.createUser || prompt!bool("Create new user"); + auto createUser = args.createUser || prompt!bool("Create new user"); if (!createUser) { string[] existingUsers = getExistingUsers(); - string userName = params.userName != "" ? params.userName : prompt!string("Select an existing username", existingUsers); + string userName = args.userName != "" ? args.userName : prompt!string("Select an existing username", existingUsers); return getUser(userName); } else { User user; - user.userName = params.userName != "" ? params.userName : prompt!string("Enter the new username"); - user.userInfo.description = params.description != "" ? params.description : prompt!string("Enter the user's description/full name"); - user.userInfo.isNormalUser = params.isNormalUser || prompt!bool("Is this a normal or root user"); - user.userInfo.extraGroups = (params.extraGroups != "" ? params.extraGroups : prompt!string("Enter the user's extra groups (comma delimited)", getGroups())).split(",").map!(strip).array; + user.userName = args.userName != "" ? args.userName : prompt!string("Enter the new username"); + user.userInfo.description = args.description != "" ? args.description : prompt!string("Enter the user's description/full name"); + user.userInfo.isNormalUser = args.isNormalUser || prompt!bool("Is this a normal or root user"); + user.userInfo.extraGroups = (args.extraGroups != "" ? args.extraGroups : prompt!string("Enter the user's extra groups (comma delimited)", getGroups())).split(",").map!(strip).array; createUserDir(user); return user; } @@ -220,8 +222,8 @@ struct MachineConfiguration MachineUserInfo users; } -void createMachine(MachineType machineType, string machineName, User user) { - auto infoJSON = execute(["ssh", params.sshPath, "sudo nix --experimental-features \\'nix-command flakes\\' --refresh --accept-flake-config run github:metacraft-labs/nixos-modules/#mcl host_info"],false, false); +void createMachine(CreateMachineArgs args, MachineType machineType, string machineName, User user) { + auto infoJSON = execute(["ssh", args.sshPath, "sudo nix --experimental-features \\'nix-command flakes\\' --refresh --accept-flake-config run github:metacraft-labs/nixos-modules/#mcl host-info"],false, false); auto infoJSONParsed = infoJSON.parseJSON; Info info = infoJSONParsed.fromJSON!Info; @@ -273,7 +275,7 @@ void createMachine(MachineType machineType, string machineName, User user) { // Disks hardwareConfiguration.disko.DISKO.makeZfsPartitions.swapSizeGB = (info.hardwareInfo.memoryInfo.totalGB.to!double*1.5).to!int; auto nvmeDevices = info.hardwareInfo.storageInfo.devices.filter!(a => a.dev.indexOf("nvme") != -1 || a.model.indexOf("SSD") != -1).array.map!(a => a.model.replace(" ", "_") ~ "_" ~ a.serial).array; - string[] disks = (nvmeDevices.length == 1 ? nvmeDevices[0] : (params.disks != "" ? params.disks : prompt!string("Enter the disks to use (comma delimited)", nvmeDevices))).split(",").map!(strip).array.map!(a => "/dev/disk/by-id/nvme-" ~ a).array; + string[] disks = (nvmeDevices.length == 1 ? nvmeDevices[0] : (args.disks != "" ? args.disks : prompt!string("Enter the disks to use (comma delimited)", nvmeDevices))).split(",").map!(strip).array.map!(a => "/dev/disk/by-id/nvme-" ~ a).array; hardwareConfiguration.disko.DISKO.makeZfsPartitions.disks = disks; hardwareConfiguration = hardwareConfiguration.uniqArrays; @@ -365,43 +367,60 @@ struct HardwareConfiguration { Services services; } -void createMachineConfiguration() +int createMachineConfiguration(CreateMachineArgs args) { checkifNixosMachineConfigRepo(); - auto machineType = cast(int)params.machineType != 0 ? params.machineType : prompt!MachineType("Machine type"); - auto machineName = params.machineName != "" ? params.machineName : prompt!string("Enter the name of the machine"); + auto machineType = cast(int)args.machineType != 0 ? args.machineType : prompt!MachineType("Machine type"); + auto machineName = args.machineName != "" ? args.machineName : prompt!string("Enter the name of the machine"); User user; - user = createUser(); - machineType.createMachine( machineName, user); + user = createUser(args); + args.createMachine(machineType, machineName, user); + return 0; } -Params params; -export void machine(string[] args) +export int machine(MachineArgs args) { - params = parseEnv!Params; - switch (args.front) - { - case "create": - createMachineConfiguration(); - break; - default: - assert(0, "Unknown machine action: " ~ args.front); - } + return args.cmd.matchCmd!( + (CreateMachineArgs a) => createMachineConfiguration(a), + (UnknownCommandArgs a) => unknown_command(a) + ); } -struct Params + +@(Command("create").Description("Create a new machine")) +struct CreateMachineArgs { + @(PositionalArgument(0).Placeholder("ssh").Description("SSH path to the machine")) string sshPath; - @optional() bool createUser; - @optional() string userName; - @optional() string machineName; - @optional() string description; - @optional() bool isNormalUser; - @optional() string extraGroups; - @optional() MachineType machineType = cast(MachineType)0; - @optional() string disks; - - void setup() - { - } + @(NamedArgument(["create-user"]).Placeholder("true/false").Description("Create a new user")) + bool createUser; + @(NamedArgument(["user-name"]).Placeholder("username").Description("Username")) + string userName; + @(NamedArgument(["machine-name"]).Placeholder("machine-name").Description("Name of the machine")) + string machineName; + @(NamedArgument(["description"]).Placeholder("description").Description("Description of the user")) + string description; + @(NamedArgument(["is-normal-user"]).Placeholder("true/false").Description("Is this a normal user")) + bool isNormalUser; + @(NamedArgument(["extra-groups"]).Placeholder("group1,group2").Description("Extra groups for the user")) + string extraGroups; + @(NamedArgument(["machine-type"]).Placeholder("desktop/server/container").Description("Type of machine")) + MachineType machineType = cast(MachineType)0; + @(NamedArgument(["disks"]).Placeholder("CT2000P3PSSD8_2402E88C1519,...").Description("Disks to use")) + string disks; +} + +@(Command(" ").Description(" ")) +struct UnknownCommandArgs { } + +int unknown_command(UnknownCommandArgs unused) +{ + stderr.writeln("Unknown machine command. Use --help for a list of available commands."); + return 1; +} + +@(Command("machine").Description("Manage machines")) +struct MachineArgs +{ + SubCommand!(CreateMachineArgs,Default!UnknownCommandArgs) cmd; } diff --git a/packages/mcl/src/src/mcl/commands/package.d b/packages/mcl/src/src/mcl/commands/package.d index 4ab52cce..0fab51ce 100644 --- a/packages/mcl/src/src/mcl/commands/package.d +++ b/packages/mcl/src/src/mcl/commands/package.d @@ -1,10 +1,26 @@ module mcl.commands; -public import mcl.commands.get_fstab : get_fstab; -public import mcl.commands.deploy_spec : deploy_spec; -public import mcl.commands.ci_matrix : ci_matrix, print_table; -public import mcl.commands.shard_matrix : shard_matrix; -public import mcl.commands.ci : ci; -public import mcl.commands.host_info : host_info; -public import mcl.commands.machine : machine; -public import mcl.commands.config : config; +import std.meta : ApplyLeft, staticMap; + +private enum commandModulesToExport = +[ + "mcl.commands.deploy_spec" : ["deploy_spec"], + "mcl.commands.ci_matrix" : ["ci_matrix", "print_table"], + "mcl.commands.shard_matrix" : ["shard_matrix"], + "mcl.commands.ci" : ["ci"], + "mcl.commands.host_info" : ["host_info"], + "mcl.commands.machine" : ["machine"], + "mcl.commands.config" : ["config"], +]; + +template ImportAll(alias aa) +{ + import std.meta : AliasSeq; + alias A = AliasSeq!(); + static foreach (moduleName, symbols; aa) + static foreach (symbolName; symbols) + A = AliasSeq!(A, __traits(getMember, imported!moduleName, symbolName)); + alias ImportAll = A; +} + +alias SubCommandFunctions = ImportAll!commandModulesToExport; diff --git a/packages/mcl/src/src/mcl/commands/shard_matrix.d b/packages/mcl/src/src/mcl/commands/shard_matrix.d index b4307844..946de5f4 100644 --- a/packages/mcl/src/src/mcl/commands/shard_matrix.d +++ b/packages/mcl/src/src/mcl/commands/shard_matrix.d @@ -13,26 +13,30 @@ import std.regex : matchFirst, regex; import std.stdio : writeln; import std.string : strip; -import mcl.utils.env : parseEnv, optional; +import argparse : Command, Description, NamedArgument, Placeholder, EnvFallback; + import mcl.utils.json : toJSON; import mcl.utils.nix : nix; import mcl.utils.path : createResultDirs, resultDir, rootDir; -export void shard_matrix(string[] args) +@(Command("shard-matrix", "shard_matrix") + .Description("Generate a shard matrix for a flake")) +struct ShardMatrixArgs { - const params = parseEnv!Params; - auto matrix = generateShardMatrix(); - saveShardMatrix(matrix, params); - + @(NamedArgument(["github-output"]) + .Placeholder("output") + .Description("Output to GitHub Actions") + .EnvFallback("GITHUB_OUTPUT") + ) + string githubOutput; } -struct Params +export int shard_matrix(ShardMatrixArgs args) { - @optional() string githubOutput; + auto matrix = generateShardMatrix(); + saveShardMatrix(matrix, args); + return 0; - void setup() - { - } } struct Shard @@ -138,15 +142,15 @@ unittest } -void saveShardMatrix(ShardMatrix matrix, Params params) +void saveShardMatrix(ShardMatrix matrix, ShardMatrixArgs args) { const matrixJson = matrix.toJSON(); const matrixString = matrixJson.toString(); infof("Shard matrix: %s", matrixJson.toPrettyString); const envLine = "gen_matrix=" ~ matrixString; - if (params.githubOutput != "") + if (args.githubOutput != "") { - params.githubOutput.append(envLine); + args.githubOutput.append(envLine); } else { diff --git a/packages/mcl/src/src/mcl/utils/env.d b/packages/mcl/src/src/mcl/utils/env.d deleted file mode 100644 index d7a9e1e8..00000000 --- a/packages/mcl/src/src/mcl/utils/env.d +++ /dev/null @@ -1,93 +0,0 @@ -module mcl.utils.env; -import mcl.utils.test; - -import std.stdio : writeln; - -struct Optional -{ -} - -auto optional() => Optional(); - -enum isOptional(alias field) = imported!`std.traits`.hasUDA!(field, Optional); - -class MissingEnvVarsException : Exception -{ - import std.exception : basicExceptionCtors; - - mixin basicExceptionCtors; -} - -T parseEnv(T)() -{ - import std.conv : to; - import std.exception : enforce; - import std.format : fmt = format; - import std.process : environment; - import std.traits : Fields; - - import mcl.utils.string : camelCaseToCapitalCase; - - T result; - string[] missingEnvVars = []; - - static foreach (idx, field; T.tupleof) - { - { - string envVarName = field.stringof.camelCaseToCapitalCase; - if (auto envVar = environment.get(envVarName)) - result.tupleof[idx] = envVar.to!(typeof(field)); - else if (!isOptional!field) - missingEnvVars ~= envVarName; - } - } - - enforce!MissingEnvVarsException( - missingEnvVars.length == 0, - "missing environment variables:\n%(* %s\n%)".fmt(missingEnvVars) - ); - - result.setup(); - - return result; -} - -version (unittest) -{ - struct Config - { - @optional() string opt; - int a; - string b; - float c = 1.0; - - void setup() - { - } - } -} - -@("parseEnv") -unittest -{ - import std.process : environment; - import std.exception : assertThrown; - - environment["A"] = "1"; - environment["B"] = "2"; - environment["C"] = "1.0"; - - auto config = parseEnv!Config; - - assert(config.a == 1); - assert(config.b == "2"); - assert(config.c == 1.0); - assert(config.opt is null); - - environment["OPT"] = "3"; - config = parseEnv!Config; - assert(config.opt == "3"); - - environment.remove("A"); - assertThrown(config = parseEnv!Config, "missing environment variables:\nA\n"); -}