diff --git a/.github/workflows/run-workspace-smoke-tests.yaml b/.github/workflows/run-workspace-smoke-tests.yaml index e314ec1be..6a99aa28c 100644 --- a/.github/workflows/run-workspace-smoke-tests.yaml +++ b/.github/workflows/run-workspace-smoke-tests.yaml @@ -15,13 +15,13 @@ on: description: Newline-separated list of plugins that failed to load value: ${{ jobs.run.outputs.failed-plugins }} error-logs: - description: Extracted error messages from container logs + description: Extracted error messages from smoke test output value: ${{ jobs.run.outputs.error-logs }} jobs: run: runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 15 permissions: contents: read packages: read @@ -36,139 +36,105 @@ jobs: name: smoke-test-artifacts path: ./artifacts - - name: Log in to GitHub Container Registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + - name: Setup Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + node-version: "24" - - name: Start RHDH with test plugins config + - name: Install skopeo run: | - set -euo pipefail - ls -la ./artifacts/ || true - + if ! command -v skopeo &>/dev/null; then + sudo apt-get update -qq + sudo apt-get install -y -qq skopeo + fi + skopeo --version + + - name: Install Python dependencies + run: pip install pyyaml + + - name: Authenticate to GHCR + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: echo $GITHUB_TOKEN | skopeo login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Verify artifacts + run: | + ls -la ./artifacts/ + ls -la ./artifacts/harness/ || true if [ ! -f "./artifacts/dynamic-plugins.test.yaml" ]; then - echo "Error: dynamic-plugins.test.yaml not found in artifacts" + echo "Error: dynamic-plugins.test.yaml not found" exit 1 fi + echo "=== dynamic-plugins.test.yaml (first 40 lines) ===" + head -40 ./artifacts/dynamic-plugins.test.yaml || true - echo "dynamic-plugins.test.yaml contents:" - sed -n '1,40p' ./artifacts/dynamic-plugins.test.yaml || true - - # Build Docker run command with conditional volume mounts - ENV_ARGS=$( [ -f ./artifacts/test.env ] && echo "--env-file ./artifacts/test.env" || echo "" ) - DOCKER_CMD="docker run -d --name rhdh -p 7007:7007 $ENV_ARGS" - if [ -f "./artifacts/app-config.yaml" ]; then - DOCKER_CMD="$DOCKER_CMD -v "$(pwd)"/artifacts/app-config.yaml:/opt/app-root/src/app-config.yaml" - fi - if [ -f "./artifacts/app-config.test.yaml" ]; then - DOCKER_CMD="$DOCKER_CMD -v "$(pwd)"/artifacts/app-config.test.yaml:/opt/app-root/src/app-config.test.yaml" - fi - DOCKER_CMD="$DOCKER_CMD -v "$(pwd)"/artifacts/dynamic-plugins.test.yaml:/opt/app-root/src/dynamic-plugins.yaml" - - # Add Docker config and environment variables - echo "Using docker auth file: $HOME/.docker/config.json" - DOCKER_CMD="$DOCKER_CMD -v $HOME/.docker/config.json:/root/.docker/config.json:ro" - DOCKER_CMD="$DOCKER_CMD -e REGISTRY_AUTH_FILE=/root/.docker/config.json" - - # Derive image tag from target branch - TARGET_BRANCH="${{ inputs.target-branch }}" - if [[ "$TARGET_BRANCH" =~ ^release-([0-9]+\.[0-9]+)$ ]]; then - IMAGE_TAG="next-${BASH_REMATCH[1]}" - else - IMAGE_TAG="next" - fi - echo "Using RHDH image tag: $IMAGE_TAG (target branch: $TARGET_BRANCH)" - - # Add image and command - DOCKER_CMD="$DOCKER_CMD --entrypoint /bin/bash quay.io/rhdh-community/rhdh:${IMAGE_TAG} -c '" - DOCKER_CMD="$DOCKER_CMD set -ex; " - DOCKER_CMD="$DOCKER_CMD PLUGINS_ROOT=/opt/app-root/src/dynamic-plugins-root; " - DOCKER_CMD="$DOCKER_CMD GENERATED_CONFIG=\$PLUGINS_ROOT/app-config.dynamic-plugins.yaml; " - DOCKER_CMD="$DOCKER_CMD INSTALL_SCRIPT=/opt/app-root/src/install-dynamic-plugins.sh; " - DOCKER_CMD="$DOCKER_CMD mkdir -p \$PLUGINS_ROOT; " - DOCKER_CMD="$DOCKER_CMD \$INSTALL_SCRIPT \$PLUGINS_ROOT; " - DOCKER_CMD="$DOCKER_CMD exec node packages/backend" - - # Add config files to command (optional) - [ -f "./artifacts/app-config.yaml" ] && DOCKER_CMD="$DOCKER_CMD --config /opt/app-root/src/app-config.yaml" - [ -f "./artifacts/app-config.test.yaml" ] && DOCKER_CMD="$DOCKER_CMD --config /opt/app-root/src/app-config.test.yaml" - DOCKER_CMD="$DOCKER_CMD --config /opt/app-root/src/dynamic-plugins.yaml" - DOCKER_CMD="$DOCKER_CMD --config \$GENERATED_CONFIG'" - - echo "Running: $DOCKER_CMD" - eval "$DOCKER_CMD" - - - name: Wait for RHDH to be ready + # TODO: Replace with native TypeScript implementation once available upstream. + - name: Download install-dynamic-plugins.py run: | - set -e - for i in $(seq 1 10); do - if curl -fsS http://localhost:7007/health >/dev/null; then - echo "RHDH is ready"; exit 0; fi - echo "Waiting for RHDH... (Attempt ${i}/10)" - # Check if container is still running - if ! docker ps | grep -q rhdh; then - echo "Container stopped unexpectedly." - exit 1 - fi - sleep 10 - done - echo "RHDH did not become ready in time." - exit 1 + curl -fsSL \ + "https://raw.githubusercontent.com/redhat-developer/rhdh/3efb9cc140ff/scripts/install-dynamic-plugins/install-dynamic-plugins.py" \ + -o ./artifacts/harness/install-dynamic-plugins.py + + - name: Download and extract plugins + working-directory: ./artifacts/harness + run: | + cp ../dynamic-plugins.test.yaml dynamic-plugins.yaml + python3 install-dynamic-plugins.py ./dynamic-plugins-root + env: + SKIP_INTEGRITY_CHECK: "true" + + - name: Install smoke test dependencies + working-directory: ./artifacts/harness + run: npm install --ignore-scripts 2>&1 | tail -5 - - name: List installed plugins - run: docker exec rhdh ls -l /opt/app-root/src/dynamic-plugins-root + - name: Run smoke test + id: smoke-test + working-directory: ./artifacts/harness + run: | + set -o pipefail + + CONFIG_ARGS="--config app-config.yaml" + [ -f ../app-config.yaml ] && CONFIG_ARGS="$CONFIG_ARGS --config ../app-config.yaml" + [ -f ../app-config.test.yaml ] && CONFIG_ARGS="$CONFIG_ARGS --config ../app-config.test.yaml" + + GENERATED_CFG="./dynamic-plugins-root/app-config.dynamic-plugins.yaml" + [ -f "$GENERATED_CFG" ] && CONFIG_ARGS="$CONFIG_ARGS --config $GENERATED_CFG" + + ENV_ARGS="" + [ -f ../test.env ] && ENV_ARGS="--env-file ../test.env" - - name: Print generated dynamic plugins config - run: docker exec rhdh cat /opt/app-root/src/dynamic-plugins-root/app-config.dynamic-plugins.yaml + echo "Running: npm test -- --plugins-yaml ../dynamic-plugins.test.yaml --skip-download $CONFIG_ARGS $ENV_ARGS" - - name: Verify plugin loading + npm test -- \ + --plugins-yaml ../dynamic-plugins.test.yaml \ + --skip-download \ + $CONFIG_ARGS \ + $ENV_ARGS \ + 2>&1 | tee ../smoke-test.log + + - name: Collect results id: collect-results + if: always() run: | - set -e - PLUGINS=$(grep -Eo '!([^[:space:]]+)' ./artifacts/dynamic-plugins.test.yaml | sed -e 's/^!//' -e 's/"$//' | sort -u) - if [ -z "$PLUGINS" ]; then - echo "No plugins found in dynamic-plugins.test.yaml" - echo "success=false" >> "$GITHUB_OUTPUT" + RESULTS_FILE="./artifacts/harness/results.json" + if [ -f "$RESULTS_FILE" ]; then + SUCCESS=$(jq -r '.success' "$RESULTS_FILE") + FAILED=$(jq -r '.failedPlugins | join("\n")' "$RESULTS_FILE") + echo "success=$SUCCESS" >> "$GITHUB_OUTPUT" echo "failed-plugins<> "$GITHUB_OUTPUT" - echo "(no-plugins)" >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" - exit 0 - fi - LOGS=$(docker logs rhdh || true) - failures=() - echo "===== Checking logs for plugin loaded messages" - for plugin in $PLUGINS; do - echo "Asserting plugin loaded: $plugin" - # Match either the unpack path or the canonical "loaded dynamic ... plugin '' from" message - if echo "$LOGS" | grep -qiE "loaded dynamic .* plugin .*${plugin}(-dynamic)?'" ; then - echo "Plugin loaded: $plugin" - else - echo "Plugin NOT loaded: $plugin" - failures+=("$plugin") - fi - done - if echo "$LOGS" | grep -E "(InstallException|Error while adding OCI plugin|Failed to load dynamic plugin|dynamic plugin.*error)" >/dev/null; then - echo "Detected dynamic plugin loading errors in logs" - failures+=("(log-errors)") - fi - if [ ${#failures[@]} -eq 0 ]; then - echo "success=true" >> "$GITHUB_OUTPUT" - echo "failed-plugins<> "$GITHUB_OUTPUT" - echo "" >> "$GITHUB_OUTPUT" + echo "$FAILED" >> "$GITHUB_OUTPUT" echo "EOF" >> "$GITHUB_OUTPUT" else echo "success=false" >> "$GITHUB_OUTPUT" echo "failed-plugins<> "$GITHUB_OUTPUT" - printf "%s\n" "${failures[@]}" >> "$GITHUB_OUTPUT" + echo "(results-file-missing)" >> "$GITHUB_OUTPUT" echo "EOF" >> "$GITHUB_OUTPUT" fi - - name: Fail if any plugin failed to load + - name: Fail if smoke test failed if: ${{ steps.collect-results.outputs.success != 'true' }} run: | - echo "The following plugins failed to load to RHDH:" + echo "Smoke test failed. Failed plugins:" echo "${{ steps.collect-results.outputs.failed-plugins }}" exit 1 @@ -177,30 +143,21 @@ jobs: id: capture-errors continue-on-error: true run: | - if ! docker ps -a | grep -q rhdh; then - echo "error-logs<> "$GITHUB_OUTPUT" - echo "'rhdh' container was not available. Check earlier steps for startup errors." >> "$GITHUB_OUTPUT" - echo "EOF" >> "$GITHUB_OUTPUT" - exit 0 - fi - - LOGS=$(docker logs rhdh 2>&1 || echo "Failed to retrieve logs") - - # Extract errors from logs with short context - ERROR_LINES=$(echo "$LOGS" | grep -iE -B 2 -A 2 "(error|exception|failed|failure|installException)" | grep -vE "(no errors|successfully|resolved|fixed)" | grep -v "^--$" | tail -n 100 || echo "") - + LOG_FILE="./artifacts/smoke-test.log" echo "error-logs<> "$GITHUB_OUTPUT" - if [ -n "$ERROR_LINES" ] && [ "$ERROR_LINES" != "" ]; then - echo "$ERROR_LINES" >> "$GITHUB_OUTPUT" + if [ -f "$LOG_FILE" ]; then + grep -iE -B 1 -A 1 "(error|exception|failed|failure)" "$LOG_FILE" \ + | grep -vE "(no errors|successfully|resolved)" \ + | tail -100 || echo "No error patterns found in logs." else - echo "No specific error patterns found in container logs. Check full workflow logs for details." >> "$GITHUB_OUTPUT" + echo "No smoke test log available." fi echo "EOF" >> "$GITHUB_OUTPUT" - - name: Print container logs + - name: Print full smoke test log if: always() - run: docker logs rhdh || true + run: cat ./artifacts/smoke-test.log 2>/dev/null || echo "No log file" - name: Cleanup if: always() - run: docker rm -f rhdh || true + run: rm -rf ./artifacts/harness/dynamic-plugins-root || true diff --git a/.github/workflows/workspace-tests.yaml b/.github/workflows/workspace-tests.yaml index 13bf63c0c..3c10cbc5e 100644 --- a/.github/workflows/workspace-tests.yaml +++ b/.github/workflows/workspace-tests.yaml @@ -88,7 +88,7 @@ jobs: console.log('Missing PR or commit; skipping pending status'); return; } - + await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, @@ -174,7 +174,7 @@ jobs: # Always include the root-level default config ROOT_CONFIG="$GITHUB_WORKSPACE/smoke-tests/app-config.yaml" [ -f "$ROOT_CONFIG" ] && cp "$ROOT_CONFIG" "$OUT_DIR/app-config.yaml" - + # Read workspace-wide test.env if it exists WORKSPACE_ENV_FILE="$WORKSPACE_PATH/smoke-tests/test.env" WORKSPACE_ENV_CONTENT="" @@ -278,6 +278,16 @@ jobs: echo "plugins-metadata-complete=$PLUGINS_METADATA_COMPLETE" >> "$GITHUB_OUTPUT" echo "skip-tests-missing-env=$SKIP_TESTS_MISSING_ENV" >> "$GITHUB_OUTPUT" + - name: Stage smoke test harness + env: + WORKSPACE_PATH: ${{ needs.resolve.outputs.workspace }} + run: | + HARNESS_DIR="$WORKSPACE_PATH/smoke-tests/harness" + mkdir -p "$HARNESS_DIR" + cp smoke-tests/package.json "$HARNESS_DIR/" + cp smoke-tests/smoke-test.mjs "$HARNESS_DIR/" + cp smoke-tests/app-config.yaml "$HARNESS_DIR/" + - name: Upload smoke test artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -321,11 +331,11 @@ jobs: const workspace = core.getInput('workspace'); const pluginsMetadataComplete = core.getInput('plugins_metadata_complete') === 'true'; const skipTestsMissingEnv = core.getInput('skip_tests_missing_env') === 'true'; - + let statusDescription = 'Skipped'; let commentDetail = ' skipped for an unknown reason. Check workflow run for details.\n'; let summaryDetail = 'Unknown reason. Check workflow run for details.'; - + if (!workspace || workspace === '') { statusDescription = 'Skipped: PR doesn\'t touch one workspace'; commentDetail = ' skipped: PR doesn\'t touch exactly one workspace.\n'; @@ -339,7 +349,7 @@ jobs: commentDetail = ' skipped: missing plugin metadata files (`/metadata/*.yaml`).\n'; summaryDetail = 'Missing plugin metadata files (`/metadata/*.yaml`).'; } - + if (overlayCommit) { await github.rest.repos.createCommitStatus({ owner: context.repo.owner, @@ -352,7 +362,7 @@ jobs: }); console.log(`Set success status on ${overlayCommit} (skipped for valid reason)`); } - + if (pr) { await github.rest.issues.createComment({ owner: context.repo.owner, @@ -361,7 +371,7 @@ jobs: body: `:warning: \n[Smoke test workflow](${runUrl})${commentDetail}`, }); } - + await core.summary .addRaw('\n### Tests Skipped\n\n' + summaryDetail) .write(); @@ -400,10 +410,10 @@ jobs: const errorLogs = (process.env.ERROR_LOGS || '').trim(); const pr = Number(process.env.PR_NUMBER); const overlayCommit = process.env.OVERLAY_COMMIT; - + const success = smokeTestsResult === 'success' && successOutput === 'true'; let failureReason = ''; - + if (!success) { switch (smokeTestsResult) { case 'failure': @@ -419,7 +429,7 @@ jobs: failureReason = `Smoke tests ended in an unexpected state: ${smokeTestsResult}. Check the workflow logs for details.`; } } - + // Write step summary (always, even if PR is unavailable) let summary; if (success) { @@ -434,23 +444,23 @@ jobs: } } await core.summary.addRaw(summary).write(); - + if (!pr) { console.log('No PR associated; skipping status and comment'); return; } - + // Get current PR head SHA const { data: prData } = await github.rest.pulls.get({ owner: context.repo.owner, repo: context.repo.repo, pull_number: pr, }); - + // Use PR head if different from overlayCommit (/smoketest command), else use overlayCommit (immediate publish) const sha = prData.head.sha !== overlayCommit ? prData.head.sha : overlayCommit; console.log(`Status SHA: ${sha} (PR head: ${prData.head.sha}, overlay: ${overlayCommit})`); - + await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, @@ -460,7 +470,7 @@ jobs: target_url: runUrl, context: 'smoketest', }); - + let body; if (success) { body = `:white_check_mark: [Smoke tests workflow](${runUrl}) passed. All plugins loaded successfully.\n`; @@ -473,7 +483,7 @@ jobs: body += `\n\n
Error logs from container\n\n\`\`\`\n${errorLogs}\n\`\`\`\n\n
`; } } - + await github.rest.issues.createComment({ issue_number: pr, owner: context.repo.owner, diff --git a/.gitignore b/.gitignore index d4222298f..dff9f5ab2 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,11 @@ Thumbs.db build/ *.tsbuildinfo +# Smoke test artifacts +smoke-tests/dynamic-plugins-root/ +smoke-tests/.tmp-oci/ +smoke-tests/results.json + # Coverage coverage/ -.nyc_output/ \ No newline at end of file +.nyc_output/ diff --git a/smoke-tests/app-config.yaml b/smoke-tests/app-config.yaml index 57711a7b4..ffe09ec52 100644 --- a/smoke-tests/app-config.yaml +++ b/smoke-tests/app-config.yaml @@ -1,46 +1,5 @@ -app: - title: Backstage Test App - baseUrl: http://localhost:7007 - -backend: - baseUrl: http://localhost:7007 - listen: - port: 7007 - database: - client: better-sqlite3 - connection: ':memory:' - csp: - connect-src: ["'self'", 'http:', 'https:'] - cors: - origin: http://localhost:7007 - methods: [GET, POST, PUT, DELETE] - credentials: true - -auth: - environment: development - providers: - guest: {} - -# Minimal catalog configuration. -catalog: - rules: - - allow: [Component, API, Group, User, Template, Location] - locations: [] - -# This is required by the frontend app plugin for schema validation. dynamicPlugins: - # This directory is used by the backend to unpack OCI plugins. - rootDirectory: /opt/app-root/src/dynamic-plugins-root - # Disable the default directory scanning for local plugins + rootDirectory: ./dynamic-plugins-root sources: local: {} - # A dummy frontend config is sufficient for our backend-only test. frontend: {} - -# Add minimal analytics key to satisfy app bundle schema validation. -analytics: {} - -# Disable the permission plugin for simplified testing. -permission: - enabled: false - diff --git a/smoke-tests/global.d.ts b/smoke-tests/global.d.ts new file mode 100644 index 000000000..2244ba98a --- /dev/null +++ b/smoke-tests/global.d.ts @@ -0,0 +1,4 @@ +declare module "@backstage/backend-test-utils"; +declare module "@backstage/backend-dynamic-feature-service"; +declare module "@backstage/cli-node"; +declare module "@backstage/backend-plugin-api"; diff --git a/smoke-tests/package.json b/smoke-tests/package.json new file mode 100644 index 000000000..9bb53ac44 --- /dev/null +++ b/smoke-tests/package.json @@ -0,0 +1,26 @@ +{ + "name": "rhdh-smoke-test", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Lightweight smoke test harness for RHDH dynamic plugins — no Docker required", + "scripts": { + "test": "node smoke-test.ts", + "prettier:check": "prettier --check .", + "prettier:fix": "prettier --write ." + }, + "dependencies": { + "@backstage/backend-dynamic-feature-service": "0.7.9", + "@backstage/backend-plugin-api": "1.7.0", + "@backstage/backend-test-utils": "1.4.0", + "@backstage/cli-node": "0.2.18", + "@backstage/config": "1.3.6", + "@backstage/config-loader": "1.10.8", + "@backstage/errors": "1.2.7", + "@backstage/types": "1.2.2", + "js-yaml": "4.1.0" + }, + "devDependencies": { + "prettier": "3.8.1" + } +} diff --git a/smoke-tests/smoke-test.ts b/smoke-tests/smoke-test.ts new file mode 100644 index 000000000..cb98fe50a --- /dev/null +++ b/smoke-tests/smoke-test.ts @@ -0,0 +1,694 @@ +#!/usr/bin/env node + +import { readFileSync, writeFileSync, existsSync, readdirSync } from "node:fs"; +import { join, resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs as parseCliArgs } from "node:util"; +import yaml from "js-yaml"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +type ResultStatus = + | "skip" + | "pass" + | "warn" + | "fail" + | "fail-bundle" + | "fail-load"; + +type CountBuckets = { + pass: number; + fail: number; + warn?: number; + skip?: number; + [key: string]: number | undefined; +}; + +type CliArgs = { + pluginsYaml: string; + configs: string[]; + envFile: string | null; + pluginsRoot: string; + resultsFile: string; + skipDownload: boolean; +}; + +type PluginEntry = { + package: string; + disabled?: boolean; +}; + +type PluginsDoc = { + plugins?: PluginEntry[]; +}; + +type OciRef = { + imageRef: string; + pluginPath: string | null; +}; + +type PluginMeta = { + pkgName: string; + role: string; + pluginId: string | null; +}; + +type ProbeResult = { + pkgName: string; + role: string; + pluginPath: string; + status: ResultStatus; + pluginId?: string; + http?: number; + detail?: string; + error?: string; +}; + +type LoadedPlugin = { + name?: string; + platform?: string; + failure?: string; +}; + +function toErrorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +function parseArgs(argv: string[]): CliArgs { + const { values } = parseCliArgs({ + args: argv, + allowPositionals: false, + options: { + "plugins-yaml": { type: "string" }, + config: { type: "string", multiple: true }, + "env-file": { type: "string" }, + "skip-download": { type: "boolean", default: false }, + }, + }); + + const pluginsYaml = values["plugins-yaml"] + ? resolve(values["plugins-yaml"]) + : null; + if (!pluginsYaml) { + console.error( + "Usage: node smoke-test.ts --plugins-yaml [--config ...] [--env-file ] [--skip-download]", + ); + process.exit(1); + } + + return { + pluginsYaml, + configs: (values.config ?? []).map((configPath) => resolve(configPath)), + envFile: values["env-file"] ? resolve(values["env-file"]) : null, + pluginsRoot: resolve(__dirname, "dynamic-plugins-root"), + resultsFile: resolve(__dirname, "results.json"), + skipDownload: values["skip-download"] ?? false, + }; +} + +function loadEnvFile(filePath: string | null): void { + if (!filePath || !existsSync(filePath)) return; + for (const line of readFileSync(filePath, "utf8").split("\n")) { + const t = line.trim(); + if (!t || t.startsWith("#")) continue; + const eq = t.indexOf("="); + if (eq === -1) continue; + process.env[t.slice(0, eq).trim()] = t.slice(eq + 1).trim(); + } +} + +// --------------------------------------------------------------------------- +// Plugins YAML +// --------------------------------------------------------------------------- + +function parsePluginsYaml(filePath: string): PluginEntry[] { + const doc = yaml.load(readFileSync(filePath, "utf8")) as PluginsDoc | undefined; + return (doc?.plugins ?? []).filter((p) => !p.disabled); +} + +function parseOciRef(packageStr: string): OciRef { + const cleaned = packageStr.replace(/^"/, "").replace(/"$/, ""); + const withoutOci = cleaned.replace(/^oci:\/\//, ""); + const bangIdx = withoutOci.indexOf("!"); + if (bangIdx === -1) return { imageRef: withoutOci, pluginPath: null }; + return { + imageRef: withoutOci.slice(0, bangIdx), + pluginPath: withoutOci.slice(bangIdx + 1), + }; +} + +const FRONTEND_ROLES = new Set(["frontend-plugin", "frontend-plugin-module"]); + +function isFrontendRole(role: string): boolean { + return FRONTEND_ROLES.has(role); +} + +// --------------------------------------------------------------------------- +// Frontend bundle validation (Layer 1) +// --------------------------------------------------------------------------- + +function findJsFiles(dir: string): boolean { + const entries = readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + if (findJsFiles(full)) return true; + } else if (/\.(js|mjs|cjs)$/.test(entry.name)) { + return true; + } + } + return false; +} + +function validateFrontendBundles( + plugins: PluginEntry[], + pluginsRoot: string, +): ProbeResult[] { + const results: ProbeResult[] = []; + for (const plugin of plugins) { + const { pluginPath } = parseOciRef(plugin.package); + if (!pluginPath) continue; + + const { pkgName, role } = readPluginMeta(pluginsRoot, pluginPath); + if (!isFrontendRole(role)) continue; + + const scalprumDir = join(pluginsRoot, pluginPath, "dist-scalprum"); + + if (!existsSync(scalprumDir)) { + results.push({ + pkgName, + role, + pluginPath, + status: "fail-bundle", + detail: "dist-scalprum/ directory missing", + }); + continue; + } + + if (!findJsFiles(scalprumDir)) { + results.push({ + pkgName, + role, + pluginPath, + status: "fail-bundle", + detail: "dist-scalprum/ contains no .js/.mjs/.cjs files", + }); + continue; + } + + results.push({ pkgName, role, pluginPath, status: "pass" }); + } + return results; +} + +// --------------------------------------------------------------------------- +// Backend boot (startTestBackend + probe plugin) +// --------------------------------------------------------------------------- + +async function bootBackend(configPaths: string[]): Promise<{ server: any; port: number }> { + const { startTestBackend } = await import("@backstage/backend-test-utils"); + const { + dynamicPluginsFeatureLoader, + CommonJSModuleLoader, + dynamicPluginsFrontendServiceRef, + dynamicPluginsServiceRef, + } = await import("@backstage/backend-dynamic-feature-service"); + const { PackageRoles } = await import("@backstage/cli-node"); + const { createServiceFactory, createBackendPlugin, coreServices } = + await import("@backstage/backend-plugin-api"); + const path = await import("node:path"); + + const smokeTestProbePlugin = createBackendPlugin({ + pluginId: "smoke-test-probe", + register(env: any) { + env.registerInit({ + deps: { + http: coreServices.httpRouter, + dynamicPlugins: dynamicPluginsServiceRef, + }, + async init({ http, dynamicPlugins }: { http: any; dynamicPlugins: any }) { + const { Router } = await import("express"); + const router = Router(); + router.get("/loaded-plugins", (_, res) => { + res.json(dynamicPlugins.plugins({ includeFailed: true })); + }); + http.use(router); + try { + http.addAuthPolicy({ + path: "/loaded-plugins", + allow: "unauthenticated", + }); + } catch { + /* API may not exist on this version */ + } + }, + }); + }, + }); + + const baseArgv = [...process.argv]; + const configArgv = configPaths.flatMap((configPath) => ["--config", configPath]); + process.argv = [...baseArgv, ...configArgv]; + + let server; + try { + ({ server } = await startTestBackend({ + features: [ + dynamicPluginsFeatureLoader({ + schemaLocator(pluginPackage: any) { + const platform = PackageRoles.getRoleInfo( + pluginPackage.manifest.backstage.role, + ).platform; + return path.join( + platform === "node" ? "dist" : "dist-scalprum", + "configSchema.json", + ); + }, + moduleLoader: (logger: any) => new CommonJSModuleLoader({ logger }), + }), + createServiceFactory({ + service: dynamicPluginsFrontendServiceRef, + deps: {}, + factory: () => ({ setResolverProvider() {} }), + }), + smokeTestProbePlugin, + ], + })); + } finally { + process.argv = baseArgv; + } + + const addr = server.address(); + const port = typeof addr === "object" ? addr.port : 7007; + return { server, port }; +} + +// --------------------------------------------------------------------------- +// Plugin metadata & route probing +// --------------------------------------------------------------------------- + +function readPluginMeta(pluginsRoot: string, pluginPath: string): PluginMeta { + try { + const pkg = JSON.parse( + readFileSync(join(pluginsRoot, pluginPath, "package.json"), "utf8"), + ); + return { + pkgName: pkg.name ?? pluginPath, + role: pkg.backstage?.role ?? "unknown", + pluginId: pkg.backstage?.pluginId ?? null, + }; + } catch { + return { pkgName: pluginPath, role: "unknown", pluginId: null }; + } +} + +async function probePluginRoutes( + plugins: PluginEntry[], + port: number, + pluginsRoot: string, +): Promise { + const results: ProbeResult[] = []; + for (const plugin of plugins) { + const { pluginPath } = parseOciRef(plugin.package); + if (!pluginPath) continue; + + const { pkgName, role, pluginId } = readPluginMeta(pluginsRoot, pluginPath); + + if (isFrontendRole(role)) continue; + + if (role !== "backend-plugin") { + results.push({ pkgName, role, pluginPath, status: "skip" }); + continue; + } + + if (!pluginId) { + results.push({ + pkgName, + role, + pluginPath, + status: "warn", + http: 0, + pluginId: "(unknown)", + }); + continue; + } + + try { + const res = await fetch(`http://localhost:${port}/api/${pluginId}`); + results.push({ + pkgName, + role, + pluginPath, + pluginId, + status: res.status === 404 ? "warn" : "pass", + http: res.status, + }); + } catch (err) { + results.push({ + pkgName, + role, + pluginPath, + pluginId, + status: "fail", + error: toErrorMessage(err), + }); + } + } + return results; +} + +// --------------------------------------------------------------------------- +// Frontend plugin probing (Layer 2) +// --------------------------------------------------------------------------- + +async function probeFrontendPlugins( + plugins: PluginEntry[], + port: number, + pluginsRoot: string, +): Promise { + const frontendPlugins: Array = []; + for (const plugin of plugins) { + const { pluginPath } = parseOciRef(plugin.package); + if (!pluginPath) continue; + const meta = readPluginMeta(pluginsRoot, pluginPath); + if (isFrontendRole(meta.role)) { + frontendPlugins.push({ ...meta, pluginPath }); + } + } + + if (frontendPlugins.length === 0) return []; + + const failAll = (detail: string): ProbeResult[] => + frontendPlugins.map((fp) => ({ + pkgName: fp.pkgName, + role: fp.role, + pluginPath: fp.pluginPath, + status: "fail-load", + detail, + })); + + let res; + try { + res = await fetch( + `http://localhost:${port}/api/smoke-test-probe/loaded-plugins`, + ); + } catch (err) { + return failAll(`probe endpoint unreachable: ${toErrorMessage(err)}`); + } + + if (!res.ok) { + return failAll(`probe returned HTTP ${res.status}`); + } + + let body; + try { + body = await res.json(); + } catch { + return failAll("invalid probe response"); + } + + if (!Array.isArray(body)) { + return failAll("invalid probe response"); + } + + const toFrontendProbeResult = ( + fp: PluginMeta & { pluginPath: string }, + loaded: LoadedPlugin | undefined, + ): ProbeResult => { + if (!loaded) { + return { + pkgName: fp.pkgName, + role: fp.role, + pluginPath: fp.pluginPath, + status: "fail-load", + detail: "not found in loaded plugins list", + }; + } + if (loaded.platform !== "web") { + return { + pkgName: fp.pkgName, + role: fp.role, + pluginPath: fp.pluginPath, + status: "fail-load", + detail: `unexpected platform: ${loaded.platform}`, + }; + } + if (loaded.failure) { + return { + pkgName: fp.pkgName, + role: fp.role, + pluginPath: fp.pluginPath, + status: "fail-load", + detail: `plugin loaded with failure: ${loaded.failure}`, + }; + } + return { + pkgName: fp.pkgName, + role: fp.role, + pluginPath: fp.pluginPath, + status: "pass", + }; + }; + + const results: ProbeResult[] = []; + for (const fp of frontendPlugins) { + const loadedCandidate = body.find( + (lp: unknown) => + lp && typeof lp === "object" && "name" in lp && (lp as LoadedPlugin).name === fp.pkgName, + ); + const loaded = + loadedCandidate && typeof loadedCandidate === "object" + ? (loadedCandidate as LoadedPlugin) + : undefined; + results.push(toFrontendProbeResult(fp, loaded)); + } + return results; +} + +// --------------------------------------------------------------------------- +// Reporting +// --------------------------------------------------------------------------- + +function logPassResult(result: ProbeResult): string | null { + if (result.pluginId) { + return ` PASS ${result.pkgName} → /api/${result.pluginId} (${result.http})`; + } + if (isFrontendRole(result.role)) { + return ` PASS ${result.pkgName} (${result.role})`; + } + return null; +} + +function logResultAndCollectFailures(result: ProbeResult, failedPlugins: string[]): void { + const statusHandlers: Partial< + Record { line: string | null; failed?: boolean }> + > = { + skip: () => ({ line: ` SKIP ${result.pkgName} (${result.role})` }), + pass: () => ({ line: logPassResult(result) }), + warn: () => ({ + line: ` WARN ${result.pkgName} → /api/${result.pluginId} (404 — pluginId guess may be wrong)`, + }), + "fail-bundle": () => ({ + line: ` FAIL ${result.pkgName} [bundle] ${result.detail}`, + failed: true, + }), + "fail-load": () => ({ + line: ` FAIL ${result.pkgName} [load] ${result.detail}`, + failed: true, + }), + }; + + const handled = statusHandlers[result.status]?.() ?? { + line: ` FAIL ${result.pkgName} ${result.error}`, + failed: true, + }; + + if (handled.line) { + console.log(handled.line); + } + if (handled.failed) { + failedPlugins.push(result.pkgName); + } +} + +function updateResultCounts( + result: ProbeResult, + backendCounts: CountBuckets, + frontendCounts: CountBuckets, +): void { + const isFrontend = isFrontendRole(result.role); + if (result.status === "fail-bundle" || result.status === "fail-load") { + if (isFrontend) frontendCounts.fail++; + else backendCounts.fail++; + return; + } + + if (isFrontend) { + frontendCounts[result.status] = (frontendCounts[result.status] ?? 0) + 1; + return; + } + + backendCounts[result.status] = (backendCounts[result.status] ?? 0) + 1; +} + +function reportAndWrite(results: ProbeResult[], resultsFile: string): boolean { + console.log("\n========== Smoke Test Results ==========\n"); + const failedPlugins: string[] = []; + + for (const r of results) { + logResultAndCollectFailures(r, failedPlugins); + } + + const be = { pass: 0, warn: 0, skip: 0, fail: 0 }; + const fe = { pass: 0, fail: 0 }; + for (const r of results) { + updateResultCounts(r, be, fe); + } + const total = results.length; + const totalFail = be.fail + fe.fail; + console.log( + `\n Total: ${total} Backend: ${be.pass} pass / ${be.warn} warn / ${be.fail} fail / ${be.skip} skip Frontend: ${fe.pass} pass / ${fe.fail} fail\n`, + ); + + const success = totalFail === 0; + writeFileSync( + resultsFile, + JSON.stringify({ success, failedPlugins, results }, null, 2), + ); + return success; +} + +// --------------------------------------------------------------------------- +// Result merging +// --------------------------------------------------------------------------- + +function mergeFrontendResults( + bundleResults: ProbeResult[], + loadResults: ProbeResult[], +): ProbeResult[] { + const loadMap = new Map(); + for (const r of loadResults) { + if (!loadMap.has(r.pkgName)) loadMap.set(r.pkgName, r); + } + + return bundleResults.map((br) => { + if (br.status === "fail-bundle") return br; + + const lr = loadMap.get(br.pkgName); + if (!lr) { + return { + ...br, + status: "fail-load", + detail: "missing load probe result", + }; + } + if (lr.status === "fail-load") return lr; + return { ...br, status: "pass" }; + }); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + + console.log("\n=== RHDH Smoke Test (Docker-free) ===\n"); + + console.log("1. Loading plugin configuration"); + const plugins = parsePluginsYaml(args.pluginsYaml); + if (!plugins.length) { + console.log(" No enabled plugins found."); + process.exit(0); + } + console.log(` ${plugins.length} plugin(s) enabled`); + + loadEnvFile(args.envFile); + + if (!args.skipDownload) { + console.log( + "\n2. Plugin download expected via install-dynamic-plugins.py pre-step", + ); + } else { + console.log("\n2. Skipping download (--skip-download)"); + } + + if (!existsSync(args.pluginsRoot)) { + console.error( + ` ERROR: plugins root directory not found: ${args.pluginsRoot}`, + ); + console.error( + " Run install-dynamic-plugins.py first, or use --skip-download with a pre-populated directory.", + ); + process.exit(1); + } + + const generatedCfg = join( + args.pluginsRoot, + "app-config.dynamic-plugins.yaml", + ); + if (!existsSync(generatedCfg)) { + console.warn( + ` WARN: ${generatedCfg} not found — install-dynamic-plugins.py may not have been run`, + ); + } + + console.log("\n2b. Validating frontend bundles"); + const bundleResults = validateFrontendBundles(plugins, args.pluginsRoot); + const bundleFailCount = bundleResults.filter( + (r) => r.status === "fail-bundle", + ).length; + console.log( + ` ${bundleResults.length} frontend plugin(s) checked, ${bundleFailCount} failed`, + ); + + console.log("\n3. Booting Backstage backend (startTestBackend)"); + const allConfigPaths = [...args.configs]; + if (existsSync(generatedCfg)) allConfigPaths.push(generatedCfg); + const configPaths = allConfigPaths.filter((configPath) => { + if (existsSync(configPath)) return true; + console.warn(` WARN: config file not found, skipping: ${configPath}`); + return false; + }); + const { server, port } = await bootBackend(configPaths); + + let success = false; + try { + console.log("\n4a. Probing backend plugin routes"); + const backendResults = await probePluginRoutes( + plugins, + port, + args.pluginsRoot, + ); + + console.log("\n4b. Probing frontend loaded plugins"); + const frontendLoadResults = await probeFrontendPlugins( + plugins, + port, + args.pluginsRoot, + ); + + const frontendResults = mergeFrontendResults( + bundleResults, + frontendLoadResults, + ); + + const allResults = [...backendResults, ...frontendResults]; + success = reportAndWrite(allResults, args.resultsFile); + } finally { + console.log("Shutting down backend..."); + server.close(); + } + + process.exit(success ? 0 : 1); +} + +await main().catch((err: unknown) => { + console.error("\nSmoke test failed:", toErrorMessage(err)); + process.exit(1); +});