diff --git a/docs/public/SUMMARY.md b/docs/public/SUMMARY.md index 51f211c5..b840f2d7 100644 --- a/docs/public/SUMMARY.md +++ b/docs/public/SUMMARY.md @@ -26,6 +26,7 @@ * [Configuration](operations/configuration.md) * [Authentication](operations/authentication.md) * [Backup & Restore](operations/backup-restore.md) +* [GitOps](operations/gitops.md) * [Security](operations/security.md) * [Upgrading](operations/upgrading.md) diff --git a/docs/public/operations/gitops.md b/docs/public/operations/gitops.md new file mode 100644 index 00000000..941f69ab --- /dev/null +++ b/docs/public/operations/gitops.md @@ -0,0 +1,114 @@ +# GitOps (Pipeline-as-Code) + +VectorFlow supports **pipeline-as-code** workflows where pipeline configurations are stored in a Git repository and kept in sync between VectorFlow and your version control system. + +## Modes + +Each environment can operate in one of three GitOps modes: + +| Mode | Direction | Description | +|------|-----------|-------------| +| **Off** | -- | Git integration is disabled (default). | +| **Push Only** | VectorFlow -> Git | Pipeline YAML is committed to the repo whenever you deploy or delete a pipeline. The repo serves as an audit trail. | +| **Bi-directional** | VectorFlow <-> Git | In addition to push, a webhook from GitHub triggers VectorFlow to import changed YAML files automatically. | + +## Setting up Push Only + +Push-only mode commits pipeline YAML files to your Git repository every time you deploy or delete a pipeline. + +{% stepper %} +{% step %} +### Configure Git Integration +On the environment detail page, fill in the **Git Integration** card: +- **Repository URL** -- HTTPS URL of the target repo (e.g., `https://github.com/org/pipeline-configs.git`) +- **Branch** -- The branch to push to (default: `main`) +- **Access Token** -- A personal access token with write access +{% endstep %} +{% step %} +### Set GitOps Mode to Push Only +In the **GitOps Mode** dropdown, select **Push Only**. +{% endstep %} +{% step %} +### Save +Click **Save**. You can verify connectivity with **Test Connection** before saving. +{% endstep %} +{% endstepper %} + +From this point forward, every pipeline deploy writes the generated YAML to `{environment-name}/{pipeline-name}.yaml` in the configured repository, and every pipeline deletion removes the file. + +{% hint style="info" %} +Git sync is a post-deploy side effect. If the Git push fails, the pipeline deploy still succeeds -- you will see a warning in the VectorFlow logs. +{% endhint %} + +## Setting up Bi-directional GitOps + +Bi-directional mode adds a webhook so that pushes to the Git repository automatically import or update pipelines in VectorFlow. + +{% stepper %} +{% step %} +### Configure Git Integration +On the environment detail page, fill in the **Repository URL**, **Branch**, and **Access Token** in the Git Integration card. +{% endstep %} +{% step %} +### Set GitOps Mode to Bi-directional +Select **Bi-directional** from the **GitOps Mode** dropdown and click **Save**. VectorFlow auto-generates a webhook secret. +{% endstep %} +{% step %} +### Copy the webhook details +After saving, the card shows: +- **Webhook URL** -- The endpoint GitHub should send push events to. +- **Webhook Secret** -- The HMAC secret for signature verification. +{% endstep %} +{% step %} +### Create a GitHub Webhook +In your GitHub repository, go to **Settings > Webhooks > Add webhook** and enter: +- **Payload URL** -- Paste the Webhook URL from VectorFlow. +- **Content type** -- Select `application/json`. +- **Secret** -- Paste the Webhook Secret from VectorFlow. +- **Events** -- Select **Just the push event**. + +Click **Add webhook**. +{% endstep %} +{% endstepper %} + +{% tabs %} +{% tab title="GitHub" %} +Navigate to your repository on GitHub, then go to **Settings > Webhooks > Add webhook**. Fill in the Payload URL, select `application/json`, paste the secret, and choose the push event. +{% endtab %} +{% tab title="GitLab" %} +GitLab uses a different header (`X-Gitlab-Token`) for secret verification. GitLab support is not yet available -- contact the team if you need it. +{% endtab %} +{% endtabs %} + +## How the import works + +When a push event arrives: + +1. VectorFlow verifies the HMAC signature using the webhook secret. +2. It checks that the push targets the configured branch. +3. For each added or modified `.yaml` / `.yml` file in the push, it fetches the file content via the GitHub API. +4. The pipeline name is derived from the filename (e.g., `production/my-pipeline.yaml` becomes `my-pipeline`). +5. If a pipeline with that name already exists in the environment, its graph is replaced. Otherwise, a new pipeline is created. + +{% hint style="warning" %} +Bi-directional mode means the Git repository is the source of truth. Any manual edits made in the VectorFlow UI may be overwritten on the next push to the repository. The pipeline editor shows a banner to remind users of this. +{% endhint %} + +## File layout + +VectorFlow expects pipeline YAML files to follow the standard Vector configuration format: + +``` +repo-root/ + environment-name/ + pipeline-a.yaml + pipeline-b.yaml + other-environment/ + pipeline-c.yaml +``` + +The directory name should match the slugified environment name. Files must have a `.yaml` or `.yml` extension. + +## Disabling GitOps + +To disable GitOps, set the mode back to **Off** and click **Save**. The webhook secret is cleared, and incoming webhook requests will be rejected. diff --git a/docs/public/user-guide/environments.md b/docs/public/user-guide/environments.md index a5a30798..91a50b98 100644 --- a/docs/public/user-guide/environments.md +++ b/docs/public/user-guide/environments.md @@ -123,5 +123,21 @@ Click **Test Connection** to verify VectorFlow can reach the repository. When you deploy a pipeline, VectorFlow commits the generated YAML to `{environment-name}/{pipeline-name}.yaml` in the configured repository. When you delete a pipeline, the file is removed with a commit. {% hint style="info" %} -Git sync is a post-deploy side effect. If the Git push fails, the pipeline deploy still succeeds — you will see a warning toast in the UI. +Git sync is a post-deploy side effect. If the Git push fails, the pipeline deploy still succeeds -- you will see a warning toast in the UI. +{% endhint %} + +### GitOps Mode + +Each environment has a **GitOps Mode** setting that controls the direction of Git synchronization: + +| Mode | Description | +|------|-------------| +| **Off** | Git integration is disabled (default). | +| **Push Only** | Pipeline YAML is committed to the repo on deploy. Changes in git are not pulled back. | +| **Bi-directional** | Pipeline YAML is committed on deploy AND pushes to the repo trigger pipeline imports via webhook. | + +When **Bi-directional** mode is enabled, a webhook URL and secret are generated. Configure these in your GitHub repository webhook settings to enable automatic pipeline imports on push. See the [GitOps guide](../operations/gitops.md) for detailed setup instructions. + +{% hint style="warning" %} +In bi-directional mode the Git repository is the source of truth. Manual edits in the VectorFlow UI may be overwritten on the next push. The pipeline editor displays a banner to remind users of this. {% endhint %} diff --git a/prisma/migrations/20260308000000_add_gitops_mode/migration.sql b/prisma/migrations/20260308000000_add_gitops_mode/migration.sql new file mode 100644 index 00000000..b500907a --- /dev/null +++ b/prisma/migrations/20260308000000_add_gitops_mode/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "Environment" ADD COLUMN "gitOpsMode" TEXT NOT NULL DEFAULT 'off'; +ALTER TABLE "Environment" ADD COLUMN "gitWebhookSecret" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7c59e304..007b72e9 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -84,6 +84,8 @@ model Environment { gitRepoUrl String? gitBranch String? @default("main") gitToken String? // Stored encrypted via crypto.ts + gitOpsMode String @default("off") // "off" | "push" | "bidirectional" + gitWebhookSecret String? // HMAC secret for validating incoming git webhooks alertRules AlertRule[] alertWebhooks AlertWebhook[] notificationChannels NotificationChannel[] diff --git a/src/app/(dashboard)/environments/[id]/page.tsx b/src/app/(dashboard)/environments/[id]/page.tsx index ad92770b..6fdb6049 100644 --- a/src/app/(dashboard)/environments/[id]/page.tsx +++ b/src/app/(dashboard)/environments/[id]/page.tsx @@ -520,6 +520,8 @@ export default function EnvironmentDetailPage({ gitRepoUrl={env.gitRepoUrl} gitBranch={env.gitBranch} hasGitToken={env.hasGitToken} + gitOpsMode={env.gitOpsMode} + hasWebhookSecret={env.hasWebhookSecret} /> {/* Created info */} diff --git a/src/app/(dashboard)/pipelines/[id]/page.tsx b/src/app/(dashboard)/pipelines/[id]/page.tsx index 4afb0da3..b99d6468 100644 --- a/src/app/(dashboard)/pipelines/[id]/page.tsx +++ b/src/app/(dashboard)/pipelines/[id]/page.tsx @@ -389,6 +389,7 @@ function PipelineBuilderInner({ pipelineId }: { pipelineId: string }) { ? aggregateProcessStatus(pipelineQuery.data.nodeStatuses) : null } + gitOpsMode={pipelineQuery.data?.gitOpsMode} />
diff --git a/src/app/api/webhooks/git/route.ts b/src/app/api/webhooks/git/route.ts new file mode 100644 index 00000000..f69189f8 --- /dev/null +++ b/src/app/api/webhooks/git/route.ts @@ -0,0 +1,284 @@ +import { NextRequest, NextResponse } from "next/server"; +import crypto from "crypto"; +import { prisma } from "@/lib/prisma"; +import { importVectorConfig } from "@/lib/config-generator"; +import { decrypt } from "@/server/services/crypto"; +import { encryptNodeConfig } from "@/server/services/config-crypto"; +import { writeAuditLog } from "@/server/services/audit"; +import { ComponentKind, Prisma } from "@/generated/prisma"; + +export async function POST(req: NextRequest) { + const body = await req.text(); + const signature = req.headers.get("x-hub-signature-256"); + + if (!signature) { + return NextResponse.json({ error: "Missing signature" }, { status: 401 }); + } + + // 1. Find environments with bidirectional gitOps + const environments = await prisma.environment.findMany({ + where: { gitOpsMode: "bidirectional", gitWebhookSecret: { not: null } }, + }); + + // 2. Verify HMAC signature against each environment's webhook secret + let matchedEnv = null; + for (const env of environments) { + if (!env.gitWebhookSecret) continue; + const webhookSecret = decrypt(env.gitWebhookSecret); + const expected = + "sha256=" + + crypto + .createHmac("sha256", webhookSecret) + .update(body) + .digest("hex"); + + // timingSafeEqual requires equal-length buffers + const sigBuf = Buffer.from(signature); + const expBuf = Buffer.from(expected); + if (sigBuf.length === expBuf.length && crypto.timingSafeEqual(sigBuf, expBuf)) { + matchedEnv = env; + break; + } + } + + if (!matchedEnv) { + return NextResponse.json({ error: "Invalid signature" }, { status: 401 }); + } + + // 3. Parse GitHub push event + let payload: Record; + try { + payload = JSON.parse(body); + } catch { + return NextResponse.json( + { error: "Invalid JSON payload" }, + { status: 400 }, + ); + } + const ref: string | undefined = payload.ref as string | undefined; // "refs/heads/main" + const branch = ref?.replace("refs/heads/", ""); + + // Sanitize branch — only allow alphanumeric, slashes, dashes, dots, underscores + const BRANCH_RE = /^[a-zA-Z0-9\/_.-]+$/; + if (!branch || !BRANCH_RE.test(branch)) { + return NextResponse.json( + { error: "Invalid branch ref" }, + { status: 400 }, + ); + } + + if (branch !== (matchedEnv.gitBranch ?? "main")) { + return NextResponse.json( + { message: "Branch mismatch, ignored" }, + { status: 200 }, + ); + } + + // 4. Find changed YAML files scoped to this environment's directory prefix + const envSlug = matchedEnv.name.toLowerCase().replace(/[^a-z0-9-]/g, "-"); + const commits = (payload.commits ?? []) as Array<{ + added?: string[]; + modified?: string[]; + }>; + const changedFiles = new Set(); + for (const commit of commits) { + for (const f of [...(commit.added ?? []), ...(commit.modified ?? [])]) { + if ( + (f.endsWith(".yaml") || f.endsWith(".yml")) && + f.startsWith(`${envSlug}/`) + ) { + changedFiles.add(f); + } + } + } + + if (changedFiles.size === 0) { + return NextResponse.json({ message: "No YAML changes", processed: 0 }); + } + + // 5. Extract owner/repo and decrypt token once (invariant across files) + const repoUrl = matchedEnv.gitRepoUrl ?? ""; + const repoMatch = repoUrl.match(/github\.com[:/](.+?)(?:\.git)?$/); + if (!repoMatch) { + return NextResponse.json( + { error: "Cannot parse repo URL" }, + { status: 422 }, + ); + } + const repoPath = repoMatch[1]; + + // Validate repoPath is a safe owner/repo format (no path traversal or encoded chars) + const REPO_PATH_RE = /^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$/; + if (!REPO_PATH_RE.test(repoPath)) { + return NextResponse.json( + { error: "Invalid repository path" }, + { status: 422 }, + ); + } + + const token = matchedEnv.gitToken ? decrypt(matchedEnv.gitToken) : null; + if (!token) { + return NextResponse.json( + { error: "No git token configured" }, + { status: 422 }, + ); + } + + // 6. For each changed file, fetch content and import + const results: Array<{ file: string; status: string; error?: string }> = []; + + for (const file of changedFiles) { + try { + // Sanitize file path — reject traversal sequences and non-printable chars + if (file.includes("..") || file.startsWith("/") || /[^\x20-\x7E]/.test(file)) { + results.push({ file, status: "skipped", error: "Invalid file path" }); + continue; + } + + // Build the URL safely with encoded path components + const encodedFile = file.split("/").map(encodeURIComponent).join("/"); + const contentRes = await fetch( + `https://api.github.com/repos/${repoPath}/contents/${encodedFile}?ref=${encodeURIComponent(branch)}`, + { + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github.raw", + }, + }, + ); + if (!contentRes.ok) { + results.push({ + file, + status: "error", + error: `GitHub API ${contentRes.status}`, + }); + continue; + } + const content = await contentRes.text(); + + // Derive pipeline name from filename (strip directory prefix and extension) + // Use only the basename (last path segment) to avoid slashes in the name + const basename = file.split("/").pop() ?? file; + const pipelineName = basename.replace(/\.(yaml|yml)$/, ""); + + // Validate the pipeline name matches the schema used by the tRPC router + const PIPELINE_NAME_RE = /^[a-zA-Z0-9][a-zA-Z0-9 _-]*$/; + if (!pipelineName || pipelineName.length > 100 || !PIPELINE_NAME_RE.test(pipelineName)) { + results.push({ + file, + status: "skipped", + error: `Invalid pipeline name "${pipelineName}" — must start with alphanumeric and contain only letters, numbers, spaces, hyphens, underscores`, + }); + continue; + } + + // Find or create pipeline by name in this environment (atomic) + // Use a serializable transaction to prevent concurrent webhooks from + // racing and creating duplicate pipelines with the same name. + const pipeline = await prisma.$transaction(async (tx) => { + const existing = await tx.pipeline.findFirst({ + where: { environmentId: matchedEnv.id, name: pipelineName }, + }); + if (existing) return existing; + return tx.pipeline.create({ + data: { name: pipelineName, environmentId: matchedEnv.id }, + }); + }, { isolationLevel: Prisma.TransactionIsolationLevel.Serializable }); + + // Import config into pipeline graph nodes/edges + // Only YAML files are collected (see filter above), so format is always "yaml" + const { nodes, edges, globalConfig } = importVectorConfig(content, "yaml"); + + // Map the component kind strings to the Prisma enum + const kindMap: Record = { + source: ComponentKind.SOURCE, + transform: ComponentKind.TRANSFORM, + sink: ComponentKind.SINK, + }; + + // Save graph within a transaction (same pattern as pipeline.saveGraph) + await prisma.$transaction(async (tx) => { + await tx.pipelineEdge.deleteMany({ + where: { pipelineId: pipeline!.id }, + }); + await tx.pipelineNode.deleteMany({ + where: { pipelineId: pipeline!.id }, + }); + + // Create nodes + for (const node of nodes) { + const data = node.data as { + componentDef: { type: string; kind: string }; + componentKey: string; + config: Record; + }; + const componentType = data.componentDef.type; + const kind = kindMap[data.componentDef.kind] ?? ComponentKind.SOURCE; + + await tx.pipelineNode.create({ + data: { + id: node.id, + pipelineId: pipeline!.id, + componentKey: data.componentKey, + componentType, + kind, + config: encryptNodeConfig( + componentType, + data.config, + ) as unknown as Prisma.InputJsonValue, + positionX: node.position.x, + positionY: node.position.y, + }, + }); + } + + // Create edges + for (const edge of edges) { + await tx.pipelineEdge.create({ + data: { + id: edge.id, + pipelineId: pipeline!.id, + sourceNodeId: edge.source, + targetNodeId: edge.target, + sourcePort: (edge as { sourceHandle?: string }).sourceHandle ?? null, + }, + }); + } + + // Update pipeline globalConfig + await tx.pipeline.update({ + where: { id: pipeline!.id }, + data: { + globalConfig: (globalConfig ?? undefined) as Prisma.InputJsonValue | undefined, + }, + }); + }); + + // Write audit log for the import — failures must not mask a successful transaction + try { + await writeAuditLog({ + userId: null, + action: "gitops.pipeline.imported", + entityType: "Pipeline", + entityId: pipeline.id, + environmentId: matchedEnv.id, + teamId: matchedEnv.teamId, + metadata: { + file, + branch, + commitRef: (payload.after as string) ?? null, + pusher: (payload.pusher as { name?: string } | undefined)?.name ?? null, + }, + }); + } catch (auditErr) { + console.error("Failed to write audit log for gitops import:", auditErr); + } + + results.push({ file, status: "imported" }); + } catch (err) { + results.push({ file, status: "error", error: String(err) }); + } + } + + return NextResponse.json({ processed: results.length, results }); +} diff --git a/src/components/environment/git-sync-section.tsx b/src/components/environment/git-sync-section.tsx index b78fa733..a76983cc 100644 --- a/src/components/environment/git-sync-section.tsx +++ b/src/components/environment/git-sync-section.tsx @@ -4,11 +4,19 @@ import { useState } from "react"; import { useTRPC } from "@/trpc/client"; import { useMutation, useQueryClient } from "@tanstack/react-query"; import { toast } from "sonner"; -import { GitBranch, Eye, EyeOff, Loader2 } from "lucide-react"; +import { GitBranch, Eye, EyeOff, Loader2, Copy, Info } from "lucide-react"; +import { copyToClipboard } from "@/lib/utils"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { Card, CardContent, @@ -22,6 +30,8 @@ interface GitSyncSectionProps { gitRepoUrl: string | null; gitBranch: string | null; hasGitToken: boolean; + gitOpsMode?: string; + hasWebhookSecret?: boolean; } export function GitSyncSection({ @@ -29,6 +39,8 @@ export function GitSyncSection({ gitRepoUrl, gitBranch, hasGitToken, + gitOpsMode = "off", + hasWebhookSecret = false, }: GitSyncSectionProps) { const trpc = useTRPC(); const queryClient = useQueryClient(); @@ -38,11 +50,20 @@ export function GitSyncSection({ const [token, setToken] = useState(""); const [showToken, setShowToken] = useState(false); const [isTesting, setIsTesting] = useState(false); + const [selectedGitOpsMode, setSelectedGitOpsMode] = useState(gitOpsMode); + // The actual webhook secret is only available from the update mutation response + const [webhookSecretFromMutation, setWebhookSecretFromMutation] = useState(null); const updateMutation = useMutation( trpc.environment.update.mutationOptions({ - onSuccess: () => { + onSuccess: (data) => { queryClient.invalidateQueries({ queryKey: trpc.environment.get.queryKey({ id: environmentId }) }); + // Capture the webhook secret from the EDITOR-gated mutation response + if (data.gitWebhookSecret) { + setWebhookSecretFromMutation(data.gitWebhookSecret); + } else { + setWebhookSecretFromMutation(null); + } }, onError: (err) => toast.error(err.message || "Failed to save Git settings"), }) @@ -72,6 +93,7 @@ export function GitSyncSection({ gitRepoUrl: repoUrl || null, gitBranch: branch || null, gitToken: token || undefined, // Only send if user entered a new token + gitOpsMode: selectedGitOpsMode as "off" | "push" | "bidirectional", }, { onSuccess: () => { @@ -107,6 +129,7 @@ export function GitSyncSection({ gitRepoUrl: null, gitBranch: null, gitToken: null, + gitOpsMode: "off", }, { onSuccess: () => { @@ -114,14 +137,24 @@ export function GitSyncSection({ setRepoUrl(""); setBranch("main"); setToken(""); + setSelectedGitOpsMode("off"); }, }, ); } - const hasChanges = repoUrl !== (gitRepoUrl ?? "") || branch !== (gitBranch ?? "main") || token !== ""; + const hasChanges = + repoUrl !== (gitRepoUrl ?? "") || + branch !== (gitBranch ?? "main") || + token !== "" || + selectedGitOpsMode !== gitOpsMode; const isConfigured = !!gitRepoUrl; + const webhookUrl = + typeof window !== "undefined" + ? `${window.location.origin}/api/webhooks/git` + : "/api/webhooks/git"; + return ( @@ -183,6 +216,96 @@ export function GitSyncSection({
+ {/* GitOps Mode */} +
+ + +

+ {selectedGitOpsMode === "off" && "Git sync is disabled."} + {selectedGitOpsMode === "push" && "Pipeline YAML is committed to the repo on deploy. Changes in git are not pulled back."} + {selectedGitOpsMode === "bidirectional" && "Pipeline YAML is committed on deploy AND pushes to the repo trigger pipeline imports via webhook."} +

+
+ + {/* Webhook configuration for bidirectional mode */} + {selectedGitOpsMode === "bidirectional" && ( +
+
+ + Webhook Configuration +
+

+ Configure a webhook in your GitHub repository settings to enable bi-directional sync. + Set the content type to application/json and + select the push event. +

+ +
+ +
+ + +
+
+ + {webhookSecretFromMutation && ( +
+ +
+ + +
+

+ Paste this secret into your GitHub webhook settings to enable HMAC signature verification. +

+
+ )} + {!webhookSecretFromMutation && hasWebhookSecret && ( +

+ Webhook secret is configured. For security, the secret is only shown once when first generated. + To get a new secret, switch GitOps mode off and back to bi-directional. +

+ )} + {!webhookSecretFromMutation && !hasWebhookSecret && ( +

+ Save settings to generate a webhook secret. +

+ )} +
+ )} +