diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 669d6532..dafe3798 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,9 +3,27 @@ name: CI on: pull_request: branches: [main] + paths: + - "src/**" + - "agent/**" + - "prisma/**" + - "docker/**" + - "package.json" + - "pnpm-lock.yaml" + - "tsconfig.json" + - ".github/workflows/ci.yml" push: branches: [main] tags: ["v*"] + paths: + - "src/**" + - "agent/**" + - "prisma/**" + - "docker/**" + - "package.json" + - "pnpm-lock.yaml" + - "tsconfig.json" + - ".github/workflows/ci.yml" permissions: contents: write @@ -58,7 +76,7 @@ jobs: with: images: ${{ env.REGISTRY }}/terrifiedbug/vectorflow-server tags: | - type=ref,event=branch + type=raw,value=dev,enable=${{ !startsWith(github.ref, 'refs/tags/v') }} type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} @@ -103,7 +121,7 @@ jobs: with: images: ${{ env.REGISTRY }}/terrifiedbug/vectorflow-agent tags: | - type=ref,event=branch + type=raw,value=dev,enable=${{ !startsWith(github.ref, 'refs/tags/v') }} type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 169a3c08..845bad86 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -3,8 +3,20 @@ name: CodeQL on: pull_request: branches: [main] + paths: + - "src/**" + - "agent/**" + - "prisma/**" + - "package.json" + - "pnpm-lock.yaml" push: branches: [main] + paths: + - "src/**" + - "agent/**" + - "prisma/**" + - "package.json" + - "pnpm-lock.yaml" schedule: - cron: "30 5 * * 1" # Monday 5:30 UTC diff --git a/README.md b/README.md index 4b60644a..d945b5cb 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,9 @@ Stop hand-editing YAML. Build observability pipelines with drag-and-drop
and
- ## Why VectorFlow? @@ -51,13 +49,17 @@ Build Vector pipelines with a drag-and-drop canvas. Browse 100+ components from Deploy pipeline configs to your entire fleet with a single click. The deploy dialog shows a full YAML diff against the previous version before you confirm. Agents pull configs automatically — no SSH, no Ansible, no manual intervention. - +

+ VectorFlow Fleet — manage and monitor all your agents +

### 📊 Real-Time Monitoring Track pipeline throughput, error rates, and host metrics (CPU, memory, disk, network) per node and per pipeline. Live event rates display directly on the pipeline canvas while you're editing. - +

+ VectorFlow Dashboard — real-time metrics per node including CPU, memory, and pipeline throughput +

### 🔄 Version Control & Rollback @@ -72,14 +74,10 @@ Every deployment creates an immutable version snapshot with a changelog. Browse - **Certificate management** — TLS cert storage referenced directly in pipeline configs - **Audit log** — immutable record of every action with before/after diffs - - ### ⚡ Alerting & Webhooks Set threshold-based alert rules on CPU, memory, disk, error rates, and more. Deliver notifications via HMAC-signed webhooks to Slack, Discord, PagerDuty, or any HTTP endpoint. - - ## 🏗️ Architecture ```mermaid diff --git a/docker/server/docker-compose.dev.yml b/docker/server/docker-compose.dev.yml index a933f60b..973fda05 100644 --- a/docker/server/docker-compose.dev.yml +++ b/docker/server/docker-compose.dev.yml @@ -33,6 +33,7 @@ services: AUTH_TRUST_HOST: "true" volumes: - vfdata:/app/.vectorflow + - backups:/backups env_file: - path: .env required: false @@ -42,3 +43,5 @@ volumes: name: vectorflow-pgdata vfdata: name: vectorflow-data + backups: + name: vectorflow-backups diff --git a/src/components/flow/sink-node.tsx b/src/components/flow/sink-node.tsx index bcfc3cb7..4d356cde 100644 --- a/src/components/flow/sink-node.tsx +++ b/src/components/flow/sink-node.tsx @@ -2,9 +2,8 @@ import { memo, useMemo } from "react"; import { Handle, Position, type Node, type NodeProps } from "@xyflow/react"; -import { Badge } from "@/components/ui/badge"; import { cn } from "@/lib/utils"; -import type { VectorComponentDef, DataType } from "@/lib/vector/types"; +import type { VectorComponentDef } from "@/lib/vector/types"; import type { NodeMetricsData } from "@/stores/flow-store"; import { getIcon } from "./node-icon"; import { NodeSparkline } from "./node-sparkline"; @@ -23,36 +22,9 @@ type SinkNodeData = { type SinkNodeType = Node; -const dataTypeBadgeColor: Record = { - log: "bg-emerald-100 text-emerald-800 dark:bg-emerald-900/40 dark:text-emerald-300", - metric: - "bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300", - trace: - "bg-violet-100 text-violet-800 dark:bg-violet-900/40 dark:text-violet-300", -}; - -function getConfigSummary(config: Record): string | null { - const entries = Object.entries(config); - if (entries.length === 0) return null; - - const [key, value] = entries[0]; - if (value === undefined || value === null) return null; - if (typeof value === "object" && !Array.isArray(value)) return `${key}: configured`; - const display = - typeof value === "string" - ? value - : Array.isArray(value) - ? value.slice(0, 2).join(", ") - : String(value); - - const truncated = display.length > 30 ? display.slice(0, 27) + "..." : display; - return `${key}: ${truncated}`; -} - function SinkNodeComponent({ data, selected }: NodeProps) { - const { componentDef, componentKey, config, metrics, disabled } = data; + const { componentDef, componentKey, metrics, disabled } = data; const Icon = useMemo(() => getIcon(componentDef.icon), [componentDef.icon]); - const configSummary = getConfigSummary(config); return (
) {

{componentKey}

- {metrics ? ( + {metrics && (

{formatRate(metrics.eventsPerSec)} ev/s{" "}{formatBytesRate(metrics.bytesPerSec)}

- ) : configSummary ? ( -

- {configSummary} -

- ) : null} - - {/* Data type badges */} -
- {(componentDef.inputTypes ?? []).map((dt) => ( - - {dt.charAt(0).toUpperCase() + dt.slice(1)} - - ))} -
+ )}
{/* Monitoring overlay */} diff --git a/src/components/flow/source-node.tsx b/src/components/flow/source-node.tsx index 4f0ec515..40874444 100644 --- a/src/components/flow/source-node.tsx +++ b/src/components/flow/source-node.tsx @@ -2,10 +2,9 @@ import { memo, useMemo } from "react"; import { Handle, Position, type Node, type NodeProps } from "@xyflow/react"; -import { Badge } from "@/components/ui/badge"; import { Lock } from "lucide-react"; import { cn } from "@/lib/utils"; -import type { VectorComponentDef, DataType } from "@/lib/vector/types"; +import type { VectorComponentDef } from "@/lib/vector/types"; import type { NodeMetricsData } from "@/stores/flow-store"; import { getIcon } from "./node-icon"; import { NodeSparkline } from "./node-sparkline"; @@ -25,36 +24,9 @@ type SourceNodeData = { type SourceNodeType = Node; -const dataTypeBadgeColor: Record = { - log: "bg-emerald-100 text-emerald-800 dark:bg-emerald-900/40 dark:text-emerald-300", - metric: - "bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300", - trace: - "bg-violet-100 text-violet-800 dark:bg-violet-900/40 dark:text-violet-300", -}; - -function getConfigSummary(config: Record): string | null { - const entries = Object.entries(config); - if (entries.length === 0) return null; - - const [key, value] = entries[0]; - if (value === undefined || value === null) return null; - if (typeof value === "object" && !Array.isArray(value)) return `${key}: configured`; - const display = - typeof value === "string" - ? value - : Array.isArray(value) - ? value.slice(0, 2).join(", ") - : String(value); - - const truncated = display.length > 30 ? display.slice(0, 27) + "..." : display; - return `${key}: ${truncated}`; -} - function SourceNodeComponent({ data, selected }: NodeProps) { - const { componentDef, componentKey, config, metrics, disabled, isSystemLocked } = data; + const { componentDef, componentKey, metrics, disabled, isSystemLocked } = data; const Icon = useMemo(() => getIcon(componentDef.icon), [componentDef.icon]); - const configSummary = getConfigSummary(config); return (
) {

{componentKey}

- {metrics ? ( + {metrics && (

{formatRate(metrics.eventsPerSec)} ev/s{" "}{formatBytesRate(metrics.bytesPerSec)}

- ) : configSummary ? ( -

- {configSummary} -

- ) : null} - - {/* Data type badges */} -
- {componentDef.outputTypes.map((dt) => ( - - {dt.charAt(0).toUpperCase() + dt.slice(1)} - - ))} -
+ )}
{/* Monitoring overlay */} diff --git a/src/components/flow/transform-node.tsx b/src/components/flow/transform-node.tsx index c53d6589..e8225b77 100644 --- a/src/components/flow/transform-node.tsx +++ b/src/components/flow/transform-node.tsx @@ -2,9 +2,8 @@ import { memo, useMemo } from "react"; import { Handle, Position, type Node, type NodeProps } from "@xyflow/react"; -import { Badge } from "@/components/ui/badge"; import { cn } from "@/lib/utils"; -import type { VectorComponentDef, DataType } from "@/lib/vector/types"; +import type { VectorComponentDef } from "@/lib/vector/types"; import type { NodeMetricsData } from "@/stores/flow-store"; import { getIcon } from "./node-icon"; import { NodeSparkline } from "./node-sparkline"; @@ -23,39 +22,12 @@ type TransformNodeData = { type TransformNodeType = Node; -const dataTypeBadgeColor: Record = { - log: "bg-emerald-100 text-emerald-800 dark:bg-emerald-900/40 dark:text-emerald-300", - metric: - "bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300", - trace: - "bg-violet-100 text-violet-800 dark:bg-violet-900/40 dark:text-violet-300", -}; - -function getConfigSummary(config: Record): string | null { - const entries = Object.entries(config); - if (entries.length === 0) return null; - - const [key, value] = entries[0]; - if (value === undefined || value === null) return null; - if (typeof value === "object" && !Array.isArray(value)) return `${key}: configured`; - const display = - typeof value === "string" - ? value - : Array.isArray(value) - ? value.slice(0, 2).join(", ") - : String(value); - - const truncated = display.length > 30 ? display.slice(0, 27) + "..." : display; - return `${key}: ${truncated}`; -} - function TransformNodeComponent({ data, selected, }: NodeProps) { - const { componentDef, componentKey, config, metrics, disabled } = data; + const { componentDef, componentKey, metrics, disabled } = data; const Icon = useMemo(() => getIcon(componentDef.icon), [componentDef.icon]); - const configSummary = getConfigSummary(config); return (

{componentKey}

- {metrics ? ( + {metrics && (

{formatRate(metrics.eventsPerSec)} ev/s{" "}{formatBytesRate(metrics.bytesPerSec)}

- ) : configSummary ? ( -

- {configSummary} -

- ) : null} - - {/* Data type badges */} -
- {componentDef.outputTypes.map((dt) => ( - - {dt.charAt(0).toUpperCase() + dt.slice(1)} - - ))} -
+ )}
{/* Monitoring overlay */}