diff --git a/.changeset/funny-planes-remember.md b/.changeset/funny-planes-remember.md new file mode 100644 index 0000000000..6c73ef4b71 --- /dev/null +++ b/.changeset/funny-planes-remember.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/build": patch +--- + +syncVercelEnvVars to skip API and read env vars directly from env.process for Vercel build environments. New syncNeonEnvVars build extension for syncing environment variablesfrom Neon database projects to Trigger.dev. The extension automatically detects branches and builds appropriate PostgreSQL connection strings for non-production, non-dev environments (staging, preview). diff --git a/docs/config/extensions/syncEnvVars.mdx b/docs/config/extensions/syncEnvVars.mdx index 6241e1f520..b76191196c 100644 --- a/docs/config/extensions/syncEnvVars.mdx +++ b/docs/config/extensions/syncEnvVars.mdx @@ -80,6 +80,25 @@ The `syncVercelEnvVars` build extension syncs environment variables from your Ve the project with the environment variables you want to sync. + + When running the build from a Vercel build environment (e.g., during a Vercel deployment), the + environment variable values will be read from `process.env` instead of fetching them from the + Vercel API. This is determined by checking if the `VERCEL` environment variable is present. The + API is still used to determine which environment variables are configured for your project, but + the actual values come from the local environment. Reading values from `process.env` allows the + extension to use values that Vercel integrations (such as the Neon integration) set per preview + deployment in the "Provisioning Integrations" phase that happens just before the Vercel build + starts. + + + + If you have the Neon database Vercel integration installed and are running builds outside of the + Vercel environment, we recommend using `syncNeonEnvVars` in addition to `syncVercelEnvVars` for your + database environment variables. This ensures that the correct database connection strings are used for your + selected environment and current branch, as `syncVercelEnvVars` may not accurately reflect + branch-specific database credentials when run locally. + + ```ts import { defineConfig } from "@trigger.dev/sdk"; import { syncVercelEnvVars } from "@trigger.dev/build/extensions/core"; @@ -114,3 +133,79 @@ export default defineConfig({ }, }); ``` + +### syncNeonEnvVars + +The `syncNeonEnvVars` build extension syncs environment variables from your Neon database project to Trigger.dev. It automatically detects branches and builds the appropriate database connection strings for your environment. + + + You need to set the `NEON_ACCESS_TOKEN` and `NEON_PROJECT_ID` environment variables, or pass them + as arguments to the `syncNeonEnvVars` build extension. You can generate a `NEON_ACCESS_TOKEN` in + your Neon [dashboard](https://console.neon.tech/app/settings/api-keys). + + + + When running the build from a Vercel environment (determined by checking if the `VERCEL` + environment variable is present), this extension is skipped entirely. This is because Neon's + Vercel integration already handles environment variable synchronization in Vercel environments. + + + + If you have the Neon database Vercel integration installed and are running builds outside of the + Vercel environment, we recommend using `syncNeonEnvVars` in addition to `syncVercelEnvVars` for your + database environment variables. This ensures that the correct database connection strings are used for your + selected environment and current branch, as `syncVercelEnvVars` may not accurately reflect + branch-specific database credentials when run locally. + + + + This extension is skipped for `prod` environments. It is designed to sync branch-specific + database connections for preview/staging environments. + + +```ts +import { defineConfig } from "@trigger.dev/sdk"; +import { syncNeonEnvVars } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + // This will automatically use the NEON_ACCESS_TOKEN and NEON_PROJECT_ID environment variables + extensions: [syncNeonEnvVars()], + }, +}); +``` + +Or you can pass in the token and project ID as arguments: + +```ts +import { defineConfig } from "@trigger.dev/sdk"; +import { syncNeonEnvVars } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [ + syncNeonEnvVars({ + projectId: "your-neon-project-id", + neonAccessToken: "your-neon-access-token", + branch: "your-branch-name", // optional, defaults to ctx.branch + databaseName: "your-database-name", // optional, defaults to the first database + roleName: "your-role-name", // optional, defaults to the database owner + envVarPrefix: "MY_PREFIX_", // optional, prefix for all synced env vars + }), + ], + }, +}); +``` + +The extension syncs the following environment variables (with optional prefix): + +- `DATABASE_URL` - Pooled connection string +- `DATABASE_URL_UNPOOLED` - Direct connection string +- `POSTGRES_URL`, `POSTGRES_URL_NO_SSL`, `POSTGRES_URL_NON_POOLING` +- `POSTGRES_PRISMA_URL` - Connection string optimized for Prisma +- `POSTGRES_HOST`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DATABASE` +- `PGHOST`, `PGHOST_UNPOOLED`, `PGUSER`, `PGPASSWORD`, `PGDATABASE` diff --git a/docs/guides/examples/vercel-sync-env-vars.mdx b/docs/guides/examples/vercel-sync-env-vars.mdx index 1397f6ccd3..bcdb7d30cb 100644 --- a/docs/guides/examples/vercel-sync-env-vars.mdx +++ b/docs/guides/examples/vercel-sync-env-vars.mdx @@ -19,6 +19,14 @@ To sync environment variables, you just need to add our build extension to your the project with the environment variables you want to sync. + + When running the build from a Vercel build environment (e.g., during a Vercel deployment), the + environment variable values will be read from `process.env` instead of fetching them from the + Vercel API. This is determined by checking if the `VERCEL` environment variable is present. The + API is still used to determine which environment variables are configured for your project, but + the actual values come from the local environment. + + ```ts trigger.config.ts import { defineConfig } from "@trigger.dev/sdk"; import { syncVercelEnvVars } from "@trigger.dev/build/extensions/core"; diff --git a/packages/build/src/extensions/core.ts b/packages/build/src/extensions/core.ts index d9ed52b5a0..62671dfd57 100644 --- a/packages/build/src/extensions/core.ts +++ b/packages/build/src/extensions/core.ts @@ -3,4 +3,5 @@ export * from "./core/additionalPackages.js"; export * from "./core/syncEnvVars.js"; export * from "./core/aptGet.js"; export * from "./core/ffmpeg.js"; +export * from "./core/neonSyncEnvVars.js"; export * from "./core/vercelSyncEnvVars.js"; diff --git a/packages/build/src/extensions/core/neonSyncEnvVars.ts b/packages/build/src/extensions/core/neonSyncEnvVars.ts new file mode 100644 index 0000000000..eeb75320e2 --- /dev/null +++ b/packages/build/src/extensions/core/neonSyncEnvVars.ts @@ -0,0 +1,294 @@ +import { BuildExtension } from "@trigger.dev/core/v3/build"; +import { syncEnvVars } from "../core.js"; + +type EnvVar = { name: string; value: string; isParentEnv?: boolean }; + +type NeonBranch = { + id: string; + name: string; +}; + +type NeonEndpoint = { + id: string; + host: string; + type: string; +}; + +type NeonDatabase = { + id: number; + name: string; + owner_name: string; +}; + +type NeonRole = { + name: string; + password?: string; +}; + +// List of Neon DB related environment variables to sync +export const NEON_ENV_VARS = [ + "PGUSER", + "POSTGRES_URL_NO_SSL", + "POSTGRES_HOST", + "POSTGRES_URL", + "POSTGRES_PRISMA_URL", + "DATABASE_URL_UNPOOLED", + "POSTGRES_URL_NON_POOLING", + "PGHOST", + "POSTGRES_USER", + "DATABASE_URL", + "POSTGRES_PASSWORD", + "POSTGRES_DATABASE", + "PGPASSWORD", + "PGDATABASE", + "PGHOST_UNPOOLED", +]; + +function buildNeonEnvVarMappings(options: { + user: string; + password: string; + database: string; + host: string; + poolerHost: string; +}): Record { + const { user, password, database, host, poolerHost } = options; + + return { + PGUSER: user, + PGPASSWORD: password, + PGDATABASE: database, + PGHOST: poolerHost, + PGHOST_UNPOOLED: host, + POSTGRES_USER: user, + POSTGRES_PASSWORD: password, + POSTGRES_DATABASE: database, + POSTGRES_HOST: poolerHost, + DATABASE_URL: `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${poolerHost}/${database}?sslmode=require`, + DATABASE_URL_UNPOOLED: `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}/${database}?sslmode=require`, + POSTGRES_URL: `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${poolerHost}/${database}?sslmode=require`, + POSTGRES_URL_NO_SSL: `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${poolerHost}/${database}`, + POSTGRES_URL_NON_POOLING: `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${host}/${database}?sslmode=require`, + POSTGRES_PRISMA_URL: `postgresql://${encodeURIComponent(user)}:${encodeURIComponent(password)}@${poolerHost}/${database}?sslmode=require&pgbouncer=true&connect_timeout=15`, + }; +} + +export function syncNeonEnvVars(options?: { + projectId?: string; + neonAccessToken?: string; + branch?: string; + databaseName?: string; + roleName?: string; + envVarPrefix?: string; +}): BuildExtension { + const sync = syncEnvVars(async (ctx) => { + const projectId = + options?.projectId ?? process.env.NEON_PROJECT_ID ?? ctx.env.NEON_PROJECT_ID; + const neonAccessToken = + options?.neonAccessToken ?? process.env.NEON_ACCESS_TOKEN ?? ctx.env.NEON_ACCESS_TOKEN; + const branch = options?.branch ?? ctx.branch; + const envVarPrefix = options?.envVarPrefix ?? ""; + const outputEnvVars = NEON_ENV_VARS; + + // Skip the whole process for Vercel environments + if (ctx.env.VERCEL) { + return []; + } + + if (!projectId) { + throw new Error( + "syncNeonEnvVars: you did not pass in a projectId or set the NEON_PROJECT_ID env var." + ); + } + + if (!neonAccessToken) { + throw new Error( + "syncNeonEnvVars: you did not pass in an neonAccessToken or set the NEON_ACCESS_TOKEN env var." + ); + } + + // Skip branch-specific logic for production environment + if (ctx.environment === "prod") { + return []; + } + + if (!branch) { + throw new Error( + "syncNeonEnvVars: you did not pass in a branch and no branch was detected from context." + ); + } + + const environmentMap = { + prod: "production", + staging: "preview", + dev: "development", + preview: "preview", + } as const; + + const environment = environmentMap[ctx.environment as keyof typeof environmentMap]; + + if (!environment) { + throw new Error( + `Invalid environment '${ctx.environment}'. Expected 'prod', 'staging', 'dev', or 'preview'.` + ); + } + + if (environment === "development") { + // Skip syncing for development environment + return []; + } + + try { + // Step 1: Search for the branch in Neon + const branchSearchParams = new URLSearchParams({ search: branch }); + const branchesUrl = `https://console.neon.tech/api/v2/projects/${projectId}/branches?${branchSearchParams}`; + const branchesResponse = await fetch(branchesUrl, { + headers: { + Authorization: `Bearer ${neonAccessToken}`, + }, + }); + + if (!branchesResponse.ok) { + throw new Error(`Failed to fetch Neon branches: ${branchesResponse.status}`); + } + + const branchesData = await branchesResponse.json(); + const branches: NeonBranch[] = branchesData.branches || []; + + if (branches.length === 0) { + // No matching branch found + return []; + } + + // Neon branch names are prefixed with environment (e.g., "preview/branch-name") + const expectedBranchName = `${environment}/${branch}`; + const matchingBranch = branches.find( + (b) => b.name === expectedBranchName || b.name === branch + ); + + if (!matchingBranch) { + // No exact match found + return []; + } + + const neonBranchId = matchingBranch.id; + + // Step 2: Get endpoints for the branch + const endpointsUrl = `https://console.neon.tech/api/v2/projects/${projectId}/branches/${neonBranchId}/endpoints`; + const endpointsResponse = await fetch(endpointsUrl, { + headers: { + Authorization: `Bearer ${neonAccessToken}`, + }, + }); + + if (!endpointsResponse.ok) { + throw new Error(`Failed to fetch Neon branch endpoints: ${endpointsResponse.status}`); + } + + const endpointsData = await endpointsResponse.json(); + const endpoints: NeonEndpoint[] = endpointsData.endpoints || []; + + if (endpoints.length === 0) { + return []; + } + + // Find an endpoint with type containing 'write', or take the first one + const writeEndpoint = endpoints.find((ep) => ep.type.includes("write")); + const endpoint = writeEndpoint || endpoints[0]; + + if (!endpoint) { + return []; + } + + // Step 3: Get databases for the branch + const databasesUrl = `https://console.neon.tech/api/v2/projects/${projectId}/branches/${neonBranchId}/databases`; + const databasesResponse = await fetch(databasesUrl, { + headers: { + Authorization: `Bearer ${neonAccessToken}`, + }, + }); + + if (!databasesResponse.ok) { + throw new Error(`Failed to fetch Neon branch databases: ${databasesResponse.status}`); + } + + const databasesData = await databasesResponse.json(); + const databases: NeonDatabase[] = databasesData.databases || []; + + if (databases.length === 0) { + return []; + } + + // Find the specified database or use the first one + const targetDatabase = options?.databaseName + ? databases.find((db) => db.name === options.databaseName) + : databases[0]; + + if (!targetDatabase) { + throw new Error( + `syncNeonEnvVars: Database '${options?.databaseName}' not found in branch.` + ); + } + + // Step 4: Get the role (user) and password + const targetRoleName = options?.roleName ?? targetDatabase.owner_name; + const rolePasswordUrl = `https://console.neon.tech/api/v2/projects/${projectId}/branches/${neonBranchId}/roles/${targetRoleName}/reveal_password`; + const rolePasswordResponse = await fetch(rolePasswordUrl, { + headers: { + Authorization: `Bearer ${neonAccessToken}`, + }, + }); + + if (!rolePasswordResponse.ok) { + throw new Error( + `Failed to fetch Neon role password: ${rolePasswordResponse.status}. Make sure the role '${targetRoleName}' exists and has a password.` + ); + } + + const rolePasswordData: NeonRole = await rolePasswordResponse.json(); + const password = rolePasswordData.password; + + if (!password) { + throw new Error( + `syncNeonEnvVars: No password found for role '${targetRoleName}'. The role may not have a password set.` + ); + } + + // Step 5: Build new environment variables based on the endpoint host + const newHost = endpoint.host; + const poolerHost = newHost.replace(/^([^.]+)\./, "$1-pooler."); + + const envVarMappings = buildNeonEnvVarMappings({ + user: targetRoleName, + password, + database: targetDatabase.name, + host: newHost, + poolerHost, + }); + + // Build output env vars + const newEnvVars: EnvVar[] = []; + + for (const neonEnvVar of outputEnvVars) { + const prefixedKey = `${envVarPrefix}${neonEnvVar}`; + if (envVarMappings[neonEnvVar]) { + newEnvVars.push({ + name: prefixedKey, + value: envVarMappings[neonEnvVar], + }); + } + } + + return newEnvVars; + } catch (error) { + console.error("Error fetching Neon branch environment variables:", error); + throw error; + } + }); + + return { + name: "SyncNeonEnvVarsExtension", + async onBuildComplete(context, manifest) { + await sync.onBuildComplete?.(context, manifest); + }, + }; +} diff --git a/packages/build/src/extensions/core/vercelSyncEnvVars.ts b/packages/build/src/extensions/core/vercelSyncEnvVars.ts index 7f58e8a120..34933063cc 100644 --- a/packages/build/src/extensions/core/vercelSyncEnvVars.ts +++ b/packages/build/src/extensions/core/vercelSyncEnvVars.ts @@ -32,6 +32,7 @@ export function syncVercelEnvVars(options?: { process.env.VERCEL_PREVIEW_BRANCH ?? ctx.env.VERCEL_PREVIEW_BRANCH ?? ctx.branch; + const isVercelEnv = !!(ctx.env.VERCEL); if (!projectId) { throw new Error( @@ -81,15 +82,19 @@ export function syncVercelEnvVars(options?: { const filteredEnvs: EnvVar[] = data.envs .filter((env: VercelEnvVar) => { - if (!env.value) return false; if (!env.target.includes(vercelEnvironment)) return false; if (isBranchable && env.gitBranch && env.gitBranch !== branch) return false; + // When running in Vercel, prefer process.env but fall back to API value + const value = isVercelEnv ? (process.env[env.key] ?? env.value) : env.value; + if (!value) return false; return true; }) .map((env: VercelEnvVar) => { + // When running in Vercel, prefer process.env but fall back to API value + const value = isVercelEnv ? (process.env[env.key] ?? env.value) : env.value; return { name: env.key, - value: env.value, + value, isParentEnv: isBranchable && !env.gitBranch, }; });