improvement(serializer): canonical subblock, serialization cleanups, schedules/webhooks are deployment version friendly (#2848)

* hide form deployment tab from docs

* progress

* fix resolution

* cleanup code

* fix positioning

* cleanup dead sockets adv mode ops

* address greptile comments

* fix tests plus more simplification

* fix cleanup

* bring back advanced mode with specific definition

* revert feature flags

* improvement(subblock): ui

* resolver change to make all var references optional chaining

* fix(webhooks/schedules): deployment version friendly

* fix tests

* fix credential sets with new lifecycle

* prep merge

* add back migration

* fix display check for adv fields

* fix trigger vs block scoping

---------

Co-authored-by: Emir Karabeg <emirkarabeg@berkeley.edu>

Author: icecrasher321

apps/docs/content/docs/en/execution/meta.json

@@ -1,3 +1,3 @@
 {
-  "pages": ["index", "basics", "api", "form", "logging", "costs"]
+  "pages": ["index", "basics", "api", "logging", "costs"]
 }

apps/sim/app/api/copilot/execute-tool/route.ts

@@ -14,8 +14,7 @@ import {
 import { generateRequestId } from '@/lib/core/utils/request'
 import { getEffectiveDecryptedEnv } from '@/lib/environment/utils'
 import { refreshTokenIfNeeded } from '@/app/api/auth/oauth/utils'
-import { REFERENCE } from '@/executor/constants'
-import { createEnvVarPattern } from '@/executor/utils/reference-validation'
+import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
 import { executeTool } from '@/tools'
 import { getTool, resolveToolId } from '@/tools/utils'
 
@@ -28,45 +27,6 @@ const ExecuteToolSchema = z.object({
   workflowId: z.string().optional(),
 })
 
-/**
- * Resolves all {{ENV_VAR}} references in a value recursively
- * Works with strings, arrays, and objects
- */
-function resolveEnvVarReferences(value: any, envVars: Record<string, string>): any {
-  if (typeof value === 'string') {
-    // Check for exact match: entire string is "{{VAR_NAME}}"
-    const exactMatchPattern = new RegExp(
-      `^\\${REFERENCE.ENV_VAR_START}([^}]+)\\${REFERENCE.ENV_VAR_END}$`
-    )
-    const exactMatch = exactMatchPattern.exec(value)
-    if (exactMatch) {
-      const envVarName = exactMatch[1].trim()
-      return envVars[envVarName] ?? value
-    }
-
-    // Check for embedded references: "prefix {{VAR}} suffix"
-    const envVarPattern = createEnvVarPattern()
-    return value.replace(envVarPattern, (match, varName) => {
-      const trimmedName = varName.trim()
-      return envVars[trimmedName] ?? match
-    })
-  }
-
-  if (Array.isArray(value)) {
-    return value.map((item) => resolveEnvVarReferences(item, envVars))
-  }
-
-  if (value !== null && typeof value === 'object') {
-    const resolved: Record<string, any> = {}
-    for (const [key, val] of Object.entries(value)) {
-      resolved[key] = resolveEnvVarReferences(val, envVars)
-    }
-    return resolved
-  }
-
-  return value
-}
-
 export async function POST(req: NextRequest) {
   const tracker = createRequestTracker()
 
@@ -145,7 +105,17 @@ export async function POST(req: NextRequest) {
 
     // Build execution params starting with LLM-provided arguments
     // Resolve all {{ENV_VAR}} references in the arguments
-    const executionParams: Record<string, any> = resolveEnvVarReferences(toolArgs, decryptedEnvVars)
+    const executionParams: Record<string, any> = resolveEnvVarReferences(
+      toolArgs,
+      decryptedEnvVars,
+      {
+        resolveExactMatch: true,
+        allowEmbedded: true,
+        trimKeys: true,
+        onMissing: 'keep',
+        deep: true,
+      }
+    ) as Record<string, any>
 
     logger.info(`[${tracker.requestId}] Resolved env var references in arguments`, {
       toolName,

apps/sim/app/api/function/execute/route.ts

@@ -9,6 +9,7 @@ import { escapeRegExp, normalizeName, REFERENCE } from '@/executor/constants'
 import {
   createEnvVarPattern,
   createWorkflowVariablePattern,
+  resolveEnvVarReferences,
 } from '@/executor/utils/reference-validation'
 export const dynamic = 'force-dynamic'
 export const runtime = 'nodejs'
@@ -479,9 +480,29 @@ function resolveEnvironmentVariables(
   const replacements: Array<{ match: string; index: number; varName: string; varValue: string }> =
     []
 
+  const resolverVars: Record<string, string> = {}
+  Object.entries(params).forEach(([key, value]) => {
+    if (value) {
+      resolverVars[key] = String(value)
+    }
+  })
+  Object.entries(envVars).forEach(([key, value]) => {
+    if (value) {
+      resolverVars[key] = value
+    }
+  })
+
   while ((match = regex.exec(code)) !== null) {
     const varName = match[1].trim()
-    const varValue = envVars[varName] || params[varName] || ''
+    const resolved = resolveEnvVarReferences(match[0], resolverVars, {
+      allowEmbedded: true,
+      resolveExactMatch: true,
+      trimKeys: true,
+      onMissing: 'empty',
+      deep: false,
+    })
+    const varValue =
+      typeof resolved === 'string' ? resolved : resolved == null ? '' : String(resolved)
     replacements.push({
       match: match[0],
       index: match.index,

apps/sim/app/api/mcp/servers/test-connection/route.ts

@@ -5,8 +5,7 @@ import { McpClient } from '@/lib/mcp/client'
 import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
 import type { McpServerConfig, McpTransport } from '@/lib/mcp/types'
 import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
-import { REFERENCE } from '@/executor/constants'
-import { createEnvVarPattern } from '@/executor/utils/reference-validation'
+import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
 
 const logger = createLogger('McpServerTestAPI')
 
@@ -24,22 +23,23 @@ function isUrlBasedTransport(transport: McpTransport): boolean {
  * Resolve environment variables in strings
  */
 function resolveEnvVars(value: string, envVars: Record<string, string>): string {
-  const envVarPattern = createEnvVarPattern()
-  const envMatches = value.match(envVarPattern)
-  if (!envMatches) return value
-
-  let resolvedValue = value
-  for (const match of envMatches) {
-    const envKey = match.slice(REFERENCE.ENV_VAR_START.length, -REFERENCE.ENV_VAR_END.length).trim()
-    const envValue = envVars[envKey]
-
-    if (envValue === undefined) {
+  const missingVars: string[] = []
+  const resolvedValue = resolveEnvVarReferences(value, envVars, {
+    allowEmbedded: true,
+    resolveExactMatch: true,
+    trimKeys: true,
+    onMissing: 'keep',
+    deep: false,
+    missingKeys: missingVars,
+  }) as string
+
+  if (missingVars.length > 0) {
+    const uniqueMissing = Array.from(new Set(missingVars))
+    uniqueMissing.forEach((envKey) => {
       logger.warn(`Environment variable "${envKey}" not found in MCP server test`)
-      continue
-    }
-
-    resolvedValue = resolvedValue.replace(match, envValue)
+    })
   }
+
   return resolvedValue
 }
 

apps/sim/app/api/schedules/execute/route.test.ts

@@ -57,6 +57,7 @@ describe('Scheduled Workflow Execution API Route', () => {
       not: vi.fn((condition) => ({ type: 'not', condition })),
       isNull: vi.fn((field) => ({ type: 'isNull', field })),
       or: vi.fn((...conditions) => ({ type: 'or', conditions })),
+      sql: vi.fn((strings, ...values) => ({ type: 'sql', strings, values })),
     }))
 
     vi.doMock('@sim/db', () => {
@@ -92,6 +93,17 @@ describe('Scheduled Workflow Execution API Route', () => {
           status: 'status',
           nextRunAt: 'nextRunAt',
           lastQueuedAt: 'lastQueuedAt',
+          deploymentVersionId: 'deploymentVersionId',
+        },
+        workflowDeploymentVersion: {
+          id: 'id',
+          workflowId: 'workflowId',
+          isActive: 'isActive',
+        },
+        workflow: {
+          id: 'id',
+          userId: 'userId',
+          workspaceId: 'workspaceId',
         },
       }
     })
@@ -134,6 +146,7 @@ describe('Scheduled Workflow Execution API Route', () => {
       not: vi.fn((condition) => ({ type: 'not', condition })),
       isNull: vi.fn((field) => ({ type: 'isNull', field })),
       or: vi.fn((...conditions) => ({ type: 'or', conditions })),
+      sql: vi.fn((strings, ...values) => ({ type: 'sql', strings, values })),
     }))
 
     vi.doMock('@sim/db', () => {
@@ -169,6 +182,17 @@ describe('Scheduled Workflow Execution API Route', () => {
           status: 'status',
           nextRunAt: 'nextRunAt',
           lastQueuedAt: 'lastQueuedAt',
+          deploymentVersionId: 'deploymentVersionId',
+        },
+        workflowDeploymentVersion: {
+          id: 'id',
+          workflowId: 'workflowId',
+          isActive: 'isActive',
+        },
+        workflow: {
+          id: 'id',
+          userId: 'userId',
+          workspaceId: 'workspaceId',
         },
       }
     })
@@ -206,6 +230,7 @@ describe('Scheduled Workflow Execution API Route', () => {
       not: vi.fn((condition) => ({ type: 'not', condition })),
       isNull: vi.fn((field) => ({ type: 'isNull', field })),
       or: vi.fn((...conditions) => ({ type: 'or', conditions })),
+      sql: vi.fn((strings, ...values) => ({ type: 'sql', strings, values })),
     }))
 
     vi.doMock('@sim/db', () => {
@@ -228,6 +253,17 @@ describe('Scheduled Workflow Execution API Route', () => {
           status: 'status',
           nextRunAt: 'nextRunAt',
           lastQueuedAt: 'lastQueuedAt',
+          deploymentVersionId: 'deploymentVersionId',
+        },
+        workflowDeploymentVersion: {
+          id: 'id',
+          workflowId: 'workflowId',
+          isActive: 'isActive',
+        },
+        workflow: {
+          id: 'id',
+          userId: 'userId',
+          workspaceId: 'workspaceId',
         },
       }
     })
@@ -265,6 +301,7 @@ describe('Scheduled Workflow Execution API Route', () => {
       not: vi.fn((condition) => ({ type: 'not', condition })),
       isNull: vi.fn((field) => ({ type: 'isNull', field })),
       or: vi.fn((...conditions) => ({ type: 'or', conditions })),
+      sql: vi.fn((strings, ...values) => ({ type: 'sql', strings, values })),
     }))
 
     vi.doMock('@sim/db', () => {
@@ -310,6 +347,17 @@ describe('Scheduled Workflow Execution API Route', () => {
           status: 'status',
           nextRunAt: 'nextRunAt',
           lastQueuedAt: 'lastQueuedAt',
+          deploymentVersionId: 'deploymentVersionId',
+        },
+        workflowDeploymentVersion: {
+          id: 'id',
+          workflowId: 'workflowId',
+          isActive: 'isActive',
+        },
+        workflow: {
+          id: 'id',
+          userId: 'userId',
+          workspaceId: 'workspaceId',
         },
       }
     })

apps/sim/app/api/schedules/execute/route.ts

@@ -1,7 +1,7 @@
-import { db, workflowSchedule } from '@sim/db'
+import { db, workflowDeploymentVersion, workflowSchedule } from '@sim/db'
 import { createLogger } from '@sim/logger'
 import { tasks } from '@trigger.dev/sdk'
-import { and, eq, isNull, lt, lte, not, or } from 'drizzle-orm'
+import { and, eq, isNull, lt, lte, not, or, sql } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { verifyCronAuth } from '@/lib/auth/internal'
 import { isTriggerDevEnabled } from '@/lib/core/config/feature-flags'
@@ -37,7 +37,8 @@ export async function GET(request: NextRequest) {
           or(
             isNull(workflowSchedule.lastQueuedAt),
             lt(workflowSchedule.lastQueuedAt, workflowSchedule.nextRunAt)
-          )
+          ),
+          sql`${workflowSchedule.deploymentVersionId} = (select ${workflowDeploymentVersion.id} from ${workflowDeploymentVersion} where ${workflowDeploymentVersion.workflowId} = ${workflowSchedule.workflowId} and ${workflowDeploymentVersion.isActive} = true)`
         )
       )
       .returning({

apps/sim/app/api/schedules/route.test.ts

@@ -29,12 +29,23 @@ vi.mock('@sim/db', () => ({
 
 vi.mock('@sim/db/schema', () => ({
   workflow: { id: 'id', userId: 'userId', workspaceId: 'workspaceId' },
-  workflowSchedule: { workflowId: 'workflowId', blockId: 'blockId' },
+  workflowSchedule: {
+    workflowId: 'workflowId',
+    blockId: 'blockId',
+    deploymentVersionId: 'deploymentVersionId',
+  },
+  workflowDeploymentVersion: {
+    id: 'id',
+    workflowId: 'workflowId',
+    isActive: 'isActive',
+  },
 }))
 
 vi.mock('drizzle-orm', () => ({
   eq: vi.fn(),
   and: vi.fn(),
+  or: vi.fn(),
+  isNull: vi.fn(),
 }))
 
 vi.mock('@/lib/core/utils/request', () => ({
@@ -56,6 +67,11 @@ function mockDbChain(results: any[]) {
       where: () => ({
         limit: () => results[callIndex++] || [],
       }),
+      leftJoin: () => ({
+        where: () => ({
+          limit: () => results[callIndex++] || [],
+        }),
+      }),
     }),
   }))
 }
@@ -74,7 +90,16 @@ describe('Schedule GET API', () => {
   it('returns schedule data for authorized user', async () => {
     mockDbChain([
       [{ userId: 'user-1', workspaceId: null }],
-      [{ id: 'sched-1', cronExpression: '0 9 * * *', status: 'active', failedCount: 0 }],
+      [
+        {
+          schedule: {
+            id: 'sched-1',
+            cronExpression: '0 9 * * *',
+            status: 'active',
+            failedCount: 0,
+          },
+        },
+      ],
     ])
 
     const res = await GET(createRequest('http://test/api/schedules?workflowId=wf-1'))
@@ -128,7 +153,7 @@ describe('Schedule GET API', () => {
   it('allows workspace members to view', async () => {
     mockDbChain([
       [{ userId: 'other-user', workspaceId: 'ws-1' }],
-      [{ id: 'sched-1', status: 'active', failedCount: 0 }],
+      [{ schedule: { id: 'sched-1', status: 'active', failedCount: 0 } }],
     ])
 
     const res = await GET(createRequest('http://test/api/schedules?workflowId=wf-1'))
@@ -139,7 +164,7 @@ describe('Schedule GET API', () => {
   it('indicates disabled schedule with failures', async () => {
     mockDbChain([
       [{ userId: 'user-1', workspaceId: null }],
-      [{ id: 'sched-1', status: 'disabled', failedCount: 100 }],
+      [{ schedule: { id: 'sched-1', status: 'disabled', failedCount: 100 } }],
     ])
 
     const res = await GET(createRequest('http://test/api/schedules?workflowId=wf-1'))