From 6a05cd7e48a948d26a2d0a29e67cc925da79ebab Mon Sep 17 00:00:00 2001 From: Bryant Gillespie Date: Wed, 4 Feb 2026 09:24:57 -0500 Subject: [PATCH 1/4] Attach prompts, content items, and visual editor elements to AI Assistant Context (#26512) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * cleanup * fix: Hide button when AI_ENABLED = false * wip: add chat to visual editor module * fix: menu tweaks * cleanup more * fix: only refresh if mutation * merge menu components * use new menu component * cleanup more * add changeset * cleanup more * add some tests * fix: make resize handles visible again * extract format context to separate util * fix: items tool prompt causing issues * fix visual editing notifications route * refactor out visual elements tool in favor of context plus items tool * refactor to use visual editor config to handle highligthing * wip: context for prompts * Update format-context.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update use-search-filter.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update ai-prompt-variables-modal.vue Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * remove logging * more fixes * formatter * add some lightweight xml prompt injection protection * failed to add to context notification * remove unneeded prop * appease ai overlords again * fix tests * actually fix tests * changeset past tense * part of claude code review * no proxy stores * more of claudes fixes * fix claudes composable * more of claudes fixes * fix focus * restore staged context if send message throws * simplify because nested field updates weren't working * add v-textoverflow * move display value to core responsiblity instead of being passed from ve package * fix test * show errors on submit * fix escaping * Update app/src/lang/translations/en-US.yaml Co-authored-by: Florian C. Wachmann * Update app/src/ai/components/ai-context-menu.vue Co-authored-by: Florian C. Wachmann * Update app/src/modules/content/routes/item.vue Co-authored-by: Florian C. Wachmann * latest round of feedback * change id to key to match conventions and fix tool calling issues * make codebot happy * change it to test * fix test * fix translation * one more * veai ← cleanup (#26577) * refactor: replace useVisualEditorAi with useAiSidebar for clarity * fix: add type annotation for ComputedRef in useAiSidebar function * fix: add type annotations for sidebarSize and sidebarCollapsed emit events * add comment * remove comment (from html) * changeset: add usage notice for visual editing library --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Florian C. Wachmann --- .changeset/fresh-doodles-relax.md | 11 + api/src/ai/chat/controllers/chat.post.ts | 5 +- api/src/ai/chat/lib/create-ui-stream.ts | 26 +- api/src/ai/chat/models/chat-request.ts | 66 ++++ api/src/ai/chat/utils/format-context.test.ts | 306 +++++++++++++++ api/src/ai/chat/utils/format-context.ts | 160 ++++++++ api/src/ai/tools/items/index.ts | 5 +- api/src/ai/tools/items/prompt.md | 16 +- app/src/ai/components/ai-context-card.vue | 139 +++++++ app/src/ai/components/ai-context-menu.vue | 358 +++++++++++++++++ .../ai-context-menu/context-menu-item.vue | 76 ++++ .../ai-context-menu/empty-state.vue | 26 ++ .../components/ai-context-menu/list-view.vue | 36 ++ app/src/ai/components/ai-conversation.vue | 6 +- app/src/ai/components/ai-input.vue | 43 +- app/src/ai/components/ai-message.vue | 33 +- app/src/ai/components/ai-pending-context.vue | 151 +++++++ .../components/ai-prompt-variables-modal.vue | 91 +++++ app/src/ai/components/ai-settings-menu.vue | 22 +- .../ai/components/parts/ai-message-file.vue | 47 ++- .../ai/components/parts/ai-message-tool.vue | 10 +- .../ai/components/parts/ai-tool-call-card.vue | 4 +- app/src/ai/composables/define-tool.test.ts | 4 +- app/src/ai/composables/define-tool.ts | 10 +- .../composables/use-context-staging.test.ts | 371 ++++++++++++++++++ app/src/ai/composables/use-context-staging.ts | 219 +++++++++++ app/src/ai/composables/use-prompts.test.ts | 322 +++++++++++++++ app/src/ai/composables/use-prompts.ts | 175 +++++++++ app/src/ai/composables/use-search-filter.ts | 21 + .../use-visual-element-highlight.ts | 23 ++ app/src/ai/stores/use-ai-context.test.ts | 335 ++++++++++++++++ app/src/ai/stores/use-ai-context.ts | 154 ++++++++ app/src/ai/stores/use-ai-tools.test.ts | 188 +++++++++ app/src/ai/stores/use-ai-tools.ts | 85 ++++ app/src/ai/stores/use-ai.test.ts | 27 +- app/src/ai/stores/use-ai.ts | 244 +++++++----- app/src/ai/types/context.ts | 27 ++ app/src/ai/types/index.ts | 2 + app/src/ai/types/prompts.ts | 8 + .../v-form/composables/use-ai-tools.ts | 6 +- app/src/lang/translations/en-US.yaml | 24 ++ app/src/layouts/calendar/index.ts | 6 +- app/src/layouts/cards/index.ts | 6 +- app/src/layouts/kanban/index.ts | 6 +- app/src/layouts/map/index.ts | 6 +- app/src/layouts/tabular/index.ts | 6 +- app/src/modules/content/routes/item.vue | 16 +- .../visual/components/editing-layer.vue | 115 +++++- .../modules/visual/routes/visual-editor.vue | 114 +++++- app/src/modules/visual/types/index.ts | 34 +- app/src/stores/collections.ts | 6 +- app/src/stores/fields.ts | 6 +- app/src/stores/flows.ts | 6 +- app/src/stores/relations.ts | 6 +- .../private/components/file-lightbox.vue | 3 +- .../views/private/components/file-preview.vue | 12 +- .../views/private/components/live-preview.vue | 106 ++++- packages/ai/src/types.ts | 33 +- pnpm-lock.yaml | 8 +- 59 files changed, 4130 insertions(+), 247 deletions(-) create mode 100644 .changeset/fresh-doodles-relax.md create mode 100644 api/src/ai/chat/utils/format-context.test.ts create mode 100644 api/src/ai/chat/utils/format-context.ts create mode 100644 app/src/ai/components/ai-context-card.vue create mode 100644 app/src/ai/components/ai-context-menu.vue create mode 100644 app/src/ai/components/ai-context-menu/context-menu-item.vue create mode 100644 app/src/ai/components/ai-context-menu/empty-state.vue create mode 100644 app/src/ai/components/ai-context-menu/list-view.vue create mode 100644 app/src/ai/components/ai-pending-context.vue create mode 100644 app/src/ai/components/ai-prompt-variables-modal.vue create mode 100644 app/src/ai/composables/use-context-staging.test.ts create mode 100644 app/src/ai/composables/use-context-staging.ts create mode 100644 app/src/ai/composables/use-prompts.test.ts create mode 100644 app/src/ai/composables/use-prompts.ts create mode 100644 app/src/ai/composables/use-search-filter.ts create mode 100644 app/src/ai/composables/use-visual-element-highlight.ts create mode 100644 app/src/ai/stores/use-ai-context.test.ts create mode 100644 app/src/ai/stores/use-ai-context.ts create mode 100644 app/src/ai/stores/use-ai-tools.test.ts create mode 100644 app/src/ai/stores/use-ai-tools.ts create mode 100644 app/src/ai/types/context.ts create mode 100644 app/src/ai/types/index.ts create mode 100644 app/src/ai/types/prompts.ts diff --git a/.changeset/fresh-doodles-relax.md b/.changeset/fresh-doodles-relax.md new file mode 100644 index 0000000000000..dd696bc0ceec4 --- /dev/null +++ b/.changeset/fresh-doodles-relax.md @@ -0,0 +1,11 @@ +--- +'@directus/ai': minor +'@directus/api': minor +'@directus/app': minor +--- + +Attached prompts, content items, and visual editor elements to AI Assistant Context + +:::notice +To use this feature, update [@directus/visual-editing](https://github.com/directus/visual-editing) to v1.2.0+ on your website. +::: diff --git a/api/src/ai/chat/controllers/chat.post.ts b/api/src/ai/chat/controllers/chat.post.ts index b8ff8c155b469..eecc28b5f80e2 100644 --- a/api/src/ai/chat/controllers/chat.post.ts +++ b/api/src/ai/chat/controllers/chat.post.ts @@ -19,7 +19,7 @@ export const aiChatPostHandler: RequestHandler = async (req, res, _next) => { throw new InvalidPayloadError({ reason: fromZodError(parseResult.error).message }); } - const { provider, model, messages: rawMessages, tools: requestedTools, toolApprovals } = parseResult.data; + const { provider, model, messages: rawMessages, tools: requestedTools, toolApprovals, context } = parseResult.data; const aiSettings = res.locals['ai'].settings; @@ -68,9 +68,10 @@ export const aiChatPostHandler: RequestHandler = async (req, res, _next) => { const stream = await createUiStream(validationResult.data, { provider, model, - tools: tools, + tools, aiSettings, systemPrompt: res.locals['ai'].systemPrompt, + ...(context && { context }), onUsage: (usage) => { res.write(`data: ${JSON.stringify({ type: 'data-usage', data: usage })}\n\n`); }, diff --git a/api/src/ai/chat/lib/create-ui-stream.ts b/api/src/ai/chat/lib/create-ui-stream.ts index 068b7fb509693..d304b725546a5 100644 --- a/api/src/ai/chat/lib/create-ui-stream.ts +++ b/api/src/ai/chat/lib/create-ui-stream.ts @@ -16,6 +16,8 @@ import { getProviderOptions, } from '../../providers/index.js'; import { SYSTEM_PROMPT } from '../constants/system-prompt.js'; +import type { ChatContext } from '../models/chat-request.js'; +import { formatContextForSystemPrompt } from '../utils/format-context.js'; export interface CreateUiStreamOptions { provider: ProviderType; @@ -23,12 +25,13 @@ export interface CreateUiStreamOptions { tools: { [x: string]: Tool }; aiSettings: AISettings; systemPrompt?: string; + context?: ChatContext; onUsage?: (usage: Pick) => void | Promise; } export const createUiStream = async ( messages: UIMessage[], - { provider, model, tools, aiSettings, systemPrompt, onUsage }: CreateUiStreamOptions, + { provider, model, tools, aiSettings, systemPrompt, context, onUsage }: CreateUiStreamOptions, ): Promise>, any>> => { const configs = buildProviderConfigs(aiSettings); const providerConfig = configs.find((c) => c.type === provider); @@ -39,17 +42,32 @@ export const createUiStream = async ( const registry = createAIProviderRegistry(configs, aiSettings); - systemPrompt ||= SYSTEM_PROMPT; - + const baseSystemPrompt = systemPrompt || SYSTEM_PROMPT; + const contextBlock = context ? formatContextForSystemPrompt(context) : null; const providerOptions = getProviderOptions(provider, model, aiSettings); + // Compute the full system prompt once to avoid re-computing on each step + const fullSystemPrompt = contextBlock ? baseSystemPrompt + contextBlock : baseSystemPrompt; const stream = streamText({ - system: systemPrompt, + system: baseSystemPrompt, model: registry.languageModel(`${provider}:${model}`), messages: await convertToModelMessages(messages), stopWhen: [stepCountIs(10)], providerOptions, tools, + /** + * prepareStep is called before each AI step to prepare the system prompt. + * When context exists, we override the system prompt to include context attachments. + * This allows the initial system prompt to be simple while ensuring all steps + * (including tool continuation steps) receive the full context. + */ + prepareStep: () => { + if (contextBlock) { + return { system: fullSystemPrompt }; + } + + return {}; + }, onFinish({ usage }) { if (onUsage) { const { inputTokens, outputTokens, totalTokens } = usage; diff --git a/api/src/ai/chat/models/chat-request.ts b/api/src/ai/chat/models/chat-request.ts index 5ca59ab39f73e..edf9b6815e630 100644 --- a/api/src/ai/chat/models/chat-request.ts +++ b/api/src/ai/chat/models/chat-request.ts @@ -27,12 +27,78 @@ export type ChatRequestTool = z.infer; export const ToolApprovalMode = z.enum(['always', 'ask', 'disabled']); export type ToolApprovalMode = z.infer; +const ItemContextData = z.object({ + collection: z.string(), + key: z.union([z.string(), z.number()]), +}); + +const VisualElementContextData = z.object({ + key: z.string(), + collection: z.string(), + item: z.union([z.string(), z.number()]), + fields: z.array(z.string()).optional(), + rect: z + .object({ + top: z.number(), + left: z.number(), + width: z.number(), + height: z.number(), + }) + .optional(), +}); + +const PromptContextData = z.object({ + text: z.string(), + prompt: z.record(z.string(), z.unknown()), + values: z.record(z.string(), z.string()), +}); + +export const ContextAttachment = z.discriminatedUnion('type', [ + z.object({ + type: z.literal('item'), + display: z.string(), + data: ItemContextData, + snapshot: z.record(z.string(), z.unknown()), + }), + z.object({ + type: z.literal('visual-element'), + display: z.string(), + data: VisualElementContextData, + snapshot: z.record(z.string(), z.unknown()), + }), + z.object({ + type: z.literal('prompt'), + display: z.string(), + data: PromptContextData, + snapshot: z.record(z.string(), z.unknown()), + }), +]); + +export type ContextAttachment = z.infer; + +export const PageContext = z.object({ + path: z.string(), + collection: z.string().optional(), + item: z.union([z.string(), z.number()]).optional(), + module: z.string().optional(), +}); + +export type PageContext = z.infer; + +export const ChatContext = z.object({ + attachments: z.array(ContextAttachment).max(10).optional(), + page: PageContext.optional(), +}); + +export type ChatContext = z.infer; + export const ChatRequest = z.intersection( z.discriminatedUnion('provider', [ProviderOpenAi, ProviderAnthropic, ProviderGoogle, ProviderOpenAiCompatible]), z.object({ tools: z.array(ChatRequestTool), messages: z.array(z.looseObject({})), toolApprovals: z.record(z.string(), ToolApprovalMode).optional(), + context: ChatContext.optional(), }), ); diff --git a/api/src/ai/chat/utils/format-context.test.ts b/api/src/ai/chat/utils/format-context.test.ts new file mode 100644 index 0000000000000..e25b7f94bc994 --- /dev/null +++ b/api/src/ai/chat/utils/format-context.test.ts @@ -0,0 +1,306 @@ +import { describe, expect, test } from 'vitest'; +import { formatContextForSystemPrompt } from './format-context.js'; + +describe('formatContextForSystemPrompt', () => { + test('includes current date even with empty context', () => { + const result = formatContextForSystemPrompt({}); + expect(result).toContain(''); + expect(result).toContain('## Current Date'); + expect(result).toMatch(/\d{4}-\d{2}-\d{2}/); + }); + + test('includes current date with empty attachments', () => { + const result = formatContextForSystemPrompt({ attachments: [] }); + expect(result).toContain(''); + expect(result).toContain('## Current Date'); + }); + + test('formats page context correctly', () => { + const result = formatContextForSystemPrompt({ + page: { + path: '/content/posts/123', + collection: 'posts', + item: '123', + module: 'content', + }, + }); + + expect(result).toContain(''); + expect(result).toContain('Path: /content/posts/123'); + expect(result).toContain('Collection: posts'); + expect(result).toContain('Item: 123'); + expect(result).toContain('Module: content'); + expect(result).toContain(''); + }); + + test('formats prompt attachments in custom_instructions block', () => { + const result = formatContextForSystemPrompt({ + attachments: [ + { + type: 'prompt', + display: 'Test Prompt', + data: { + text: 'Be helpful', + prompt: {}, + values: {}, + }, + snapshot: { + text: 'Be helpful', + messages: [{ role: 'user', text: 'Hello' }], + }, + }, + ], + }); + + expect(result).toContain(''); + expect(result).toContain('### Test Prompt'); + expect(result).toContain('Be helpful'); + expect(result).toContain('**user**: Hello'); + expect(result).toContain(''); + }); + + test('formats item attachments in user_context section', () => { + const result = formatContextForSystemPrompt({ + attachments: [ + { + type: 'item', + display: 'My Post', + data: { + collection: 'posts', + key: '123', + }, + snapshot: { + title: 'Hello World', + body: 'Content here', + }, + }, + ], + }); + + expect(result).toContain(''); + expect(result).toContain('[Item: My Post (posts) — key: 123]'); + expect(result).toContain('"title": "Hello World"'); + expect(result).toContain(''); + }); + + test('formats visual elements in visual_editing block', () => { + const result = formatContextForSystemPrompt({ + attachments: [ + { + type: 'visual-element', + display: 'Hero Section', + data: { + key: 'hero-1', + collection: 'sections', + item: '456', + fields: ['title', 'subtitle'], + }, + snapshot: { + title: 'Welcome', + subtitle: 'Hello there', + }, + }, + ], + }); + + expect(result).toContain(''); + expect(result).toContain('### sections/456 — "Hero Section"'); + expect(result).toContain('Editable fields: title, subtitle'); + expect(result).toContain('"title": "Welcome"'); + expect(result).toContain(''); + }); + + test('escapes XML tags in user-controlled display strings', () => { + const result = formatContextForSystemPrompt({ + attachments: [ + { + type: 'item', + display: 'Injected + + + + diff --git a/app/src/ai/components/ai-context-menu.vue b/app/src/ai/components/ai-context-menu.vue new file mode 100644 index 0000000000000..3f1bf6caa166b --- /dev/null +++ b/app/src/ai/components/ai-context-menu.vue @@ -0,0 +1,358 @@ + + + + + diff --git a/app/src/ai/components/ai-context-menu/context-menu-item.vue b/app/src/ai/components/ai-context-menu/context-menu-item.vue new file mode 100644 index 0000000000000..72a95f3f54675 --- /dev/null +++ b/app/src/ai/components/ai-context-menu/context-menu-item.vue @@ -0,0 +1,76 @@ + + + + + diff --git a/app/src/ai/components/ai-context-menu/empty-state.vue b/app/src/ai/components/ai-context-menu/empty-state.vue new file mode 100644 index 0000000000000..da070b26d93cf --- /dev/null +++ b/app/src/ai/components/ai-context-menu/empty-state.vue @@ -0,0 +1,26 @@ + + + + + diff --git a/app/src/ai/components/ai-context-menu/list-view.vue b/app/src/ai/components/ai-context-menu/list-view.vue new file mode 100644 index 0000000000000..cbbac6be14c43 --- /dev/null +++ b/app/src/ai/components/ai-context-menu/list-view.vue @@ -0,0 +1,36 @@ + + + diff --git a/app/src/ai/components/ai-conversation.vue b/app/src/ai/components/ai-conversation.vue index 5b3080877a99d..d069246c8a74d 100644 --- a/app/src/ai/components/ai-conversation.vue +++ b/app/src/ai/components/ai-conversation.vue @@ -75,6 +75,7 @@ function scrollToBottom(behavior: ScrollBehavior = 'smooth') { - + diff --git a/app/src/views/private/components/file-preview.vue b/app/src/views/private/components/file-preview.vue index a30cfbe400e49..f3f0eff66ab89 100644 --- a/app/src/views/private/components/file-preview.vue +++ b/app/src/views/private/components/file-preview.vue @@ -12,6 +12,8 @@ export interface Props { inModal?: boolean; disabled?: boolean; nonEditable?: boolean; + /** Direct source URL, bypasses asset URL computation from file.id */ + src?: string; } const props = withDefaults(defineProps(), { preset: 'system-large-contain' }); @@ -22,12 +24,14 @@ defineEmits<{ const file = toRef(props, 'file'); -const src = computed(() => - getAssetUrl(file.value.id, { +const src = computed(() => { + if (props.src) return props.src; + + return getAssetUrl(file.value.id, { imageKey: props.preset ?? undefined, cacheBuster: file.value.modified_on, - }), -); + }); +}); const type = computed<'image' | 'video' | 'audio' | string>(() => { const mimeType = file.value.type; diff --git a/app/src/views/private/components/live-preview.vue b/app/src/views/private/components/live-preview.vue index a8385db7a5f04..20bdba1f5893e 100644 --- a/app/src/views/private/components/live-preview.vue +++ b/app/src/views/private/components/live-preview.vue @@ -1,5 +1,6 @@