Skip to content

Commit 31dca03

Browse files
jrualesaeschli
andauthored
Require '#tool:' prefix for tool references in prompt/instruction/agent files (#272565)
* Require '#tool:' prefix for tool references in prompt/instruction/agent files * Fix tests * Don't match parentheses or brackets * Stricter parsing for tool names * Small comment fix * Improve regex match group names and comments * Fix semantic tokens provider * support tools with `/` and `.` in qualified names * improve completions, add tests --------- Co-authored-by: Martin Aeschlimann <martinae@microsoft.com>
1 parent bb0aeca commit 31dca03

File tree

8 files changed

+283
-58
lines changed

8 files changed

+283
-58
lines changed

src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptBodyAutocompletion.ts

Lines changed: 44 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
import { dirname, extUri } from '../../../../../../base/common/resources.js';
77
import { ITextModel } from '../../../../../../editor/common/model.js';
8-
import { PromptsType } from '../promptTypes.js';
8+
import { getPromptsTypeForLanguageId, PromptsType } from '../promptTypes.js';
99
import { Position } from '../../../../../../editor/common/core/position.js';
1010
import { IFileService } from '../../../../../../platform/files/common/files.js';
1111
import { CancellationToken } from '../../../../../../base/common/cancellation.js';
@@ -15,7 +15,6 @@ import { CharCode } from '../../../../../../base/common/charCode.js';
1515
import { getWordAtText } from '../../../../../../editor/common/core/wordHelper.js';
1616
import { chatVariableLeader } from '../../chatParserTypes.js';
1717
import { ILanguageModelToolsService } from '../../languageModelToolsService.js';
18-
import { getPromptFileType } from '../config/promptFileLocations.js';
1918

2019
/**
2120
* Provides autocompletion for the variables inside prompt bodies.
@@ -44,21 +43,35 @@ export class PromptBodyAutocompletion implements CompletionItemProvider {
4443
* completion items based on the provided arguments.
4544
*/
4645
public async provideCompletionItems(model: ITextModel, position: Position, context: CompletionContext, token: CancellationToken): Promise<CompletionList | undefined> {
46+
const promptsType = getPromptsTypeForLanguageId(model.getLanguageId());
47+
if (!promptsType) {
48+
return undefined;
49+
}
4750
const reference = await this.findVariableReference(model, position, token);
4851
if (!reference) {
4952
return undefined;
5053
}
5154
const suggestions: CompletionItem[] = [];
52-
if (reference.type === 'file') {
53-
if (reference.contentRange.containsPosition(position)) {
54-
// inside the link range
55-
await this.collectFilePathCompletions(model, position, reference.contentRange, suggestions);
56-
}
57-
} else if (reference.type === '') {
58-
const promptFileType = getPromptFileType(model.uri);
59-
if (promptFileType === PromptsType.agent || promptFileType === PromptsType.prompt) {
60-
await this.collectToolCompletions(model, position, reference.contentRange, suggestions);
61-
}
55+
switch (reference.type) {
56+
case 'file':
57+
if (reference.contentRange.containsPosition(position)) {
58+
// inside the link range
59+
await this.collectFilePathCompletions(model, position, reference.contentRange, suggestions);
60+
} else {
61+
await this.collectDefaultCompletions(model, reference.range, promptsType, suggestions);
62+
}
63+
break;
64+
case 'tool':
65+
if (reference.contentRange.containsPosition(position)) {
66+
if (promptsType === PromptsType.agent || promptsType === PromptsType.prompt) {
67+
await this.collectToolCompletions(model, position, reference.contentRange, suggestions);
68+
}
69+
} else {
70+
await this.collectDefaultCompletions(model, reference.range, promptsType, suggestions);
71+
}
72+
break;
73+
default:
74+
await this.collectDefaultCompletions(model, reference.range, promptsType, suggestions);
6275
}
6376
return { suggestions };
6477
}
@@ -125,7 +138,7 @@ export class PromptBodyAutocompletion implements CompletionItemProvider {
125138
/**
126139
* Finds a file reference that suites the provided `position`.
127140
*/
128-
private async findVariableReference(model: ITextModel, position: Position, token: CancellationToken): Promise<{ contentRange: Range; type: string } | undefined> {
141+
private async findVariableReference(model: ITextModel, position: Position, token: CancellationToken): Promise<{ contentRange: Range; type: string; range: Range } | undefined> {
129142
if (model.getLineContent(1).trimEnd() === '---') {
130143
let i = 2;
131144
while (i <= model.getLineCount() && model.getLineContent(i).trimEnd() !== '---') {
@@ -142,15 +155,29 @@ export class PromptBodyAutocompletion implements CompletionItemProvider {
142155
if (!varWord) {
143156
return undefined;
144157
}
158+
const range = new Range(position.lineNumber, varWord.startColumn + 1, position.lineNumber, varWord.endColumn);
145159
const nameMatch = varWord.word.match(/^#(\w+:)?/);
146160
if (nameMatch) {
161+
const contentCol = varWord.startColumn + nameMatch[0].length;
147162
if (nameMatch[1] === 'file:') {
148-
const contentCol = varWord.startColumn + nameMatch[0].length;
149-
return { type: 'file', contentRange: new Range(position.lineNumber, contentCol, position.lineNumber, varWord.endColumn) };
163+
return { type: 'file', contentRange: new Range(position.lineNumber, contentCol, position.lineNumber, varWord.endColumn), range };
164+
} else if (nameMatch[1] === 'tool:') {
165+
return { type: 'tool', contentRange: new Range(position.lineNumber, contentCol, position.lineNumber, varWord.endColumn), range };
150166
}
151167
}
152-
return { type: '', contentRange: new Range(position.lineNumber, varWord.startColumn + 1, position.lineNumber, varWord.endColumn) };
168+
return { type: '', contentRange: range, range };
153169
}
154170

155-
171+
private async collectDefaultCompletions(model: ITextModel, range: Range, promptFileType: PromptsType, suggestions: CompletionItem[]): Promise<void> {
172+
const labels = promptFileType === PromptsType.instructions ? ['file'] : ['file', 'tool'];
173+
labels.forEach(label => {
174+
suggestions.push({
175+
label: `${label}:`,
176+
kind: CompletionItemKind.Keyword,
177+
insertText: `${label}:`,
178+
range: range,
179+
command: { id: 'editor.action.triggerSuggest', title: 'Suggest' }
180+
});
181+
});
182+
}
156183
}

src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptDocumentSemanticTokensProvider.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,11 @@ export class PromptDocumentSemanticTokensProvider implements DocumentSemanticTok
5656
: a.range.startLineNumber - b.range.startLineNumber);
5757

5858
for (const ref of ordered) {
59+
// Also include the '#tool:' prefix for syntax highlighting purposes, even if it's not originally part of the variable name itself.
60+
const extraCharCount = '#tool:'.length;
5961
const line = ref.range.startLineNumber - 1; // zero-based
60-
const char = ref.range.startColumn - 2; // zero-based, include the leading #
61-
const length = ref.range.endColumn - ref.range.startColumn + 1;
62+
const char = ref.range.startColumn - extraCharCount - 1; // zero-based
63+
const length = ref.range.endColumn - ref.range.startColumn + extraCharCount;
6264
const deltaLine = line - lastLine;
6365
const deltaChar = deltaLine === 0 ? char - lastChar : char;
6466
data.push(deltaLine, deltaChar, length, 0 /* variable token type index */, 0 /* no modifiers */);

src/vs/workbench/contrib/chat/common/promptSyntax/promptFileParser.ts

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ export class PromptBody {
302302
const bodyOffset = Iterable.reduce(Iterable.slice(this.linesWithEOL, 0, this.range.startLineNumber - 1), (len, line) => line.length + len, 0);
303303
for (let i = this.range.startLineNumber - 1, lineStartOffset = bodyOffset; i < this.range.endLineNumber - 1; i++) {
304304
const line = this.linesWithEOL[i];
305+
// Match markdown links: [text](link)
305306
const linkMatch = line.matchAll(/\[(.*?)\]\((.+?)\)/g);
306307
for (const match of linkMatch) {
307308
const linkEndOffset = match.index + match[0].length - 1; // before the parenthesis
@@ -310,26 +311,27 @@ export class PromptBody {
310311
fileReferences.push({ content: match[2], range, isMarkdownLink: true });
311312
markdownLinkRanges.push(new Range(i + 1, match.index + 1, i + 1, match.index + match[0].length + 1));
312313
}
313-
const reg = new RegExp(`#([\\w]+:)?([^\\s#]+)`, 'g');
314+
// Match #file:<filePath> and #tool:<toolName>
315+
// Regarding the <toolName> pattern below, see also the variableReg regex in chatRequestParser.ts.
316+
const reg = /#file:(?<filePath>[^\s#]+)|#tool:(?<toolName>[\w_\-\.\/]+)/gi;
314317
const matches = line.matchAll(reg);
315318
for (const match of matches) {
316-
const fullRange = new Range(i + 1, match.index + 1, i + 1, match.index + match[0].length + 1);
319+
const fullMatch = match[0];
320+
const fullRange = new Range(i + 1, match.index + 1, i + 1, match.index + fullMatch.length + 1);
317321
if (markdownLinkRanges.some(mdRange => Range.areIntersectingOrTouching(mdRange, fullRange))) {
318322
continue;
319323
}
320-
const varType = match[1];
321-
if (varType) {
322-
if (varType === 'file:') {
323-
const linkStartOffset = match.index + match[0].length - match[2].length;
324-
const linkEndOffset = match.index + match[0].length;
325-
const range = new Range(i + 1, linkStartOffset + 1, i + 1, linkEndOffset + 1);
326-
fileReferences.push({ content: match[2], range, isMarkdownLink: false });
327-
}
328-
} else {
329-
const contentStartOffset = match.index + 1; // after the #
330-
const contentEndOffset = match.index + match[0].length;
331-
const range = new Range(i + 1, contentStartOffset + 1, i + 1, contentEndOffset + 1);
332-
variableReferences.push({ name: match[2], range, offset: lineStartOffset + match.index });
324+
const contentMatch = match.groups?.['filePath'] || match.groups?.['toolName'];
325+
if (!contentMatch) {
326+
continue;
327+
}
328+
const startOffset = match.index + fullMatch.length - contentMatch.length;
329+
const endOffset = match.index + fullMatch.length;
330+
const range = new Range(i + 1, startOffset + 1, i + 1, endOffset + 1);
331+
if (match.groups?.['filePath']) {
332+
fileReferences.push({ content: match.groups?.['filePath'], range, isMarkdownLink: false });
333+
} else if (match.groups?.['toolName']) {
334+
variableReferences.push({ name: match.groups?.['toolName'], range, offset: lineStartOffset + match.index });
333335
}
334336
}
335337
lineStartOffset += line.length;
Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import assert from 'assert';
7+
import { CancellationToken } from '../../../../../../base/common/cancellation.js';
8+
import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js';
9+
import { Position } from '../../../../../../editor/common/core/position.js';
10+
import { CompletionContext, CompletionTriggerKind } from '../../../../../../editor/common/languages.js';
11+
import { ContextKeyService } from '../../../../../../platform/contextkey/browser/contextKeyService.js';
12+
import { TestConfigurationService } from '../../../../../../platform/configuration/test/common/testConfigurationService.js';
13+
import { ExtensionIdentifier } from '../../../../../../platform/extensions/common/extensions.js';
14+
import { TestInstantiationService } from '../../../../../../platform/instantiation/test/common/instantiationServiceMock.js';
15+
import { workbenchInstantiationService } from '../../../../../test/browser/workbenchTestServices.js';
16+
import { LanguageModelToolsService } from '../../../browser/languageModelToolsService.js';
17+
import { ChatConfiguration } from '../../../common/constants.js';
18+
import { ILanguageModelToolsService, IToolData, ToolDataSource } from '../../../common/languageModelToolsService.js';
19+
import { PromptBodyAutocompletion } from '../../../common/promptSyntax/languageProviders/promptBodyAutocompletion.js';
20+
import { createTextModel } from '../../../../../../editor/test/common/testTextModel.js';
21+
import { URI } from '../../../../../../base/common/uri.js';
22+
import { getLanguageIdForPromptsType, PromptsType } from '../../../common/promptSyntax/promptTypes.js';
23+
import { getPromptFileExtension } from '../../../common/promptSyntax/config/promptFileLocations.js';
24+
import { IFileService } from '../../../../../../platform/files/common/files.js';
25+
import { FileService } from '../../../../../../platform/files/common/fileService.js';
26+
import { VSBuffer } from '../../../../../../base/common/buffer.js';
27+
import { InMemoryFileSystemProvider } from '../../../../../../platform/files/common/inMemoryFilesystemProvider.js';
28+
import { ILogService, NullLogService } from '../../../../../../platform/log/common/log.js';
29+
import { Range } from '../../../../../../editor/common/core/range.js';
30+
31+
suite('PromptBodyAutocompletion', () => {
32+
const disposables = ensureNoDisposablesAreLeakedInTestSuite();
33+
34+
let instaService: TestInstantiationService;
35+
let completionProvider: PromptBodyAutocompletion;
36+
37+
setup(async () => {
38+
const testConfigService = new TestConfigurationService();
39+
testConfigService.setUserConfiguration(ChatConfiguration.ExtensionToolsEnabled, true);
40+
instaService = workbenchInstantiationService({
41+
contextKeyService: () => disposables.add(new ContextKeyService(testConfigService)),
42+
configurationService: () => testConfigService
43+
}, disposables);
44+
instaService.stub(ILogService, new NullLogService());
45+
const fileService = disposables.add(instaService.createInstance(FileService));
46+
instaService.stub(IFileService, fileService);
47+
48+
const fileSystemProvider = disposables.add(new InMemoryFileSystemProvider());
49+
disposables.add(fileService.registerProvider('test', fileSystemProvider));
50+
51+
// Create some test files and directories
52+
await fileService.createFolder(URI.parse('test:///workspace'));
53+
await fileService.createFolder(URI.parse('test:///workspace/src'));
54+
await fileService.createFolder(URI.parse('test:///workspace/docs'));
55+
await fileService.writeFile(URI.parse('test:///workspace/src/index.ts'), VSBuffer.fromString('export function hello() {}'));
56+
await fileService.writeFile(URI.parse('test:///workspace/README.md'), VSBuffer.fromString('# Project'));
57+
await fileService.writeFile(URI.parse('test:///workspace/package.json'), VSBuffer.fromString('{}'));
58+
59+
const toolService = disposables.add(instaService.createInstance(LanguageModelToolsService));
60+
61+
const testTool1 = { id: 'testTool1', displayName: 'tool1', canBeReferencedInPrompt: true, modelDescription: 'Test Tool 1', source: ToolDataSource.External, inputSchema: {} } satisfies IToolData;
62+
disposables.add(toolService.registerToolData(testTool1));
63+
64+
const testTool2 = { id: 'testTool2', displayName: 'tool2', canBeReferencedInPrompt: true, toolReferenceName: 'tool2', modelDescription: 'Test Tool 2', source: ToolDataSource.External, inputSchema: {} } satisfies IToolData;
65+
disposables.add(toolService.registerToolData(testTool2));
66+
67+
const myExtSource = { type: 'extension', label: 'My Extension', extensionId: new ExtensionIdentifier('My.extension') } satisfies ToolDataSource;
68+
const testTool3 = { id: 'testTool3', displayName: 'tool3', canBeReferencedInPrompt: true, toolReferenceName: 'tool3', modelDescription: 'Test Tool 3', source: myExtSource, inputSchema: {} } satisfies IToolData;
69+
disposables.add(toolService.registerToolData(testTool3));
70+
71+
const prExtSource = { type: 'extension', label: 'GitHub Pull Request Extension', extensionId: new ExtensionIdentifier('github.vscode-pull-request-github') } satisfies ToolDataSource;
72+
const prExtTool1 = { id: 'suggestFix', canBeReferencedInPrompt: true, toolReferenceName: 'suggest-fix', modelDescription: 'tool4', displayName: 'Test Tool 4', source: prExtSource, inputSchema: {} } satisfies IToolData;
73+
disposables.add(toolService.registerToolData(prExtTool1));
74+
75+
instaService.set(ILanguageModelToolsService, toolService);
76+
77+
completionProvider = instaService.createInstance(PromptBodyAutocompletion);
78+
});
79+
80+
async function getCompletions(content: string, line: number, column: number, promptType: PromptsType) {
81+
const languageId = getLanguageIdForPromptsType(promptType);
82+
const model = disposables.add(createTextModel(content, languageId, undefined, URI.parse('test://workspace/test' + getPromptFileExtension(promptType))));
83+
const position = new Position(line, column);
84+
const context: CompletionContext = { triggerKind: CompletionTriggerKind.Invoke };
85+
const result = await completionProvider.provideCompletionItems(model, position, context, CancellationToken.None);
86+
if (!result || !result.suggestions) {
87+
return [];
88+
}
89+
const lineContent = model.getLineContent(position.lineNumber);
90+
return result.suggestions.map(s => {
91+
assert(s.range instanceof Range);
92+
return {
93+
label: s.label,
94+
result: lineContent.substring(0, s.range.startColumn - 1) + s.insertText + lineContent.substring(s.range.endColumn - 1)
95+
};
96+
});
97+
}
98+
99+
suite('prompt body completions', () => {
100+
test('default suggestions', async () => {
101+
const content = [
102+
'---',
103+
'description: "Test"',
104+
'---',
105+
'',
106+
'Use # to reference a file or tool.',
107+
'One more #to'
108+
].join('\n');
109+
110+
{
111+
const actual = (await getCompletions(content, 5, 6, PromptsType.prompt));
112+
assert.deepEqual(actual, [
113+
{
114+
label: 'file:',
115+
result: 'Use #file: to reference a file or tool.'
116+
},
117+
{
118+
label: 'tool:',
119+
result: 'Use #tool: to reference a file or tool.'
120+
}
121+
]);
122+
}
123+
{
124+
const actual = (await getCompletions(content, 6, 13, PromptsType.prompt));
125+
assert.deepEqual(actual, [
126+
{
127+
label: 'file:',
128+
result: 'One more #file:'
129+
},
130+
{
131+
label: 'tool:',
132+
result: 'One more #tool:'
133+
}
134+
]);
135+
}
136+
});
137+
138+
test('tool suggestions', async () => {
139+
const content = [
140+
'---',
141+
'description: "Test"',
142+
'---',
143+
'',
144+
'Use #tool: to reference a tool.',
145+
].join('\n');
146+
{
147+
const actual = (await getCompletions(content, 5, 11, PromptsType.prompt));
148+
assert.deepEqual(actual, [
149+
{
150+
label: 'tool1',
151+
result: 'Use #tool:tool1 to reference a tool.'
152+
},
153+
{
154+
label: 'tool2',
155+
result: 'Use #tool:tool2 to reference a tool.'
156+
},
157+
{
158+
label: 'my.extension/tool3',
159+
result: 'Use #tool:my.extension/tool3 to reference a tool.'
160+
},
161+
{
162+
label: 'github.vscode-pull-request-github/suggest-fix',
163+
result: 'Use #tool:github.vscode-pull-request-github/suggest-fix to reference a tool.'
164+
}
165+
]);
166+
}
167+
});
168+
});
169+
});

src/vs/workbench/contrib/chat/test/browser/promptSytntax/promptHovers.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ suite('PromptHoverProvider', () => {
5050
const toolService = disposables.add(instaService.createInstance(LanguageModelToolsService));
5151

5252
const testTool1 = { id: 'testTool1', displayName: 'tool1', canBeReferencedInPrompt: true, modelDescription: 'Test Tool 1', source: ToolDataSource.External, inputSchema: {} } satisfies IToolData;
53-
const testTool2 = { id: 'testTool2', displayName: 'tool2', canBeReferencedInPrompt: true, toolReferenceName: 'tool2', modelDescription: 'Test Tool 2', source: ToolDataSource.External, inputSchema: {} } satisfies IToolData;
54-
5553
disposables.add(toolService.registerToolData(testTool1));
54+
55+
const testTool2 = { id: 'testTool2', displayName: 'tool2', canBeReferencedInPrompt: true, toolReferenceName: 'tool2', modelDescription: 'Test Tool 2', source: ToolDataSource.External, inputSchema: {} } satisfies IToolData;
5656
disposables.add(toolService.registerToolData(testTool2));
5757

5858
instaService.set(ILanguageModelToolsService, toolService);

0 commit comments

Comments
 (0)