diff --git a/src/extension/completions-core/vscode-node/bridge/src/completionsTelemetryServiceBridge.ts b/src/extension/completions-core/vscode-node/bridge/src/completionsTelemetryServiceBridge.ts index 9b5777832b..5ed57f520a 100644 --- a/src/extension/completions-core/vscode-node/bridge/src/completionsTelemetryServiceBridge.ts +++ b/src/extension/completions-core/vscode-node/bridge/src/completionsTelemetryServiceBridge.ts @@ -20,6 +20,10 @@ export class CompletionsTelemetryServiceBridge { this.enhancedReporter = undefined; } + public getTelemetryService(): ITelemetryService { + return this.telemetryService; + } + sendGHTelemetryEvent(eventName: string, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements, store?: TelemetryStore): void { this.telemetryService.sendGHTelemetryEvent(wrapEventNameForPrefixRemoval(`copilot/${eventName}`), properties, measurements); this.getSpyReporters(store ?? TelemetryStore.Standard)?.sendTelemetryEvent(eventName, properties as TelemetryProperties, measurements as TelemetryMeasurements); diff --git a/src/extension/completions-core/vscode-node/lib/src/ghostText/ghostText.ts b/src/extension/completions-core/vscode-node/lib/src/ghostText/ghostText.ts index 2388cd3a29..5cf13fde7d 100644 --- a/src/extension/completions-core/vscode-node/lib/src/ghostText/ghostText.ts +++ b/src/extension/completions-core/vscode-node/lib/src/ghostText/ghostText.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { createSha256Hash } from '../../../../../../util/common/crypto'; import { generateUuid } from '../../../../../../util/vs/base/common/uuid'; +import { CompletionsTelemetryServiceBridge } from '../../../bridge/src/completionsTelemetryServiceBridge'; import { isSupportedLanguageId } from '../../../prompt/src/parse'; import { initializeTokenizers } from '../../../prompt/src/tokenization'; import { CancellationTokenSource, CancellationToken as ICancellationToken } from '../../../types/src'; @@ -28,6 +29,7 @@ import { APIChoice, getTemperatureForSamples } from '../openai/openai'; import { CopilotNamedAnnotationList } from '../openai/stream'; import { StatusReporter } from '../progress'; import { ContextProviderBridge } from '../prompt/components/contextProviderBridge'; +import { ContextProviderStatistics } from '../prompt/contextProviderStatistics'; import { ContextIndentation, contextIndentation, @@ -1093,7 +1095,38 @@ export async function getGhostText( options ); ctx.get(CompletionNotifier).notifyRequest(completionState, id, telemetryData, token, options); - return await getGhostTextWithoutAbortHandling(ctx, completionState, id, telemetryData, token, options); + const result = await getGhostTextWithoutAbortHandling(ctx, completionState, id, telemetryData, token, options); + const statistics = ctx.get(ContextProviderStatistics).getStatisticsForCompletion(id); + const opportunityId = options?.opportunityId ?? 'unknown'; + const telemetryService = ctx.get(CompletionsTelemetryServiceBridge).getTelemetryService(); + for (const [providerId, statistic] of statistics.getAllUsageStatistics()) { + /* __GDPR__ + "context-provider.completion-stats" : { + "owner": "dirkb", + "comment": "Telemetry for copilot inline completion context", + "requestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The request correlation id" }, + "opportunityId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The opportunity id" }, + "providerId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The context provider id" }, + "resolution": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The resolution of the context" }, + "usage": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "How the context was used" }, + "usageDetails": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Additional details about the usage as a JSON string" } + } + */ + telemetryService.sendMSFTTelemetryEvent( + 'context-provider.completion-stats', + { + requestId: id, + opportunityId, + providerId, + resolution: statistic.resolution, + usage: statistic.usage, + usageDetails: JSON.stringify(statistic.usageDetails), + }, + { + } + ); + } + return result; } catch (e) { // The cancellation token may be called after the request is done but while we still process data. // The underlying implementation catches abort errors for specific scenarios but we still have uncovered paths. diff --git a/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistry.ts b/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistry.ts index 3f689886a5..52bb24dc82 100644 --- a/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistry.ts +++ b/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistry.ts @@ -234,6 +234,10 @@ class CoreContextProviderRegistry extends ContextProviderRegistry { const pendingContextItem = provider.resolver.resolve(request, providerCancellationTokenSource.token); resolutionMap.set(provider.id, pendingContextItem); } + + const statistics = this.ctx.get(ContextProviderStatistics).getStatisticsForCompletion(completionId); + statistics.setOpportunityId(opportunityId); + const results = await resolveAll(resolutionMap, providerCancellationTokenSource.token); // Once done, clear the timeout so that we don't cancel the request once it has finished. @@ -303,10 +307,7 @@ class CoreContextProviderRegistry extends ContextProviderRegistry { resolvedContextItems.push(resolvedContextItem); } - this.ctx - .get(ContextProviderStatistics) - .getStatisticsForCompletion(completionId) - .setLastResolution(provider.id, result.status); + statistics.setLastResolution(provider.id, result.status); } else { // This can't happen logger.error(this.ctx, `Context provider ${provider.id} not found in results`); diff --git a/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderStatistics.ts b/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderStatistics.ts index 8d8998c54b..665692cac2 100644 --- a/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderStatistics.ts +++ b/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderStatistics.ts @@ -54,11 +54,18 @@ export class ContextProviderStatistics { } export class PerCompletionContextProviderStatistics { + + public opportunityId: string | undefined; + // Keyed by the providerId, contains an array of tuples [context item, expectation] protected _expectations = new Map(); protected _lastResolution = new Map(); protected _statistics = new Map(); + constructor() { + this.opportunityId = undefined; + } + addExpectations(providerId: string, expectations: [SupportedContextItemWithId, PromptExpectation][]) { const providerExpectations = this._expectations.get(providerId) ?? []; this._expectations.set(providerId, [...providerExpectations, ...expectations]); @@ -72,10 +79,18 @@ export class PerCompletionContextProviderStatistics { this._lastResolution.set(providerId, resolution); } + setOpportunityId(opportunityId: string) { + this.opportunityId = opportunityId; + } + get(providerId: string): ContextUsageStatistics | undefined { return this._statistics.get(providerId); } + getAllUsageStatistics(): IterableIterator<[string, ContextUsageStatistics]> { + return this._statistics.entries(); + } + computeMatch(promptMatchers: PromptMatcher[]) { try { for (const [providerId, expectations] of this._expectations) { diff --git a/src/extension/typescriptContext/common/serverProtocol.ts b/src/extension/typescriptContext/common/serverProtocol.ts index 8cc67bfa36..4d769d4a05 100644 --- a/src/extension/typescriptContext/common/serverProtocol.ts +++ b/src/extension/typescriptContext/common/serverProtocol.ts @@ -286,6 +286,11 @@ export type ContextRunnableResult = { * document and position. */ speculativeKind: SpeculativeKind; + + /** + * A human readable path to ease debugging. + */ + debugPath?: ContextRunnableResultId | undefined; } export type CachedContextRunnableResult = { diff --git a/src/extension/typescriptContext/serverPlugin/src/common/baseContextProviders.ts b/src/extension/typescriptContext/serverPlugin/src/common/baseContextProviders.ts index 4e7cac3cf0..c5a589b512 100644 --- a/src/extension/typescriptContext/serverPlugin/src/common/baseContextProviders.ts +++ b/src/extension/typescriptContext/serverPlugin/src/common/baseContextProviders.ts @@ -109,7 +109,9 @@ export class SignatureRunnable extends FunctionLikeContextRunnable { protected override createRunnableResult(result: ContextResult): RunnableResult { const scope = this.getCacheScope(); const cacheInfo: CacheInfo | undefined = scope !== undefined ? { emitMode: EmitMode.ClientBased, scope } : undefined; - return result.createRunnableResult(this.id, this.priority, SpeculativeKind.emit, cacheInfo); + const runnableResult = result.createRunnableResult(this.id, this.priority, SpeculativeKind.emit, cacheInfo); + runnableResult.debugPath = this.getDebugPath(); + return runnableResult; } protected override run(result: RunnableResult, token: tt.CancellationToken): void { @@ -141,22 +143,30 @@ export class SignatureRunnable extends FunctionLikeContextRunnable { } } + private getDebugPath(): string | undefined { + if (!this.session.host.isDebugging()) { + return undefined; + } + const { sourceFile, startPos, endPos } = SignatureRunnable.getSourceFileAndPositions(this.declaration); + const start = ts.getLineAndCharacterOfPosition(sourceFile, startPos); + const end = ts.getLineAndCharacterOfPosition(sourceFile, endPos); + return `SignatureRunnable:${sourceFile.fileName}:[${start.line},${start.character},${end.line},${end.character}]`; + + } + private static computeId(session: ComputeContextSession, declaration: tt.FunctionLikeDeclarationBase): string { - const host = session.host; + const { sourceFile, startPos, endPos } = SignatureRunnable.getSourceFileAndPositions(declaration); + const hash = session.host.createHash('md5'); // CodeQL [SM04514] The 'md5' algorithm is used to compute a shorter string to represent a symbol in a map. It has no security implications. + hash.update(sourceFile.fileName); + hash.update(`[${startPos},${endPos}]`); + return `SignatureRunnable:${hash.digest('base64')}`; + } + + private static getSourceFileAndPositions(declaration: tt.FunctionLikeDeclarationBase): { sourceFile: tt.SourceFile; startPos: number; endPos: number } { const startPos = declaration.parameters.pos; const endPos = declaration.type?.end ?? declaration.parameters.end; - if (host.isDebugging()) { - const sourceFile = declaration.getSourceFile(); - const start = ts.getLineAndCharacterOfPosition(sourceFile, startPos); - const end = ts.getLineAndCharacterOfPosition(sourceFile, endPos); - return `SignatureRunnable:${declaration.getSourceFile().fileName}:[${start.line},${start.character},${end.line},${end.character}]`; - } else { - const hash = session.host.createHash('md5'); // CodeQL [SM04514] The 'md5' algorithm is used to compute a shorter string to represent a symbol in a map. It has no security implications. - const sourceFile = declaration.getSourceFile(); - hash.update(sourceFile.fileName); - hash.update(`[${startPos},${endPos}]`); - return `SignatureRunnable:${hash.digest('base64')}`; - } + const sourceFile = declaration.getSourceFile(); + return { sourceFile, startPos, endPos }; } } diff --git a/src/extension/typescriptContext/serverPlugin/src/common/contextProvider.ts b/src/extension/typescriptContext/serverPlugin/src/common/contextProvider.ts index 460e924e3a..27c2e13d6e 100644 --- a/src/extension/typescriptContext/serverPlugin/src/common/contextProvider.ts +++ b/src/extension/typescriptContext/serverPlugin/src/common/contextProvider.ts @@ -430,6 +430,7 @@ export class RunnableResult { public readonly priority: number; public readonly items: ContextItem[]; + public debugPath: string | undefined; constructor(id: ContextRunnableResultId, priority: number, runnableResultContext: RunnableResultContext, primaryBudget: CharacterBudget, secondaryBudget: CharacterBudget, speculativeKind: SpeculativeKind, cache?: CacheInfo | undefined) { this.id = id; @@ -511,7 +512,8 @@ export class RunnableResult { priority: this.priority, items: this.items, cache: this.cache, - speculativeKind: this.speculativeKind + speculativeKind: this.speculativeKind, + debugPath: this.debugPath }; } } @@ -834,7 +836,6 @@ export abstract class AbstractContextRunnable implements ContextRunnable { this.cost = cost; } - public initialize(result: ContextResult): void { if (this.result !== undefined) { throw new Error('Runnable already initialized'); diff --git a/src/extension/typescriptContext/serverPlugin/src/common/protocol.ts b/src/extension/typescriptContext/serverPlugin/src/common/protocol.ts index 8cc67bfa36..dea001b002 100644 --- a/src/extension/typescriptContext/serverPlugin/src/common/protocol.ts +++ b/src/extension/typescriptContext/serverPlugin/src/common/protocol.ts @@ -286,6 +286,11 @@ export type ContextRunnableResult = { * document and position. */ speculativeKind: SpeculativeKind; + + /** + * A human readable path to the signature to ease debugging. + */ + debugPath?: ContextRunnableResultId | undefined; } export type CachedContextRunnableResult = { diff --git a/src/extension/typescriptContext/vscode-node/inspector.ts b/src/extension/typescriptContext/vscode-node/inspector.ts index ad7326720b..7d776f2cc8 100644 --- a/src/extension/typescriptContext/vscode-node/inspector.ts +++ b/src/extension/typescriptContext/vscode-node/inspector.ts @@ -218,6 +218,9 @@ class TreeRunnableResult { result.push(new TreeCacheInfo(this.from.cache)); } result.push(new TreePropertyItem(this, 'priority', this.from.priority.toString())); + if (this.from.debugPath !== undefined) { + result.push(new TreePropertyItem(this, 'debugPath', this.from.debugPath)); + } return result; } diff --git a/src/extension/typescriptContext/vscode-node/languageContextService.ts b/src/extension/typescriptContext/vscode-node/languageContextService.ts index 92e14115a4..5431d7e5ca 100644 --- a/src/extension/typescriptContext/vscode-node/languageContextService.ts +++ b/src/extension/typescriptContext/vscode-node/languageContextService.ts @@ -2001,6 +2001,7 @@ export class InlineCompletionContribution implements vscode.Disposable, TokenBud if (item.kind === ContextKind.Snippet) { const converted: Copilot.CodeSnippet = { importance: item.priority * 100, + id: item.id, uri: item.uri.toString(), value: item.value }; @@ -2011,6 +2012,7 @@ export class InlineCompletionContribution implements vscode.Disposable, TokenBud } else if (item.kind === ContextKind.Trait) { const converted: Copilot.Trait = { importance: item.priority * 100, + id: item.id, name: item.name, value: item.value }; diff --git a/src/extension/typescriptContext/vscode-node/types.ts b/src/extension/typescriptContext/vscode-node/types.ts index 991b25039f..fc2037916e 100644 --- a/src/extension/typescriptContext/vscode-node/types.ts +++ b/src/extension/typescriptContext/vscode-node/types.ts @@ -14,6 +14,7 @@ export type ResolvedRunnableResult = { priority: number; items: protocol.FullContextItem[]; cache?: protocol.CacheInfo; + debugPath?: protocol.ContextRunnableResultId | undefined; } export namespace ResolvedRunnableResult { export function from(result: protocol.ContextRunnableResult, items: protocol.FullContextItem[]): ResolvedRunnableResult { @@ -22,7 +23,8 @@ export namespace ResolvedRunnableResult { state: result.state, priority: result.priority, items: items, - cache: result.cache + cache: result.cache, + debugPath: result.debugPath }; } } @@ -140,6 +142,8 @@ export class ContextItemResultBuilder implements ContextItemSummary { public contextComputeTime: number; public totalTime: number; + private counter: number; + constructor(totalTime: number) { this.seenRunnableResults = new Set(); this.seenContextItems = new Set(); @@ -157,6 +161,8 @@ export class ContextItemResultBuilder implements ContextItemSummary { this.serverTime = -1; this.contextComputeTime = -1; this.totalTime = totalTime; + + this.counter = 0; } public updateResponse(result: protocol.ContextRequestResult, token: vscode.CancellationToken): void { @@ -181,7 +187,7 @@ export class ContextItemResultBuilder implements ContextItemSummary { } this.seenContextItems.add(item.key); } - const converted = ContextItemResultBuilder.doConvert(item, runnableResult.priority); + const converted = ContextItemResultBuilder.doConvert(item, runnableResult.priority, (this.counter++).toString()); if (converted === undefined) { continue; } @@ -193,7 +199,7 @@ export class ContextItemResultBuilder implements ContextItemSummary { public *convert(runnableResult: ResolvedRunnableResult): IterableIterator { Stats.update(this.stats, runnableResult); for (const item of runnableResult.items) { - const converted = ContextItemResultBuilder.doConvert(item, runnableResult.priority); + const converted = ContextItemResultBuilder.doConvert(item, runnableResult.priority, (this.counter++).toString()); if (converted === undefined) { continue; } @@ -202,11 +208,12 @@ export class ContextItemResultBuilder implements ContextItemSummary { } } - private static doConvert(item: protocol.ContextItem, priority: number): ContextItem | undefined { + private static doConvert(item: protocol.ContextItem, priority: number, id: string): ContextItem | undefined { switch (item.kind) { case protocol.ContextKind.Snippet: return { kind: ContextKind.Snippet, + id: id, priority: priority, uri: vscode.Uri.file(item.fileName), additionalUris: item.additionalFileNames?.map(uri => vscode.Uri.file(uri)), @@ -215,6 +222,7 @@ export class ContextItemResultBuilder implements ContextItemSummary { case protocol.ContextKind.Trait: return { kind: ContextKind.Trait, + id: id, priority: priority, name: item.name, value: item.value diff --git a/src/platform/inlineCompletions/common/api.ts b/src/platform/inlineCompletions/common/api.ts index f244b3f96d..4d55f78428 100644 --- a/src/platform/inlineCompletions/common/api.ts +++ b/src/platform/inlineCompletions/common/api.ts @@ -177,6 +177,12 @@ export namespace Copilot { * Default value is 0. */ importance?: number; + + /** + * A unique ID for the context item, used to provide detailed statistics about + * the item's usage. If an ID is not provided, it will be generated randomly. + */ + id?: string; } // A key-value pair used for short string snippets. diff --git a/src/platform/languageServer/common/languageContextService.ts b/src/platform/languageServer/common/languageContextService.ts index 39ec8150e7..cf055224e4 100644 --- a/src/platform/languageServer/common/languageContextService.ts +++ b/src/platform/languageServer/common/languageContextService.ts @@ -23,6 +23,13 @@ export interface SnippetContext { */ kind: ContextKind.Snippet; + /** + * A unique ID for the context item, used to provide + * detailed statistics about the item's usage. If an ID + * is not provided, it will be generated randomly. + */ + id?: string; + /** * The priority of the snippet. Value range is [0, 1]. */ @@ -50,6 +57,13 @@ export interface TraitContext { */ kind: ContextKind.Trait; + /** + * A unique ID for the context item, used to provide + * detailed statistics about the item's usage. If an ID + * is not provided, it will be generated randomly. + */ + id?: string; + /** * The priority of the context. */