Write telemetry about context provider stats

src/extension/completions-core/vscode-node/bridge/src/completionsTelemetryServiceBridge.ts

@@ -20,6 +20,10 @@ export class CompletionsTelemetryServiceBridge {
 		this.enhancedReporter = undefined;
 	}
 
+	public getTelemetryService(): ITelemetryService {
+		return this.telemetryService;
+	}
+
 	sendGHTelemetryEvent(eventName: string, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements, store?: TelemetryStore): void {
 		this.telemetryService.sendGHTelemetryEvent(wrapEventNameForPrefixRemoval(`copilot/${eventName}`), properties, measurements);
 		this.getSpyReporters(store ?? TelemetryStore.Standard)?.sendTelemetryEvent(eventName, properties as TelemetryProperties, measurements as TelemetryMeasurements);

src/extension/completions-core/vscode-node/lib/src/ghostText/ghostText.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 import { createSha256Hash } from '../../../../../../util/common/crypto';
 import { generateUuid } from '../../../../../../util/vs/base/common/uuid';
+import { CompletionsTelemetryServiceBridge } from '../../../bridge/src/completionsTelemetryServiceBridge';
 import { isSupportedLanguageId } from '../../../prompt/src/parse';
 import { initializeTokenizers } from '../../../prompt/src/tokenization';
 import { CancellationTokenSource, CancellationToken as ICancellationToken } from '../../../types/src';
@@ -28,6 +29,7 @@ import { APIChoice, getTemperatureForSamples } from '../openai/openai';
 import { CopilotNamedAnnotationList } from '../openai/stream';
 import { StatusReporter } from '../progress';
 import { ContextProviderBridge } from '../prompt/components/contextProviderBridge';
+import { ContextProviderStatistics } from '../prompt/contextProviderStatistics';
 import {
 	ContextIndentation,
 	contextIndentation,
@@ -1093,7 +1095,38 @@ export async function getGhostText(
 			options
 		);
 		ctx.get(CompletionNotifier).notifyRequest(completionState, id, telemetryData, token, options);
-		return await getGhostTextWithoutAbortHandling(ctx, completionState, id, telemetryData, token, options);
+		const result = await getGhostTextWithoutAbortHandling(ctx, completionState, id, telemetryData, token, options);
+		const statistics = ctx.get(ContextProviderStatistics).getStatisticsForCompletion(id);
+		const opportunityId = options?.opportunityId ?? 'unknown';
+		const telemetryService = ctx.get(CompletionsTelemetryServiceBridge).getTelemetryService();
+		for (const [providerId, statistic] of statistics.getAllUsageStatistics()) {
+			/* __GDPR__
+				"context-provider.completion-stats" : {
+					"owner": "dirkb",
+					"comment": "Telemetry for copilot inline completion context",
+					"requestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The request correlation id" },
+					"opportunityId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The opportunity id" },
+					"providerId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The context provider id" },
+					"resolution": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The resolution of the context" },
+					"usage": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "How the context was used" },
+					"usageDetails": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Additional details about the usage as a JSON string" }
+				}
+			*/
+			telemetryService.sendMSFTTelemetryEvent(
+				'context-provider.completion-stats',
+				{
+					requestId: id,
+					opportunityId,
+					providerId,
+					resolution: statistic.resolution,
+					usage: statistic.usage,
+					usageDetails: JSON.stringify(statistic.usageDetails),
+				},
+				{
+				}
+			);
+		}
+		return result;
 	} catch (e) {
 		// The cancellation token may be called after the request is done but while we still process data.
 		// The underlying implementation catches abort errors for specific scenarios but we still have uncovered paths.

src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistry.ts

@@ -234,6 +234,10 @@ class CoreContextProviderRegistry extends ContextProviderRegistry {
 			const pendingContextItem = provider.resolver.resolve(request, providerCancellationTokenSource.token);
 			resolutionMap.set(provider.id, pendingContextItem);
 		}
+
+		const statistics = this.ctx.get(ContextProviderStatistics).getStatisticsForCompletion(completionId);
+		statistics.setOpportunityId(opportunityId);
+
 		const results = await resolveAll(resolutionMap, providerCancellationTokenSource.token);
 
 		// Once done, clear the timeout so that we don't cancel the request once it has finished.
@@ -303,10 +307,7 @@ class CoreContextProviderRegistry extends ContextProviderRegistry {
 
 					resolvedContextItems.push(resolvedContextItem);
 				}
-				this.ctx
-					.get(ContextProviderStatistics)
-					.getStatisticsForCompletion(completionId)
-					.setLastResolution(provider.id, result.status);
+				statistics.setLastResolution(provider.id, result.status);
 			} else {
 				// This can't happen
 				logger.error(this.ctx, `Context provider ${provider.id} not found in results`);

src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderStatistics.ts

@@ -54,11 +54,18 @@ export class ContextProviderStatistics {
 }
 
 export class PerCompletionContextProviderStatistics {
+
+	public opportunityId: string | undefined;
+
 	// Keyed by the providerId, contains an array of tuples [context item, expectation]
 	protected _expectations = new Map<string, [SupportedContextItemWithId, PromptExpectation][]>();
 	protected _lastResolution = new Map<string, ResolutionStatus>();
 	protected _statistics = new Map<string, ContextUsageStatistics>();
 
+	constructor() {
+		this.opportunityId = undefined;
+	}
+
 	addExpectations(providerId: string, expectations: [SupportedContextItemWithId, PromptExpectation][]) {
 		const providerExpectations = this._expectations.get(providerId) ?? [];
 		this._expectations.set(providerId, [...providerExpectations, ...expectations]);
@@ -72,10 +79,18 @@ export class PerCompletionContextProviderStatistics {
 		this._lastResolution.set(providerId, resolution);
 	}
 
+	setOpportunityId(opportunityId: string) {
+		this.opportunityId = opportunityId;
+	}
+
 	get(providerId: string): ContextUsageStatistics | undefined {
 		return this._statistics.get(providerId);
 	}
 
+	getAllUsageStatistics(): IterableIterator<[string, ContextUsageStatistics]> {
+		return this._statistics.entries();
+	}
+
 	computeMatch(promptMatchers: PromptMatcher[]) {
 		try {
 			for (const [providerId, expectations] of this._expectations) {

src/extension/typescriptContext/common/serverProtocol.ts

@@ -286,6 +286,11 @@ export type ContextRunnableResult = {
 	 * document and position.
 	 */
 	speculativeKind: SpeculativeKind;
+
+	/**
+	 * A human readable path to ease debugging.
+	 */
+	debugPath?: ContextRunnableResultId | undefined;
 }
 
 export type CachedContextRunnableResult = {

src/extension/typescriptContext/serverPlugin/src/common/baseContextProviders.ts

@@ -109,7 +109,9 @@ export class SignatureRunnable extends FunctionLikeContextRunnable {
 	protected override createRunnableResult(result: ContextResult): RunnableResult {
 		const scope = this.getCacheScope();
 		const cacheInfo: CacheInfo | undefined = scope !== undefined ? { emitMode: EmitMode.ClientBased, scope } : undefined;
-		return result.createRunnableResult(this.id, this.priority, SpeculativeKind.emit, cacheInfo);
+		const runnableResult = result.createRunnableResult(this.id, this.priority, SpeculativeKind.emit, cacheInfo);
+		runnableResult.debugPath = this.getDebugPath();
+		return runnableResult;
 	}
 
 	protected override run(result: RunnableResult, token: tt.CancellationToken): void {
@@ -141,22 +143,30 @@ export class SignatureRunnable extends FunctionLikeContextRunnable {
 		}
 	}
 
+	private getDebugPath(): string | undefined {
+		if (!this.session.host.isDebugging()) {
+			return undefined;
+		}
+		const { sourceFile, startPos, endPos } = SignatureRunnable.getSourceFileAndPositions(this.declaration);
+		const start = ts.getLineAndCharacterOfPosition(sourceFile, startPos);
+		const end = ts.getLineAndCharacterOfPosition(sourceFile, endPos);
+		return `SignatureRunnable:${sourceFile.fileName}:[${start.line},${start.character},${end.line},${end.character}]`;
+
+	}
+
 	private static computeId(session: ComputeContextSession, declaration: tt.FunctionLikeDeclarationBase): string {
-		const host = session.host;
+		const { sourceFile, startPos, endPos } = SignatureRunnable.getSourceFileAndPositions(declaration);
+		const hash = session.host.createHash('md5'); // CodeQL [SM04514] The 'md5' algorithm is used to compute a shorter string to represent a symbol in a map. It has no security implications.
+		hash.update(sourceFile.fileName);
+		hash.update(`[${startPos},${endPos}]`);
+		return `SignatureRunnable:${hash.digest('base64')}`;
+	}
+
+	private static getSourceFileAndPositions(declaration: tt.FunctionLikeDeclarationBase): { sourceFile: tt.SourceFile; startPos: number; endPos: number } {
 		const startPos = declaration.parameters.pos;
 		const endPos = declaration.type?.end ?? declaration.parameters.end;
-		if (host.isDebugging()) {
-			const sourceFile = declaration.getSourceFile();
-			const start = ts.getLineAndCharacterOfPosition(sourceFile, startPos);
-			const end = ts.getLineAndCharacterOfPosition(sourceFile, endPos);
-			return `SignatureRunnable:${declaration.getSourceFile().fileName}:[${start.line},${start.character},${end.line},${end.character}]`;
-		} else {
-			const hash = session.host.createHash('md5'); // CodeQL [SM04514] The 'md5' algorithm is used to compute a shorter string to represent a symbol in a map. It has no security implications.
-			const sourceFile = declaration.getSourceFile();
-			hash.update(sourceFile.fileName);
-			hash.update(`[${startPos},${endPos}]`);
-			return `SignatureRunnable:${hash.digest('base64')}`;
-		}
+		const sourceFile = declaration.getSourceFile();
+		return { sourceFile, startPos, endPos };
 	}
 }
 

src/extension/typescriptContext/serverPlugin/src/common/contextProvider.ts

@@ -430,6 +430,7 @@ export class RunnableResult {
 
 	public readonly priority: number;
 	public readonly items: ContextItem[];
+	public debugPath: string | undefined;
 
 	constructor(id: ContextRunnableResultId, priority: number, runnableResultContext: RunnableResultContext, primaryBudget: CharacterBudget, secondaryBudget: CharacterBudget, speculativeKind: SpeculativeKind, cache?: CacheInfo | undefined) {
 		this.id = id;
@@ -511,7 +512,8 @@ export class RunnableResult {
 			priority: this.priority,
 			items: this.items,
 			cache: this.cache,
-			speculativeKind: this.speculativeKind
+			speculativeKind: this.speculativeKind,
+			debugPath: this.debugPath
 		};
 	}
 }
@@ -834,7 +836,6 @@ export abstract class AbstractContextRunnable implements ContextRunnable {
 		this.cost = cost;
 	}
 
-
 	public initialize(result: ContextResult): void {
 		if (this.result !== undefined) {
 			throw new Error('Runnable already initialized');

src/extension/typescriptContext/serverPlugin/src/common/protocol.ts

@@ -286,6 +286,11 @@ export type ContextRunnableResult = {
 	 * document and position.
 	 */
 	speculativeKind: SpeculativeKind;
+
+	/**
+	 * A human readable path to the signature to ease debugging.
+	 */
+	debugPath?: ContextRunnableResultId | undefined;
 }
 
 export type CachedContextRunnableResult = {

src/extension/typescriptContext/vscode-node/inspector.ts

@@ -218,6 +218,9 @@ class TreeRunnableResult {
 			result.push(new TreeCacheInfo(this.from.cache));
 		}
 		result.push(new TreePropertyItem(this, 'priority', this.from.priority.toString()));
+		if (this.from.debugPath !== undefined) {
+			result.push(new TreePropertyItem(this, 'debugPath', this.from.debugPath));
+		}
 
 		return result;
 	}

src/extension/typescriptContext/vscode-node/languageContextService.ts

@@ -2001,6 +2001,7 @@ export class InlineCompletionContribution implements vscode.Disposable, TokenBud
 		if (item.kind === ContextKind.Snippet) {
 			const converted: Copilot.CodeSnippet = {
 				importance: item.priority * 100,
+				id: item.id,
 				uri: item.uri.toString(),
 				value: item.value
 			};
@@ -2011,6 +2012,7 @@ export class InlineCompletionContribution implements vscode.Disposable, TokenBud
 		} else if (item.kind === ContextKind.Trait) {
 			const converted: Copilot.Trait = {
 				importance: item.priority * 100,
+				id: item.id,
 				name: item.name,
 				value: item.value
 			};

src/extension/typescriptContext/vscode-node/types.ts

@@ -14,6 +14,7 @@ export type ResolvedRunnableResult = {
 	priority: number;
 	items: protocol.FullContextItem[];
 	cache?: protocol.CacheInfo;
+	debugPath?: protocol.ContextRunnableResultId | undefined;
 }
 export namespace ResolvedRunnableResult {
 	export function from(result: protocol.ContextRunnableResult, items: protocol.FullContextItem[]): ResolvedRunnableResult {
@@ -22,7 +23,8 @@ export namespace ResolvedRunnableResult {
 			state: result.state,
 			priority: result.priority,
 			items: items,
-			cache: result.cache
+			cache: result.cache,
+			debugPath: result.debugPath
 		};
 	}
 }
@@ -140,6 +142,8 @@ export class ContextItemResultBuilder implements ContextItemSummary {
 	public contextComputeTime: number;
 	public totalTime: number;
 
+	private counter: number;
+
 	constructor(totalTime: number) {
 		this.seenRunnableResults = new Set();
 		this.seenContextItems = new Set();
@@ -157,6 +161,8 @@ export class ContextItemResultBuilder implements ContextItemSummary {
 		this.serverTime = -1;
 		this.contextComputeTime = -1;
 		this.totalTime = totalTime;
+
+		this.counter = 0;
 	}
 
 	public updateResponse(result: protocol.ContextRequestResult, token: vscode.CancellationToken): void {
@@ -181,7 +187,7 @@ export class ContextItemResultBuilder implements ContextItemSummary {
 				}
 				this.seenContextItems.add(item.key);
 			}
-			const converted = ContextItemResultBuilder.doConvert(item, runnableResult.priority);
+			const converted = ContextItemResultBuilder.doConvert(item, runnableResult.priority, (this.counter++).toString());
 			if (converted === undefined) {
 				continue;
 			}
@@ -193,7 +199,7 @@ export class ContextItemResultBuilder implements ContextItemSummary {
 	public *convert(runnableResult: ResolvedRunnableResult): IterableIterator<ContextItem> {
 		Stats.update(this.stats, runnableResult);
 		for (const item of runnableResult.items) {
-			const converted = ContextItemResultBuilder.doConvert(item, runnableResult.priority);
+			const converted = ContextItemResultBuilder.doConvert(item, runnableResult.priority, (this.counter++).toString());
 			if (converted === undefined) {
 				continue;
 			}
@@ -202,11 +208,12 @@ export class ContextItemResultBuilder implements ContextItemSummary {
 		}
 	}
 
-	private static doConvert(item: protocol.ContextItem, priority: number): ContextItem | undefined {
+	private static doConvert(item: protocol.ContextItem, priority: number, id: string): ContextItem | undefined {
 		switch (item.kind) {
 			case protocol.ContextKind.Snippet:
 				return {
 					kind: ContextKind.Snippet,
+					id: id,
 					priority: priority,
 					uri: vscode.Uri.file(item.fileName),
 					additionalUris: item.additionalFileNames?.map(uri => vscode.Uri.file(uri)),
@@ -215,6 +222,7 @@ export class ContextItemResultBuilder implements ContextItemSummary {
 			case protocol.ContextKind.Trait:
 				return {
 					kind: ContextKind.Trait,
+					id: id,
 					priority: priority,
 					name: item.name,
 					value: item.value

src/platform/inlineCompletions/common/api.ts

@@ -177,6 +177,12 @@ export namespace Copilot {
 		 * Default value is 0.
 		 */
 		importance?: number;
+
+		/**
+		* A unique ID for the context item, used to provide detailed statistics about
+		* the item's usage. If an ID is not provided, it will be generated randomly.
+		*/
+		id?: string;
 	}
 
 	// A key-value pair used for short string snippets.

src/platform/languageServer/common/languageContextService.ts

@@ -23,6 +23,13 @@ export interface SnippetContext {
 	 */
 	kind: ContextKind.Snippet;
 
+	/**
+	 * A unique ID for the context item, used to provide
+	 * detailed statistics about the item's usage. If an ID
+	 * is not provided, it will be generated randomly.
+	 */
+	id?: string;
+
 	/**
 	 * The priority of the snippet. Value range is [0, 1].
 	 */
@@ -50,6 +57,13 @@ export interface TraitContext {
 	 */
 	kind: ContextKind.Trait;
 
+	/**
+	 * A unique ID for the context item, used to provide
+	 * detailed statistics about the item's usage. If an ID
+	 * is not provided, it will be generated randomly.
+	 */
+	id?: string;
+
 	/**
 	 * The priority of the context.
 	 */