diff --git a/docs/tool-reference.md b/docs/tool-reference.md index b5106dcf..72a41da7 100644 --- a/docs/tool-reference.md +++ b/docs/tool-reference.md @@ -329,7 +329,8 @@ so returned values have to JSON-serializable. ### `take_snapshot` **Description:** Take a text snapshot of the currently selected page based on the a11y tree. The snapshot lists page elements along with a unique -identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot. +identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot. The snapshot indicates the element selected +in the DevTools Elements panel (if any). **Parameters:** diff --git a/src/McpContext.ts b/src/McpContext.ts index 47c5030a..59fc72b1 100644 --- a/src/McpContext.ts +++ b/src/McpContext.ts @@ -25,12 +25,13 @@ import type { import {listPages} from './tools/pages.js'; import {takeSnapshot} from './tools/snapshot.js'; import {CLOSE_PAGE_ERROR} from './tools/ToolDefinition.js'; -import type {Context} from './tools/ToolDefinition.js'; +import type {Context, DevToolsData} from './tools/ToolDefinition.js'; import type {TraceResult} from './trace-processing/parse.js'; import {WaitForHelper} from './WaitForHelper.js'; export interface TextSnapshotNode extends SerializedAXNode { id: string; + backendNodeId?: number; children: TextSnapshotNode[]; } @@ -38,6 +39,7 @@ export interface TextSnapshot { root: TextSnapshotNode; idToNode: Map; snapshotId: string; + selectedElementUid?: string; } interface McpContextOptions { @@ -151,6 +153,42 @@ export class McpContext implements Context { return context; } + resolveCdpRequestId(cdpRequestId: string): number | undefined { + const selectedPage = this.getSelectedPage(); + if (!cdpRequestId) { + this.logger('no network request'); + return; + } + const request = this.#networkCollector.find(selectedPage, request => { + // @ts-expect-error id is internal. + return request.id === cdpRequestId; + }); + if (!request) { + this.logger('no network request for ' + cdpRequestId); + return; + } + return this.#networkCollector.getIdForResource(request); + } + + resolveCdpElementId(cdpBackendNodeId: number): string | undefined { + if (!cdpBackendNodeId) { + this.logger('no cdpBackendNodeId'); + return; + } + // TODO: index by backendNodeId instead. + const queue = [this.#textSnapshot?.root]; + while (queue.length) { + const current = queue.pop()!; + if (current.backendNodeId === cdpBackendNodeId) { + return current.id; + } + for (const child of current.children) { + queue.push(child); + } + } + return; + } + getNetworkRequests(includePreservedRequests?: boolean): HTTPRequest[] { const page = this.getSelectedPage(); return this.#networkCollector.getData(page, includePreservedRequests); @@ -380,12 +418,17 @@ export class McpContext implements Context { return this.#pageToDevToolsPage.get(page); } - async getDevToolsData(): Promise { + async getDevToolsData(): Promise { try { + this.logger('Getting DevTools UI data'); const selectedPage = this.getSelectedPage(); const devtoolsPage = this.getDevToolsPage(selectedPage); - if (devtoolsPage) { - const cdpRequestId = await devtoolsPage.evaluate(async () => { + if (!devtoolsPage) { + this.logger('No DevTools page detected'); + return {}; + } + const {cdpRequestId, cdpBackendNodeId} = await devtoolsPage.evaluate( + async () => { // @ts-expect-error no types const UI = await import('/bundled/ui/legacy/legacy.js'); // @ts-expect-error no types @@ -393,36 +436,29 @@ export class McpContext implements Context { const request = UI.Context.Context.instance().flavor( SDK.NetworkRequest.NetworkRequest, ); - return request?.requestId(); - }); - if (!cdpRequestId) { - this.logger('no context request'); - return; - } - const request = this.#networkCollector.find(selectedPage, request => { - // @ts-expect-error id is internal. - return request.id === cdpRequestId; - }); - if (!request) { - this.logger('no collected request for ' + cdpRequestId); - return; - } - return { - requestId: this.#networkCollector.getIdForResource(request), - }; - } else { - this.logger('no devtools page deteched'); - } + const node = UI.Context.Context.instance().flavor( + SDK.DOMModel.DOMNode, + ); + return { + cdpRequestId: request?.requestId(), + cdpBackendNodeId: node?.backendNodeId(), + }; + }, + ); + return {cdpBackendNodeId, cdpRequestId}; } catch (err) { this.logger('error getting devtools data', err); } - return; + return {}; } /** * Creates a text snapshot of a page. */ - async createTextSnapshot(verbose = false): Promise { + async createTextSnapshot( + verbose = false, + devtoolsData: DevToolsData | undefined = undefined, + ): Promise { const page = this.getSelectedPage(); const rootNode = await page.accessibility.snapshot({ includeIframes: true, @@ -465,6 +501,12 @@ export class McpContext implements Context { snapshotId: String(snapshotId), idToNode, }; + const data = devtoolsData ?? (await this.getDevToolsData()); + if (data?.cdpBackendNodeId) { + this.#textSnapshot.selectedElementUid = this.resolveCdpElementId( + data?.cdpBackendNodeId, + ); + } } getTextSnapshot(): TextSnapshot | null { diff --git a/src/McpResponse.ts b/src/McpResponse.ts index 6798c90b..b5ea74aa 100644 --- a/src/McpResponse.ts +++ b/src/McpResponse.ts @@ -15,7 +15,7 @@ import { getShortDescriptionForRequest, getStatusFromRequest, } from './formatters/networkFormatter.js'; -import {formatA11ySnapshot} from './formatters/snapshotFormatter.js'; +import {formatSnapshotNode} from './formatters/snapshotFormatter.js'; import type {McpContext} from './McpContext.js'; import type { ConsoleMessage, @@ -25,6 +25,7 @@ import type { } from './third_party/index.js'; import {handleDialog} from './tools/pages.js'; import type { + DevToolsData, ImageContentData, Response, SnapshotParams, @@ -52,6 +53,11 @@ export class McpResponse implements Response { types?: string[]; includePreservedMessages?: boolean; }; + #devToolsData?: DevToolsData; + + attachDevToolsData(data: DevToolsData): void { + this.#devToolsData = data; + } setIncludePages(value: boolean): void { this.#includePages = value; @@ -179,17 +185,22 @@ export class McpResponse implements Response { let formattedSnapshot: string | undefined; if (this.#snapshotParams) { - await context.createTextSnapshot(this.#snapshotParams.verbose); + await context.createTextSnapshot( + this.#snapshotParams.verbose, + this.#devToolsData, + ); const snapshot = context.getTextSnapshot(); if (snapshot) { if (this.#snapshotParams.filePath) { await context.saveFile( - new TextEncoder().encode(formatA11ySnapshot(snapshot.root)), + new TextEncoder().encode( + formatSnapshotNode(snapshot.root, snapshot), + ), this.#snapshotParams.filePath, ); formattedSnapshot = `Saved snapshot to ${this.#snapshotParams.filePath}.`; } else { - formattedSnapshot = formatA11ySnapshot(snapshot.root); + formattedSnapshot = formatSnapshotNode(snapshot.root, snapshot); } } } diff --git a/src/formatters/snapshotFormatter.ts b/src/formatters/snapshotFormatter.ts index e77d7e29..42d567e5 100644 --- a/src/formatters/snapshotFormatter.ts +++ b/src/formatters/snapshotFormatter.ts @@ -3,19 +3,26 @@ * Copyright 2025 Google LLC * SPDX-License-Identifier: Apache-2.0 */ -import type {TextSnapshotNode} from '../McpContext.js'; +import type {TextSnapshot, TextSnapshotNode} from '../McpContext.js'; -export function formatA11ySnapshot( - serializedAXNodeRoot: TextSnapshotNode, +export function formatSnapshotNode( + root: TextSnapshotNode, + snapshot?: TextSnapshot, depth = 0, ): string { let result = ''; - const attributes = getAttributes(serializedAXNodeRoot); - const line = ' '.repeat(depth * 2) + attributes.join(' ') + '\n'; + const attributes = getAttributes(root); + const line = + ' '.repeat(depth * 2) + + attributes.join(' ') + + (root.id === snapshot?.selectedElementUid + ? ' [selected in the DevTools Elements panel]' + : '') + + '\n'; result += line; - for (const child of serializedAXNodeRoot.children) { - result += formatA11ySnapshot(child, depth + 1); + for (const child of root.children) { + result += formatSnapshotNode(child, snapshot, depth + 1); } return result; diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.ts index 8df0f720..9651718b 100644 --- a/src/tools/ToolDefinition.ts +++ b/src/tools/ToolDefinition.ts @@ -47,6 +47,11 @@ export interface SnapshotParams { filePath?: string; } +export interface DevToolsData { + cdpRequestId?: string; + cdpBackendNodeId?: number; +} + export interface Response { appendResponseLine(value: string): void; setIncludePages(value: boolean): void; @@ -69,6 +74,8 @@ export interface Response { attachImage(value: ImageContentData): void; attachNetworkRequest(reqid: number): void; attachConsoleMessage(msgid: number): void; + // Allows re-using DevTools data queried by some tools. + attachDevToolsData(data: DevToolsData): void; } /** @@ -103,7 +110,15 @@ export type Context = Readonly<{ text: string; timeout?: number | undefined; }): Promise; - getDevToolsData(): Promise; + getDevToolsData(): Promise; + /** + * Returns a reqid for a cdpRequestId. + */ + resolveCdpRequestId(cdpRequestId: string): number | undefined; + /** + * Returns a reqid for a cdpRequestId. + */ + resolveCdpElementId(cdpBackendNodeId: number): string | undefined; }>; export function defineTool( diff --git a/src/tools/network.ts b/src/tools/network.ts index e2923c42..d3b6c1fe 100644 --- a/src/tools/network.ts +++ b/src/tools/network.ts @@ -72,12 +72,16 @@ export const listNetworkRequests = defineTool({ }, handler: async (request, response, context) => { const data = await context.getDevToolsData(); + response.attachDevToolsData(data); + const reqid = data?.cdpRequestId + ? context.resolveCdpRequestId(data.cdpRequestId) + : undefined; response.setIncludeNetworkRequests(true, { pageSize: request.params.pageSize, pageIdx: request.params.pageIdx, resourceTypes: request.params.resourceTypes, includePreservedRequests: request.params.includePreservedRequests, - networkRequestIdInDevToolsUI: data?.requestId, + networkRequestIdInDevToolsUI: reqid, }); }, }); @@ -102,8 +106,12 @@ export const getNetworkRequest = defineTool({ response.attachNetworkRequest(request.params.reqid); } else { const data = await context.getDevToolsData(); - if (data?.requestId) { - response.attachNetworkRequest(data?.requestId); + response.attachDevToolsData(data); + const reqid = data?.cdpRequestId + ? context.resolveCdpRequestId(data.cdpRequestId) + : undefined; + if (reqid) { + response.attachNetworkRequest(reqid); } else { response.appendResponseLine( `Nothing is currently selected in the DevTools Network panel.`, diff --git a/src/tools/snapshot.ts b/src/tools/snapshot.ts index 0187f7fc..21099f14 100644 --- a/src/tools/snapshot.ts +++ b/src/tools/snapshot.ts @@ -12,7 +12,8 @@ import {defineTool, timeoutSchema} from './ToolDefinition.js'; export const takeSnapshot = defineTool({ name: 'take_snapshot', description: `Take a text snapshot of the currently selected page based on the a11y tree. The snapshot lists page elements along with a unique -identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot.`, +identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot. The snapshot indicates the element selected +in the DevTools Elements panel (if any).`, annotations: { category: ToolCategory.DEBUGGING, // Not read-only due to filePath param. diff --git a/tests/formatters/snapshotFormatter.test.ts b/tests/formatters/snapshotFormatter.test.ts index 47b7d288..1232929f 100644 --- a/tests/formatters/snapshotFormatter.test.ts +++ b/tests/formatters/snapshotFormatter.test.ts @@ -9,12 +9,12 @@ import {describe, it} from 'node:test'; import type {ElementHandle} from 'puppeteer-core'; -import {formatA11ySnapshot} from '../../src/formatters/snapshotFormatter.js'; +import {formatSnapshotNode} from '../../src/formatters/snapshotFormatter.js'; import type {TextSnapshotNode} from '../../src/McpContext.js'; describe('snapshotFormatter', () => { it('formats a snapshot with value properties', () => { - const snapshot: TextSnapshotNode = { + const node: TextSnapshotNode = { id: '1_1', role: 'textbox', name: 'textbox', @@ -35,7 +35,7 @@ describe('snapshotFormatter', () => { }, }; - const formatted = formatA11ySnapshot(snapshot); + const formatted = formatSnapshotNode(node); assert.strictEqual( formatted, `uid=1_1 textbox "textbox" value="value" @@ -45,7 +45,7 @@ describe('snapshotFormatter', () => { }); it('formats a snapshot with boolean properties', () => { - const snapshot: TextSnapshotNode = { + const node: TextSnapshotNode = { id: '1_1', role: 'button', name: 'button', @@ -66,7 +66,7 @@ describe('snapshotFormatter', () => { }, }; - const formatted = formatA11ySnapshot(snapshot); + const formatted = formatSnapshotNode(node); assert.strictEqual( formatted, `uid=1_1 button "button" disableable disabled @@ -76,7 +76,7 @@ describe('snapshotFormatter', () => { }); it('formats a snapshot with checked properties', () => { - const snapshot: TextSnapshotNode = { + const node: TextSnapshotNode = { id: '1_1', role: 'checkbox', name: 'checkbox', @@ -97,7 +97,7 @@ describe('snapshotFormatter', () => { }, }; - const formatted = formatA11ySnapshot(snapshot); + const formatted = formatSnapshotNode(node); assert.strictEqual( formatted, `uid=1_1 checkbox "checkbox" checked @@ -107,7 +107,7 @@ describe('snapshotFormatter', () => { }); it('formats a snapshot with multiple different type attributes', () => { - const snapshot: TextSnapshotNode = { + const node: TextSnapshotNode = { id: '1_1', role: 'root', name: 'root', @@ -139,7 +139,7 @@ describe('snapshotFormatter', () => { }, }; - const formatted = formatA11ySnapshot(snapshot); + const formatted = formatSnapshotNode(node); assert.strictEqual( formatted, `uid=1_1 root "root"