feat: read elements from DevTools UI

OrKoN · OrKoN · commit f8c676d74cfa · 2025-10-31T10:30:35.000+01:00
diff --git a/docs/tool-reference.md b/docs/tool-reference.md
@@ -329,7 +329,8 @@ so returned values have to JSON-serializable.
 ### `take_snapshot`
 
 **Description:** Take a text snapshot of the currently selected page based on the a11y tree. The snapshot lists page elements along with a unique
-identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot.
+identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot. The snapshot indicates the element selected
+in the DevTools Elements panel (if any).
 
 **Parameters:**
 
diff --git a/src/McpContext.ts b/src/McpContext.ts
@@ -25,19 +25,21 @@ import type {
 import {listPages} from './tools/pages.js';
 import {takeSnapshot} from './tools/snapshot.js';
 import {CLOSE_PAGE_ERROR} from './tools/ToolDefinition.js';
-import type {Context} from './tools/ToolDefinition.js';
+import type {Context, DevToolsData} from './tools/ToolDefinition.js';
 import type {TraceResult} from './trace-processing/parse.js';
 import {WaitForHelper} from './WaitForHelper.js';
 
 export interface TextSnapshotNode extends SerializedAXNode {
   id: string;
+  backendNodeId?: number;
   children: TextSnapshotNode[];
 }
 
 export interface TextSnapshot {
   root: TextSnapshotNode;
   idToNode: Map<string, TextSnapshotNode>;
   snapshotId: string;
+  selectedElementUid?: string;
 }
 
 interface McpContextOptions {
@@ -151,6 +153,42 @@ export class McpContext implements Context {
     return context;
   }
 
+  resolveCdpRequestId(cdpRequestId: string): number | undefined {
+    const selectedPage = this.getSelectedPage();
+    if (!cdpRequestId) {
+      this.logger('no network request');
+      return;
+    }
+    const request = this.#networkCollector.find(selectedPage, request => {
+      // @ts-expect-error id is internal.
+      return request.id === cdpRequestId;
+    });
+    if (!request) {
+      this.logger('no network request for ' + cdpRequestId);
+      return;
+    }
+    return this.#networkCollector.getIdForResource(request);
+  }
+
+  resolveCdpElementId(cdpBackendNodeId: number): string | undefined {
+    if (!cdpBackendNodeId) {
+      this.logger('no cdpBackendNodeId');
+      return;
+    }
+    // TODO: index by backendNodeId instead.
+    const queue = [this.#textSnapshot?.root];
+    while (queue.length) {
+      const current = queue.pop()!;
+      if (current.backendNodeId === cdpBackendNodeId) {
+        return current.id;
+      }
+      for (const child of current.children) {
+        queue.push(child);
+      }
+    }
+    return;
+  }
+
   getNetworkRequests(includePreservedRequests?: boolean): HTTPRequest[] {
     const page = this.getSelectedPage();
     return this.#networkCollector.getData(page, includePreservedRequests);
@@ -380,49 +418,47 @@ export class McpContext implements Context {
     return this.#pageToDevToolsPage.get(page);
   }
 
-  async getDevToolsData(): Promise<undefined | {requestId?: number}> {
+  async getDevToolsData(): Promise<DevToolsData> {
     try {
+      this.logger('Getting DevTools UI data');
       const selectedPage = this.getSelectedPage();
       const devtoolsPage = this.getDevToolsPage(selectedPage);
-      if (devtoolsPage) {
-        const cdpRequestId = await devtoolsPage.evaluate(async () => {
+      if (!devtoolsPage) {
+        this.logger('No DevTools page detected');
+        return {};
+      }
+      const {cdpRequestId, cdpBackendNodeId} = await devtoolsPage.evaluate(
+        async () => {
           // @ts-expect-error no types
           const UI = await import('/bundled/ui/legacy/legacy.js');
           // @ts-expect-error no types
           const SDK = await import('/bundled/core/sdk/sdk.js');
           const request = UI.Context.Context.instance().flavor(
             SDK.NetworkRequest.NetworkRequest,
           );
-          return request?.requestId();
-        });
-        if (!cdpRequestId) {
-          this.logger('no context request');
-          return;
-        }
-        const request = this.#networkCollector.find(selectedPage, request => {
-          // @ts-expect-error id is internal.
-          return request.id === cdpRequestId;
-        });
-        if (!request) {
-          this.logger('no collected request for ' + cdpRequestId);
-          return;
-        }
-        return {
-          requestId: this.#networkCollector.getIdForResource(request),
-        };
-      } else {
-        this.logger('no devtools page deteched');
-      }
+          const node = UI.Context.Context.instance().flavor(
+            SDK.DOMModel.DOMNode,
+          );
+          return {
+            cdpRequestId: request?.requestId(),
+            cdpBackendNodeId: node?.backendNodeId(),
+          };
+        },
+      );
+      return {cdpBackendNodeId, cdpRequestId};
     } catch (err) {
       this.logger('error getting devtools data', err);
     }
-    return;
+    return {};
   }
 
   /**
    * Creates a text snapshot of a page.
    */
-  async createTextSnapshot(verbose = false): Promise<void> {
+  async createTextSnapshot(
+    verbose = false,
+    devtoolsData: DevToolsData | undefined = undefined,
+  ): Promise<void> {
     const page = this.getSelectedPage();
     const rootNode = await page.accessibility.snapshot({
       includeIframes: true,
@@ -465,6 +501,12 @@ export class McpContext implements Context {
       snapshotId: String(snapshotId),
       idToNode,
     };
+    const data = devtoolsData ?? (await this.getDevToolsData());
+    if (data?.cdpBackendNodeId) {
+      this.#textSnapshot.selectedElementUid = this.resolveCdpElementId(
+        data?.cdpBackendNodeId,
+      );
+    }
   }
 
   getTextSnapshot(): TextSnapshot | null {
diff --git a/src/McpResponse.ts b/src/McpResponse.ts
@@ -15,7 +15,7 @@ import {
   getShortDescriptionForRequest,
   getStatusFromRequest,
 } from './formatters/networkFormatter.js';
-import {formatA11ySnapshot} from './formatters/snapshotFormatter.js';
+import {formatSnapshotNode} from './formatters/snapshotFormatter.js';
 import type {McpContext} from './McpContext.js';
 import type {
   ConsoleMessage,
@@ -25,6 +25,7 @@ import type {
 } from './third_party/index.js';
 import {handleDialog} from './tools/pages.js';
 import type {
+  DevToolsData,
   ImageContentData,
   Response,
   SnapshotParams,
@@ -52,6 +53,11 @@ export class McpResponse implements Response {
     types?: string[];
     includePreservedMessages?: boolean;
   };
+  #devToolsData?: DevToolsData;
+
+  attachDevToolsData(data: DevToolsData): void {
+    this.#devToolsData = data;
+  }
 
   setIncludePages(value: boolean): void {
     this.#includePages = value;
@@ -179,17 +185,22 @@ export class McpResponse implements Response {
 
     let formattedSnapshot: string | undefined;
     if (this.#snapshotParams) {
-      await context.createTextSnapshot(this.#snapshotParams.verbose);
+      await context.createTextSnapshot(
+        this.#snapshotParams.verbose,
+        this.#devToolsData,
+      );
       const snapshot = context.getTextSnapshot();
       if (snapshot) {
         if (this.#snapshotParams.filePath) {
           await context.saveFile(
-            new TextEncoder().encode(formatA11ySnapshot(snapshot.root)),
+            new TextEncoder().encode(
+              formatSnapshotNode(snapshot.root, snapshot),
+            ),
             this.#snapshotParams.filePath,
           );
           formattedSnapshot = `Saved snapshot to ${this.#snapshotParams.filePath}.`;
         } else {
-          formattedSnapshot = formatA11ySnapshot(snapshot.root);
+          formattedSnapshot = formatSnapshotNode(snapshot.root, snapshot);
         }
       }
     }
diff --git a/src/formatters/snapshotFormatter.ts b/src/formatters/snapshotFormatter.ts
@@ -3,19 +3,26 @@
  * Copyright 2025 Google LLC
  * SPDX-License-Identifier: Apache-2.0
  */
-import type {TextSnapshotNode} from '../McpContext.js';
+import type {TextSnapshot, TextSnapshotNode} from '../McpContext.js';
 
-export function formatA11ySnapshot(
-  serializedAXNodeRoot: TextSnapshotNode,
+export function formatSnapshotNode(
+  root: TextSnapshotNode,
+  snapshot?: TextSnapshot,
   depth = 0,
 ): string {
   let result = '';
-  const attributes = getAttributes(serializedAXNodeRoot);
-  const line = ' '.repeat(depth * 2) + attributes.join(' ') + '\n';
+  const attributes = getAttributes(root);
+  const line =
+    ' '.repeat(depth * 2) +
+    attributes.join(' ') +
+    (root.id === snapshot?.selectedElementUid
+      ? ' [selected in the DevTools Elements panel]'
+      : '') +
+    '\n';
   result += line;
 
-  for (const child of serializedAXNodeRoot.children) {
-    result += formatA11ySnapshot(child, depth + 1);
+  for (const child of root.children) {
+    result += formatSnapshotNode(child, snapshot, depth + 1);
   }
 
   return result;
diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.ts
@@ -47,6 +47,11 @@ export interface SnapshotParams {
   filePath?: string;
 }
 
+export interface DevToolsData {
+  cdpRequestId?: string;
+  cdpBackendNodeId?: number;
+}
+
 export interface Response {
   appendResponseLine(value: string): void;
   setIncludePages(value: boolean): void;
@@ -69,6 +74,8 @@ export interface Response {
   attachImage(value: ImageContentData): void;
   attachNetworkRequest(reqid: number): void;
   attachConsoleMessage(msgid: number): void;
+  // Allows re-using DevTools data queried by some tools.
+  attachDevToolsData(data: DevToolsData): void;
 }
 
 /**
@@ -103,7 +110,15 @@ export type Context = Readonly<{
     text: string;
     timeout?: number | undefined;
   }): Promise<Element>;
-  getDevToolsData(): Promise<undefined | {requestId?: number}>;
+  getDevToolsData(): Promise<DevToolsData>;
+  /**
+   * Returns a reqid for a cdpRequestId.
+   */
+  resolveCdpRequestId(cdpRequestId: string): number | undefined;
+  /**
+   * Returns a reqid for a cdpRequestId.
+   */
+  resolveCdpElementId(cdpBackendNodeId: number): string | undefined;
 }>;
 
 export function defineTool<Schema extends zod.ZodRawShape>(
diff --git a/src/tools/network.ts b/src/tools/network.ts
@@ -72,12 +72,16 @@ export const listNetworkRequests = defineTool({
   },
   handler: async (request, response, context) => {
     const data = await context.getDevToolsData();
+    response.attachDevToolsData(data);
+    const reqid = data?.cdpRequestId
+      ? context.resolveCdpRequestId(data.cdpRequestId)
+      : undefined;
     response.setIncludeNetworkRequests(true, {
       pageSize: request.params.pageSize,
       pageIdx: request.params.pageIdx,
       resourceTypes: request.params.resourceTypes,
       includePreservedRequests: request.params.includePreservedRequests,
-      networkRequestIdInDevToolsUI: data?.requestId,
+      networkRequestIdInDevToolsUI: reqid,
     });
   },
 });
@@ -102,8 +106,12 @@ export const getNetworkRequest = defineTool({
       response.attachNetworkRequest(request.params.reqid);
     } else {
       const data = await context.getDevToolsData();
-      if (data?.requestId) {
-        response.attachNetworkRequest(data?.requestId);
+      response.attachDevToolsData(data);
+      const reqid = data?.cdpRequestId
+        ? context.resolveCdpRequestId(data.cdpRequestId)
+        : undefined;
+      if (reqid) {
+        response.attachNetworkRequest(reqid);
       } else {
         response.appendResponseLine(
           `Nothing is currently selected in the DevTools Network panel.`,
diff --git a/src/tools/snapshot.ts b/src/tools/snapshot.ts
@@ -12,7 +12,8 @@ import {defineTool, timeoutSchema} from './ToolDefinition.js';
 export const takeSnapshot = defineTool({
   name: 'take_snapshot',
   description: `Take a text snapshot of the currently selected page based on the a11y tree. The snapshot lists page elements along with a unique
-identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot.`,
+identifier (uid). Always use the latest snapshot. Prefer taking a snapshot over taking a screenshot. The snapshot indicates the element selected
+in the DevTools Elements panel (if any).`,
   annotations: {
     category: ToolCategory.DEBUGGING,
     // Not read-only due to filePath param.
diff --git a/tests/formatters/snapshotFormatter.test.ts b/tests/formatters/snapshotFormatter.test.ts
