diff --git a/actions/setup/js/parse_mcp_gateway_log.cjs b/actions/setup/js/parse_mcp_gateway_log.cjs index 3d7f36f4ba..367f575220 100644 --- a/actions/setup/js/parse_mcp_gateway_log.cjs +++ b/actions/setup/js/parse_mcp_gateway_log.cjs @@ -9,7 +9,8 @@ const { ERR_PARSE } = require("./error_codes.cjs"); /** * Parses MCP gateway logs and creates a step summary * Log file locations: - * - /tmp/gh-aw/mcp-logs/gateway.md (markdown summary from gateway, preferred) + * - /tmp/gh-aw/mcp-logs/gateway.jsonl (structured JSONL log, parsed for DIFC_FILTERED events) + * - /tmp/gh-aw/mcp-logs/gateway.md (markdown summary from gateway, preferred for general content) * - /tmp/gh-aw/mcp-logs/gateway.log (main gateway log, fallback) * - /tmp/gh-aw/mcp-logs/stderr.log (stderr output, fallback) */ @@ -22,6 +23,68 @@ function printAllGatewayFiles() { displayDirectories(gatewayDirs, 64 * 1024); } +/** + * Parses gateway.jsonl content and extracts DIFC_FILTERED events + * @param {string} jsonlContent - The gateway.jsonl file content + * @returns {Array} Array of DIFC_FILTERED event objects + */ +function parseGatewayJsonlForDifcFiltered(jsonlContent) { + const filteredEvents = []; + const lines = jsonlContent.split("\n"); + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed || !trimmed.includes("DIFC_FILTERED")) continue; + try { + const entry = JSON.parse(trimmed); + if (entry.type === "DIFC_FILTERED") { + filteredEvents.push(entry); + } + } catch { + // skip malformed lines + } + } + return filteredEvents; +} + +/** + * Generates a markdown summary section for DIFC_FILTERED events + * @param {Array} filteredEvents - Array of DIFC_FILTERED event objects + * @returns {string} Markdown section, or empty string if no events + */ +function generateDifcFilteredSummary(filteredEvents) { + if (!filteredEvents || filteredEvents.length === 0) return ""; + + const lines = []; + lines.push("
"); + lines.push(`🔒 DIFC Filtered Events (${filteredEvents.length})\n`); + lines.push(""); + lines.push("The following tool calls were blocked by DIFC integrity or secrecy checks:\n"); + lines.push(""); + lines.push("| Time | Server | Tool | Reason | User | Resource |"); + lines.push("|------|--------|------|--------|------|----------|"); + + for (const event of filteredEvents) { + const time = event.timestamp ? event.timestamp.replace("T", " ").replace(/\.\d+Z$/, "Z") : "-"; + const server = event.server_id || "-"; + const tool = event.tool_name ? `\`${event.tool_name}\`` : "-"; + const reason = (event.reason || "-").replace(/\n/g, " ").replace(/\|/g, "\\|"); + const user = event.author_login ? `${event.author_login} (${event.author_association || "NONE"})` : "-"; + let resource; + if (event.html_url) { + const lastSegment = event.html_url.split("/").filter(Boolean).pop(); + const label = event.number ? `#${event.number}` : lastSegment || event.html_url; + resource = `[${label}](${event.html_url})`; + } else { + resource = event.description || "-"; + } + lines.push(`| ${time} | ${server} | ${tool} | ${reason} | ${user} | ${resource} |`); + } + + lines.push(""); + lines.push("
\n"); + return lines.join("\n"); +} + /** * Main function to parse and display MCP gateway logs */ @@ -30,18 +93,49 @@ async function main() { // First, print all gateway-related files for debugging printAllGatewayFiles(); + const gatewayJsonlPath = "/tmp/gh-aw/mcp-logs/gateway.jsonl"; + const rpcMessagesPath = "/tmp/gh-aw/mcp-logs/rpc-messages.jsonl"; const gatewayMdPath = "/tmp/gh-aw/mcp-logs/gateway.md"; const gatewayLogPath = "/tmp/gh-aw/mcp-logs/gateway.log"; const stderrLogPath = "/tmp/gh-aw/mcp-logs/stderr.log"; - // First, try to read gateway.md if it exists + // Parse DIFC_FILTERED events from gateway.jsonl (preferred) or rpc-messages.jsonl (fallback). + // Both files use the same JSONL format with DIFC_FILTERED entries interleaved. + let difcFilteredEvents = []; + if (fs.existsSync(gatewayJsonlPath)) { + const jsonlContent = fs.readFileSync(gatewayJsonlPath, "utf8"); + core.info(`Found gateway.jsonl (${jsonlContent.length} bytes)`); + difcFilteredEvents = parseGatewayJsonlForDifcFiltered(jsonlContent); + if (difcFilteredEvents.length > 0) { + core.info(`Found ${difcFilteredEvents.length} DIFC_FILTERED event(s) in gateway.jsonl`); + } + } else if (fs.existsSync(rpcMessagesPath)) { + const jsonlContent = fs.readFileSync(rpcMessagesPath, "utf8"); + core.info(`No gateway.jsonl found; scanning rpc-messages.jsonl (${jsonlContent.length} bytes) for DIFC_FILTERED events`); + difcFilteredEvents = parseGatewayJsonlForDifcFiltered(jsonlContent); + if (difcFilteredEvents.length > 0) { + core.info(`Found ${difcFilteredEvents.length} DIFC_FILTERED event(s) in rpc-messages.jsonl`); + } + } else { + core.info(`No gateway.jsonl or rpc-messages.jsonl found for DIFC_FILTERED scanning`); + } + + // Try to read gateway.md if it exists (preferred for general gateway summary) if (fs.existsSync(gatewayMdPath)) { const gatewayMdContent = fs.readFileSync(gatewayMdPath, "utf8"); if (gatewayMdContent && gatewayMdContent.trim().length > 0) { core.info(`Found gateway.md (${gatewayMdContent.length} bytes)`); // Write the markdown directly to the step summary - core.summary.addRaw(gatewayMdContent).write(); + core.summary.addRaw(gatewayMdContent.endsWith("\n") ? gatewayMdContent : gatewayMdContent + "\n"); + + // Append DIFC_FILTERED section if any events found + if (difcFilteredEvents.length > 0) { + const difcSummary = generateDifcFilteredSummary(difcFilteredEvents); + core.summary.addRaw(difcSummary); + } + + core.summary.write(); return; } } else { @@ -68,19 +162,26 @@ async function main() { core.info(`No stderr.log found at: ${stderrLogPath}`); } - // If neither log file has content, nothing to do - if ((!gatewayLogContent || gatewayLogContent.trim().length === 0) && (!stderrLogContent || stderrLogContent.trim().length === 0)) { + // If no legacy log content and no DIFC events, nothing to do + if ((!gatewayLogContent || gatewayLogContent.trim().length === 0) && (!stderrLogContent || stderrLogContent.trim().length === 0) && difcFilteredEvents.length === 0) { core.info("MCP gateway log files are empty or missing"); return; } // Generate plain text summary for core.info - const plainTextSummary = generatePlainTextLegacySummary(gatewayLogContent, stderrLogContent); - core.info(plainTextSummary); + if ((gatewayLogContent && gatewayLogContent.trim().length > 0) || (stderrLogContent && stderrLogContent.trim().length > 0)) { + const plainTextSummary = generatePlainTextLegacySummary(gatewayLogContent, stderrLogContent); + core.info(plainTextSummary); + } - // Generate step summary for both logs - const summary = generateGatewayLogSummary(gatewayLogContent, stderrLogContent); - core.summary.addRaw(summary).write(); + // Generate step summary: legacy logs + DIFC filtered section + const legacySummary = generateGatewayLogSummary(gatewayLogContent, stderrLogContent); + const difcSummary = generateDifcFilteredSummary(difcFilteredEvents); + const fullSummary = [legacySummary, difcSummary].filter(s => s.length > 0).join("\n"); + + if (fullSummary.length > 0) { + core.summary.addRaw(fullSummary).write(); + } } catch (error) { core.setFailed(`${ERR_PARSE}: ${getErrorMessage(error)}`); } @@ -195,6 +296,8 @@ if (typeof module !== "undefined" && module.exports) { generateGatewayLogSummary, generatePlainTextGatewaySummary, generatePlainTextLegacySummary, + parseGatewayJsonlForDifcFiltered, + generateDifcFilteredSummary, printAllGatewayFiles, }; } diff --git a/actions/setup/js/parse_mcp_gateway_log.test.cjs b/actions/setup/js/parse_mcp_gateway_log.test.cjs index f2c3291357..ca9614d4f2 100644 --- a/actions/setup/js/parse_mcp_gateway_log.test.cjs +++ b/actions/setup/js/parse_mcp_gateway_log.test.cjs @@ -1,7 +1,7 @@ // @ts-check /// -const { generateGatewayLogSummary, generatePlainTextGatewaySummary, generatePlainTextLegacySummary, printAllGatewayFiles } = require("./parse_mcp_gateway_log.cjs"); +const { generateGatewayLogSummary, generatePlainTextGatewaySummary, generatePlainTextLegacySummary, parseGatewayJsonlForDifcFiltered, generateDifcFilteredSummary, printAllGatewayFiles } = require("./parse_mcp_gateway_log.cjs"); describe("parse_mcp_gateway_log", () => { // Note: The main() function now checks for gateway.md first before falling back to log files. @@ -569,4 +569,158 @@ Some content here.`; } }); }); + + describe("parseGatewayJsonlForDifcFiltered", () => { + test("extracts DIFC_FILTERED events from JSONL content", () => { + const jsonlContent = [ + JSON.stringify({ + timestamp: "2026-03-18T17:30:00.123456789Z", + type: "DIFC_FILTERED", + server_id: "github", + tool_name: "list_issues", + description: "resource:list_issues", + reason: "Integrity check failed, missingTags=[approved:github/copilot-indexing-issues-prs]", + secrecy_tags: ["private:github/copilot-indexing-issues-prs"], + integrity_tags: ["none:github/copilot-indexing-issues-prs"], + author_association: "NONE", + author_login: "external-user", + html_url: "https://github.com/github/copilot-indexing-issues-prs/issues/42", + number: "42", + }), + JSON.stringify({ timestamp: "2026-03-18T17:30:01Z", type: "RESPONSE", server_id: "github" }), + JSON.stringify({ + timestamp: "2026-03-18T17:31:00Z", + type: "DIFC_FILTERED", + server_id: "github", + tool_name: "get_issue", + reason: "Secrecy check failed", + author_login: "user2", + }), + ].join("\n"); + + const events = parseGatewayJsonlForDifcFiltered(jsonlContent); + + expect(events).toHaveLength(2); + expect(events[0].tool_name).toBe("list_issues"); + expect(events[0].server_id).toBe("github"); + expect(events[0].author_login).toBe("external-user"); + expect(events[1].tool_name).toBe("get_issue"); + }); + + test("returns empty array when no DIFC_FILTERED events", () => { + const jsonlContent = [JSON.stringify({ timestamp: "2026-03-18T17:30:01Z", type: "RESPONSE", server_id: "github" }), JSON.stringify({ timestamp: "2026-03-18T17:30:02Z", type: "REQUEST", server_id: "github" })].join("\n"); + + const events = parseGatewayJsonlForDifcFiltered(jsonlContent); + expect(events).toHaveLength(0); + }); + + test("returns empty array for empty content", () => { + expect(parseGatewayJsonlForDifcFiltered("")).toHaveLength(0); + }); + + test("skips malformed JSON lines", () => { + const jsonlContent = ["not valid json", JSON.stringify({ type: "DIFC_FILTERED", tool_name: "valid_tool" }), "{broken}"].join("\n"); + + const events = parseGatewayJsonlForDifcFiltered(jsonlContent); + expect(events).toHaveLength(1); + expect(events[0].tool_name).toBe("valid_tool"); + }); + + test("skips blank lines", () => { + const jsonlContent = "\n" + JSON.stringify({ type: "DIFC_FILTERED", tool_name: "t1" }) + "\n\n" + JSON.stringify({ type: "DIFC_FILTERED", tool_name: "t2" }) + "\n"; + + const events = parseGatewayJsonlForDifcFiltered(jsonlContent); + expect(events).toHaveLength(2); + }); + }); + + describe("generateDifcFilteredSummary", () => { + const sampleEvents = [ + { + timestamp: "2026-03-18T17:30:00.123456789Z", + type: "DIFC_FILTERED", + server_id: "github", + tool_name: "list_issues", + description: "resource:list_issues", + reason: "Integrity check failed, missingTags=[approved:github/copilot-indexing-issues-prs]", + secrecy_tags: ["private:github/copilot-indexing-issues-prs"], + integrity_tags: ["none:github/copilot-indexing-issues-prs"], + author_association: "NONE", + author_login: "external-user", + html_url: "https://github.com/github/copilot-indexing-issues-prs/issues/42", + number: "42", + }, + ]; + + test("returns empty string for empty events array", () => { + expect(generateDifcFilteredSummary([])).toBe(""); + }); + + test("returns empty string for null/undefined", () => { + expect(generateDifcFilteredSummary(null)).toBe(""); + expect(generateDifcFilteredSummary(undefined)).toBe(""); + }); + + test("generates details/summary section with event count", () => { + const summary = generateDifcFilteredSummary(sampleEvents); + expect(summary).toContain("
"); + expect(summary).toContain("DIFC Filtered Events (1)"); + expect(summary).toContain("
"); + }); + + test("includes tool name in code formatting", () => { + const summary = generateDifcFilteredSummary(sampleEvents); + expect(summary).toContain("`list_issues`"); + }); + + test("includes server_id", () => { + const summary = generateDifcFilteredSummary(sampleEvents); + expect(summary).toContain("github"); + }); + + test("includes reason for filtering", () => { + const summary = generateDifcFilteredSummary(sampleEvents); + expect(summary).toContain("Integrity check failed"); + }); + + test("includes author login and association", () => { + const summary = generateDifcFilteredSummary(sampleEvents); + expect(summary).toContain("external-user"); + expect(summary).toContain("NONE"); + }); + + test("renders resource as linked issue number", () => { + const summary = generateDifcFilteredSummary(sampleEvents); + expect(summary).toContain("[#42]"); + expect(summary).toContain("https://github.com/github/copilot-indexing-issues-prs/issues/42"); + }); + + test("uses description as resource when html_url absent", () => { + const events = [{ type: "DIFC_FILTERED", tool_name: "my_tool", description: "resource:my_tool" }]; + const summary = generateDifcFilteredSummary(events); + expect(summary).toContain("resource:my_tool"); + }); + + test("escapes pipe characters in reason", () => { + const events = [{ type: "DIFC_FILTERED", tool_name: "t", reason: "failed | check" }]; + const summary = generateDifcFilteredSummary(events); + expect(summary).toContain("failed \\| check"); + }); + + test("generates correct table header", () => { + const summary = generateDifcFilteredSummary(sampleEvents); + expect(summary).toContain("| Time | Server | Tool | Reason | User | Resource |"); + expect(summary).toContain("|------|--------|------|--------|------|----------|"); + }); + + test("shows event count in summary for multiple events", () => { + const multiEvents = [ + { type: "DIFC_FILTERED", tool_name: "t1", reason: "r1" }, + { type: "DIFC_FILTERED", tool_name: "t2", reason: "r2" }, + { type: "DIFC_FILTERED", tool_name: "t3", reason: "r3" }, + ]; + const summary = generateDifcFilteredSummary(multiEvents); + expect(summary).toContain("DIFC Filtered Events (3)"); + }); + }); }); diff --git a/pkg/cli/audit_report.go b/pkg/cli/audit_report.go index b591e0822b..756b356b1e 100644 --- a/pkg/cli/audit_report.go +++ b/pkg/cli/audit_report.go @@ -136,9 +136,10 @@ type ToolUsageInfo struct { // MCPToolUsageData contains detailed MCP tool usage statistics and individual call records type MCPToolUsageData struct { - Summary []MCPToolSummary `json:"summary"` // Aggregated statistics per tool - ToolCalls []MCPToolCall `json:"tool_calls"` // Individual tool call records - Servers []MCPServerStats `json:"servers,omitempty"` // Server-level statistics + Summary []MCPToolSummary `json:"summary"` // Aggregated statistics per tool + ToolCalls []MCPToolCall `json:"tool_calls"` // Individual tool call records + Servers []MCPServerStats `json:"servers,omitempty"` // Server-level statistics + FilteredEvents []DifcFilteredEvent `json:"filtered_events,omitempty"` // DIFC filtered events } // MCPToolSummary contains aggregated statistics for a single MCP tool diff --git a/pkg/cli/gateway_logs.go b/pkg/cli/gateway_logs.go index cd17e14295..790c2df8f9 100644 --- a/pkg/cli/gateway_logs.go +++ b/pkg/cli/gateway_logs.go @@ -36,19 +36,44 @@ const maxScannerBufferSize = 1024 * 1024 // GatewayLogEntry represents a single log entry from gateway.jsonl type GatewayLogEntry struct { - Timestamp string `json:"timestamp"` - Level string `json:"level"` - Type string `json:"type"` - Event string `json:"event"` - ServerName string `json:"server_name,omitempty"` - ToolName string `json:"tool_name,omitempty"` - Method string `json:"method,omitempty"` - Duration float64 `json:"duration,omitempty"` // in milliseconds - InputSize int `json:"input_size,omitempty"` - OutputSize int `json:"output_size,omitempty"` - Status string `json:"status,omitempty"` - Error string `json:"error,omitempty"` - Message string `json:"message,omitempty"` + Timestamp string `json:"timestamp"` + Level string `json:"level"` + Type string `json:"type"` + Event string `json:"event"` + ServerName string `json:"server_name,omitempty"` + ServerID string `json:"server_id,omitempty"` // used by DIFC_FILTERED events + ToolName string `json:"tool_name,omitempty"` + Method string `json:"method,omitempty"` + Duration float64 `json:"duration,omitempty"` // in milliseconds + InputSize int `json:"input_size,omitempty"` + OutputSize int `json:"output_size,omitempty"` + Status string `json:"status,omitempty"` + Error string `json:"error,omitempty"` + Message string `json:"message,omitempty"` + Description string `json:"description,omitempty"` + Reason string `json:"reason,omitempty"` + SecrecyTags []string `json:"secrecy_tags,omitempty"` + IntegrityTags []string `json:"integrity_tags,omitempty"` + AuthorAssociation string `json:"author_association,omitempty"` + AuthorLogin string `json:"author_login,omitempty"` + HTMLURL string `json:"html_url,omitempty"` + Number string `json:"number,omitempty"` +} + +// DifcFilteredEvent represents a DIFC_FILTERED log entry from gateway.jsonl. +// These events occur when a tool call is blocked by DIFC integrity or secrecy checks. +type DifcFilteredEvent struct { + Timestamp string `json:"timestamp"` + ServerID string `json:"server_id"` + ToolName string `json:"tool_name"` + Description string `json:"description,omitempty"` + Reason string `json:"reason"` + SecrecyTags []string `json:"secrecy_tags,omitempty"` + IntegrityTags []string `json:"integrity_tags,omitempty"` + AuthorAssociation string `json:"author_association,omitempty"` + AuthorLogin string `json:"author_login,omitempty"` + HTMLURL string `json:"html_url,omitempty"` + Number string `json:"number,omitempty"` } // GatewayServerMetrics represents usage metrics for a single MCP server @@ -58,6 +83,7 @@ type GatewayServerMetrics struct { ToolCallCount int TotalDuration float64 // in milliseconds ErrorCount int + FilteredCount int // number of DIFC_FILTERED events for this server Tools map[string]*GatewayToolMetrics } @@ -79,7 +105,9 @@ type GatewayMetrics struct { TotalRequests int TotalToolCalls int TotalErrors int + TotalFiltered int // number of DIFC_FILTERED events Servers map[string]*GatewayServerMetrics + FilteredEvents []DifcFilteredEvent StartTime time.Time EndTime time.Time TotalDuration float64 // in milliseconds @@ -87,13 +115,23 @@ type GatewayMetrics struct { // RPCMessageEntry represents a single entry from rpc-messages.jsonl. // This file is written by the Copilot CLI and contains raw JSON-RPC protocol messages -// exchanged between the AI engine and MCP servers. +// exchanged between the AI engine and MCP servers, as well as DIFC_FILTERED events. type RPCMessageEntry struct { Timestamp string `json:"timestamp"` - Direction string `json:"direction"` // "IN" = received from server, "OUT" = sent to server - Type string `json:"type"` // "REQUEST" or "RESPONSE" + Direction string `json:"direction"` // "IN" = received from server, "OUT" = sent to server; empty for DIFC_FILTERED + Type string `json:"type"` // "REQUEST", "RESPONSE", or "DIFC_FILTERED" ServerID string `json:"server_id"` Payload json.RawMessage `json:"payload"` + // Fields populated only for DIFC_FILTERED entries + ToolName string `json:"tool_name,omitempty"` + Description string `json:"description,omitempty"` + Reason string `json:"reason,omitempty"` + SecrecyTags []string `json:"secrecy_tags,omitempty"` + IntegrityTags []string `json:"integrity_tags,omitempty"` + AuthorAssociation string `json:"author_association,omitempty"` + AuthorLogin string `json:"author_login,omitempty"` + HTMLURL string `json:"html_url,omitempty"` + Number string `json:"number,omitempty"` } // rpcRequestPayload represents the JSON-RPC request payload fields we care about. @@ -186,6 +224,25 @@ func parseRPCMessages(logPath string, verbose bool) (*GatewayMetrics, error) { } switch { + case entry.Type == "DIFC_FILTERED": + // DIFC integrity/secrecy filter event — not a REQUEST or RESPONSE + metrics.TotalFiltered++ + server := getOrCreateServer(metrics, entry.ServerID) + server.FilteredCount++ + metrics.FilteredEvents = append(metrics.FilteredEvents, DifcFilteredEvent{ + Timestamp: entry.Timestamp, + ServerID: entry.ServerID, + ToolName: entry.ToolName, + Description: entry.Description, + Reason: entry.Reason, + SecrecyTags: entry.SecrecyTags, + IntegrityTags: entry.IntegrityTags, + AuthorAssociation: entry.AuthorAssociation, + AuthorLogin: entry.AuthorLogin, + HTMLURL: entry.HTMLURL, + Number: entry.Number, + }) + case entry.Direction == "OUT" && entry.Type == "REQUEST": // Outgoing request from AI engine to MCP server var req rpcRequestPayload @@ -375,9 +432,13 @@ func parseGatewayLogs(logDir string, verbose bool) (*GatewayMetrics, error) { // processGatewayLogEntry processes a single log entry and updates metrics func processGatewayLogEntry(entry *GatewayLogEntry, metrics *GatewayMetrics, verbose bool) { - // Parse timestamp for time range + // Parse timestamp for time range (supports both RFC3339 and RFC3339Nano) if entry.Timestamp != "" { - if t, err := time.Parse(time.RFC3339, entry.Timestamp); err == nil { + t, err := time.Parse(time.RFC3339Nano, entry.Timestamp) + if err != nil { + t, err = time.Parse(time.RFC3339, entry.Timestamp) + } + if err == nil { if metrics.StartTime.IsZero() || t.Before(metrics.StartTime) { metrics.StartTime = t } @@ -387,6 +448,34 @@ func processGatewayLogEntry(entry *GatewayLogEntry, metrics *GatewayMetrics, ver } } + // Handle DIFC_FILTERED events + if entry.Type == "DIFC_FILTERED" { + metrics.TotalFiltered++ + // DIFC_FILTERED events use server_id; fall back to server_name for compatibility + serverKey := entry.ServerID + if serverKey == "" { + serverKey = entry.ServerName + } + if serverKey != "" { + server := getOrCreateServer(metrics, serverKey) + server.FilteredCount++ + } + metrics.FilteredEvents = append(metrics.FilteredEvents, DifcFilteredEvent{ + Timestamp: entry.Timestamp, + ServerID: serverKey, + ToolName: entry.ToolName, + Description: entry.Description, + Reason: entry.Reason, + SecrecyTags: entry.SecrecyTags, + IntegrityTags: entry.IntegrityTags, + AuthorAssociation: entry.AuthorAssociation, + AuthorLogin: entry.AuthorLogin, + HTMLURL: entry.HTMLURL, + Number: entry.Number, + }) + return + } + // Track errors if entry.Status == "error" || entry.Error != "" { metrics.TotalErrors++ @@ -503,6 +592,9 @@ func renderGatewayMetricsTable(metrics *GatewayMetrics, verbose bool) string { fmt.Fprintf(&output, "Total Requests: %d\n", metrics.TotalRequests) fmt.Fprintf(&output, "Total Tool Calls: %d\n", metrics.TotalToolCalls) fmt.Fprintf(&output, "Total Errors: %d\n", metrics.TotalErrors) + if metrics.TotalFiltered > 0 { + fmt.Fprintf(&output, "Total DIFC Filtered: %d\n", metrics.TotalFiltered) + } fmt.Fprintf(&output, "Servers: %d\n", len(metrics.Servers)) if !metrics.StartTime.IsZero() && !metrics.EndTime.IsZero() { @@ -517,6 +609,7 @@ func renderGatewayMetricsTable(metrics *GatewayMetrics, verbose bool) string { // Sort servers by request count serverNames := getSortedServerNames(metrics) + hasFiltered := metrics.TotalFiltered > 0 serverRows := make([][]string, 0, len(serverNames)) for _, serverName := range serverNames { server := metrics.Servers[serverName] @@ -524,19 +617,61 @@ func renderGatewayMetricsTable(metrics *GatewayMetrics, verbose bool) string { if server.RequestCount > 0 { avgTime = server.TotalDuration / float64(server.RequestCount) } - serverRows = append(serverRows, []string{ - serverName, - strconv.Itoa(server.RequestCount), - strconv.Itoa(server.ToolCallCount), - fmt.Sprintf("%.0fms", avgTime), - strconv.Itoa(server.ErrorCount), - }) + if hasFiltered { + serverRows = append(serverRows, []string{ + serverName, + strconv.Itoa(server.RequestCount), + strconv.Itoa(server.ToolCallCount), + fmt.Sprintf("%.0fms", avgTime), + strconv.Itoa(server.ErrorCount), + strconv.Itoa(server.FilteredCount), + }) + } else { + serverRows = append(serverRows, []string{ + serverName, + strconv.Itoa(server.RequestCount), + strconv.Itoa(server.ToolCallCount), + fmt.Sprintf("%.0fms", avgTime), + strconv.Itoa(server.ErrorCount), + }) + } + } + + if hasFiltered { + output.WriteString(console.RenderTable(console.TableConfig{ + Title: "Server Usage", + Headers: []string{"Server", "Requests", "Tool Calls", "Avg Time", "Errors", "Filtered"}, + Rows: serverRows, + })) + } else { + output.WriteString(console.RenderTable(console.TableConfig{ + Title: "Server Usage", + Headers: []string{"Server", "Requests", "Tool Calls", "Avg Time", "Errors"}, + Rows: serverRows, + })) } + } + // DIFC filtered events table + if len(metrics.FilteredEvents) > 0 { + output.WriteString("\n") + filteredRows := make([][]string, 0, len(metrics.FilteredEvents)) + for _, fe := range metrics.FilteredEvents { + reason := fe.Reason + if len(reason) > 80 { + reason = reason[:77] + "..." + } + filteredRows = append(filteredRows, []string{ + fe.ServerID, + fe.ToolName, + fe.AuthorLogin, + reason, + }) + } output.WriteString(console.RenderTable(console.TableConfig{ - Title: "Server Usage", - Headers: []string{"Server", "Requests", "Tool Calls", "Avg Time", "Errors"}, - Rows: serverRows, + Title: "DIFC Filtered Events", + Headers: []string{"Server", "Tool", "User", "Reason"}, + Rows: filteredRows, })) } @@ -738,9 +873,10 @@ func extractMCPToolUsageData(logDir string, verbose bool) (*MCPToolUsageData, er } mcpData := &MCPToolUsageData{ - Summary: []MCPToolSummary{}, - ToolCalls: []MCPToolCall{}, - Servers: []MCPServerStats{}, + Summary: []MCPToolSummary{}, + ToolCalls: []MCPToolCall{}, + Servers: []MCPServerStats{}, + FilteredEvents: gatewayMetrics.FilteredEvents, } // Read the log file again to get individual tool call records. @@ -930,7 +1066,9 @@ func displayAggregatedGatewayMetrics(processedRuns []ProcessedRun, outputDir str aggregated.TotalRequests += runMetrics.TotalRequests aggregated.TotalToolCalls += runMetrics.TotalToolCalls aggregated.TotalErrors += runMetrics.TotalErrors + aggregated.TotalFiltered += runMetrics.TotalFiltered aggregated.TotalDuration += runMetrics.TotalDuration + aggregated.FilteredEvents = append(aggregated.FilteredEvents, runMetrics.FilteredEvents...) // Merge server metrics for serverName, serverMetrics := range runMetrics.Servers { @@ -939,6 +1077,7 @@ func displayAggregatedGatewayMetrics(processedRuns []ProcessedRun, outputDir str aggServer.ToolCallCount += serverMetrics.ToolCallCount aggServer.TotalDuration += serverMetrics.TotalDuration aggServer.ErrorCount += serverMetrics.ErrorCount + aggServer.FilteredCount += serverMetrics.FilteredCount // Merge tool metrics for toolName, toolMetrics := range serverMetrics.Tools { diff --git a/pkg/cli/gateway_logs_test.go b/pkg/cli/gateway_logs_test.go index 4b2f83aff1..5845548d0e 100644 --- a/pkg/cli/gateway_logs_test.go +++ b/pkg/cli/gateway_logs_test.go @@ -322,6 +322,86 @@ func TestProcessGatewayLogEntry(t *testing.T) { assert.Equal(t, 1, server.ErrorCount) } +func TestProcessGatewayLogEntryDifcFiltered(t *testing.T) { + metrics := &GatewayMetrics{ + Servers: make(map[string]*GatewayServerMetrics), + } + + entry := &GatewayLogEntry{ + Timestamp: "2024-01-12T10:00:00Z", + Type: "DIFC_FILTERED", + ServerID: "github", + ToolName: "pull_request_read", + Reason: "Resource has lower integrity than agent requires.", + AuthorLogin: "octocat", + AuthorAssociation: "CONTRIBUTOR", + HTMLURL: "https://github.com/github/gh-aw/pull/42", + Number: "42", + } + + processGatewayLogEntry(entry, metrics, false) + + assert.Equal(t, 0, metrics.TotalRequests, "DIFC_FILTERED should not increment TotalRequests") + assert.Equal(t, 1, metrics.TotalFiltered, "DIFC_FILTERED should increment TotalFiltered") + require.Len(t, metrics.FilteredEvents, 1, "should record one filtered event") + + evt := metrics.FilteredEvents[0] + assert.Equal(t, "github", evt.ServerID) + assert.Equal(t, "pull_request_read", evt.ToolName) + assert.Equal(t, "Resource has lower integrity than agent requires.", evt.Reason) + assert.Equal(t, "octocat", evt.AuthorLogin) + assert.Equal(t, "CONTRIBUTOR", evt.AuthorAssociation) + assert.Equal(t, "https://github.com/github/gh-aw/pull/42", evt.HTMLURL) + assert.Equal(t, "42", evt.Number) + + require.Len(t, metrics.Servers, 1, "should create server entry for DIFC_FILTERED server") + githubServer := metrics.Servers["github"] + require.NotNil(t, githubServer) + assert.Equal(t, 1, githubServer.FilteredCount) +} + +func TestParseRPCMessagesDifcFiltered(t *testing.T) { + tmpDir := t.TempDir() + + content := `{"timestamp":"2024-01-12T10:00:00.000000000Z","type":"DIFC_FILTERED","server_id":"github","tool_name":"pull_request_read","reason":"Resource has lower integrity than agent requires.","author_login":"octocat","author_association":"CONTRIBUTOR","html_url":"https://github.com/github/gh-aw/pull/42","number":"42"} +{"timestamp":"2024-01-12T10:00:01.000000000Z","direction":"OUT","type":"REQUEST","server_id":"github","payload":{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"list_issues","arguments":{}}}} +{"timestamp":"2024-01-12T10:00:01.200000000Z","direction":"IN","type":"RESPONSE","server_id":"github","payload":{"jsonrpc":"2.0","id":1,"result":{}}} +{"timestamp":"2024-01-12T10:00:02.000000000Z","type":"DIFC_FILTERED","server_id":"github","tool_name":"issue_read","reason":"Secrecy violation.","secrecy_tags":["private"]} +` + logPath := filepath.Join(tmpDir, "rpc-messages.jsonl") + require.NoError(t, os.WriteFile(logPath, []byte(content), 0644)) + + metrics, err := parseRPCMessages(logPath, false) + require.NoError(t, err) + require.NotNil(t, metrics) + + assert.Equal(t, 2, metrics.TotalFiltered, "should count 2 DIFC_FILTERED events") + assert.Equal(t, 1, metrics.TotalRequests, "should count 1 REQUEST") + require.Len(t, metrics.FilteredEvents, 2) + + // First filtered event — with user and resource metadata + first := metrics.FilteredEvents[0] + assert.Equal(t, "github", first.ServerID) + assert.Equal(t, "pull_request_read", first.ToolName) + assert.Equal(t, "Resource has lower integrity than agent requires.", first.Reason) + assert.Equal(t, "octocat", first.AuthorLogin) + assert.Equal(t, "CONTRIBUTOR", first.AuthorAssociation) + assert.Equal(t, "https://github.com/github/gh-aw/pull/42", first.HTMLURL) + assert.Equal(t, "42", first.Number) + + // Second filtered event — with secrecy tags + second := metrics.FilteredEvents[1] + assert.Equal(t, "github", second.ServerID) + assert.Equal(t, "issue_read", second.ToolName) + assert.Equal(t, "Secrecy violation.", second.Reason) + assert.Equal(t, []string{"private"}, second.SecrecyTags) + + // Server should have FilteredCount = 2 + githubServer := metrics.Servers["github"] + require.NotNil(t, githubServer) + assert.Equal(t, 2, githubServer.FilteredCount) +} + func TestGetSortedServerNames(t *testing.T) { metrics := &GatewayMetrics{ Servers: map[string]*GatewayServerMetrics{ diff --git a/pkg/cli/logs_mcp_tool_usage_test.go b/pkg/cli/logs_mcp_tool_usage_test.go index 4dc2da11d1..a069142cbc 100644 --- a/pkg/cli/logs_mcp_tool_usage_test.go +++ b/pkg/cli/logs_mcp_tool_usage_test.go @@ -368,3 +368,43 @@ func TestBuildMCPToolUsageSummarySorting(t *testing.T) { assert.Equal(t, "playwright", summary.Summary[2].ServerName) assert.Equal(t, "navigate", summary.Summary[2].ToolName) } + +func TestBuildMCPToolUsageSummaryFilteredEvents(t *testing.T) { + // Verify FilteredEvents are aggregated across runs and that a non-nil summary + // is returned when filtered events exist even if there is no tool usage data. + event1 := DifcFilteredEvent{ + Timestamp: "2024-01-12T10:00:00Z", + ServerID: "github", + ToolName: "pull_request_read", + Reason: "integrity check failed", + } + event2 := DifcFilteredEvent{ + Timestamp: "2024-01-12T10:00:01Z", + ServerID: "github", + ToolName: "issue_read", + Reason: "secrecy violation", + } + + processedRuns := []ProcessedRun{ + { + Run: WorkflowRun{DatabaseID: 1}, + MCPToolUsage: &MCPToolUsageData{ + FilteredEvents: []DifcFilteredEvent{event1}, + }, + }, + { + Run: WorkflowRun{DatabaseID: 2}, + MCPToolUsage: &MCPToolUsageData{ + FilteredEvents: []DifcFilteredEvent{event2}, + }, + }, + } + + summary := buildMCPToolUsageSummary(processedRuns) + + // Should not be nil even though there is no tool data + require.NotNil(t, summary, "summary should not be nil when filtered events exist") + require.Len(t, summary.FilteredEvents, 2, "should aggregate filtered events from all runs") + assert.Equal(t, event1, summary.FilteredEvents[0]) + assert.Equal(t, event2, summary.FilteredEvents[1]) +} diff --git a/pkg/cli/logs_models.go b/pkg/cli/logs_models.go index df64463322..70c3571496 100644 --- a/pkg/cli/logs_models.go +++ b/pkg/cli/logs_models.go @@ -151,9 +151,10 @@ type MissingDataSummary struct { // MCPToolUsageSummary aggregates MCP tool usage across all runs type MCPToolUsageSummary struct { - Summary []MCPToolSummary `json:"summary" console:"title:Tool Statistics"` // Aggregated statistics per tool - Servers []MCPServerStats `json:"servers,omitempty" console:"title:Server Statistics"` // Server-level statistics - ToolCalls []MCPToolCall `json:"tool_calls" console:"-"` // Individual tool call records (excluded from console) + Summary []MCPToolSummary `json:"summary" console:"title:Tool Statistics"` // Aggregated statistics per tool + Servers []MCPServerStats `json:"servers,omitempty" console:"title:Server Statistics"` // Server-level statistics + ToolCalls []MCPToolCall `json:"tool_calls" console:"-"` // Individual tool call records (excluded from console) + FilteredEvents []DifcFilteredEvent `json:"filtered_events,omitempty" console:"-"` // DIFC filtered events (excluded from console display) } // ErrNoArtifacts indicates that a workflow run has no artifacts diff --git a/pkg/cli/logs_report.go b/pkg/cli/logs_report.go index 8df933ddd4..93190cf735 100644 --- a/pkg/cli/logs_report.go +++ b/pkg/cli/logs_report.go @@ -721,6 +721,7 @@ func buildMCPToolUsageSummary(processedRuns []ProcessedRun) *MCPToolUsageSummary toolSummaryMap := make(map[string]*MCPToolSummary) // Key: serverName:toolName serverStatsMap := make(map[string]*MCPServerStats) // Key: serverName var allToolCalls []MCPToolCall + var allFilteredEvents []DifcFilteredEvent // Aggregate data from all runs for _, pr := range processedRuns { @@ -731,6 +732,9 @@ func buildMCPToolUsageSummary(processedRuns []ProcessedRun) *MCPToolUsageSummary // Aggregate tool calls allToolCalls = append(allToolCalls, pr.MCPToolUsage.ToolCalls...) + // Aggregate DIFC filtered events + allFilteredEvents = append(allFilteredEvents, pr.MCPToolUsage.FilteredEvents...) + // Aggregate tool summaries for _, summary := range pr.MCPToolUsage.Summary { key := summary.ServerName + ":" + summary.ToolName @@ -809,7 +813,7 @@ func buildMCPToolUsageSummary(processedRuns []ProcessedRun) *MCPToolUsageSummary } // Return nil if no MCP tool usage data was found - if len(toolSummaryMap) == 0 && len(serverStatsMap) == 0 { + if len(toolSummaryMap) == 0 && len(serverStatsMap) == 0 && len(allFilteredEvents) == 0 { return nil } @@ -837,13 +841,14 @@ func buildMCPToolUsageSummary(processedRuns []ProcessedRun) *MCPToolUsageSummary return cmp.Compare(a.ServerName, b.ServerName) }) - reportLog.Printf("Built MCP tool usage summary: %d tool summaries, %d servers, %d total tool calls", - len(summaries), len(servers), len(allToolCalls)) + reportLog.Printf("Built MCP tool usage summary: %d tool summaries, %d servers, %d total tool calls, %d DIFC filtered events", + len(summaries), len(servers), len(allToolCalls), len(allFilteredEvents)) return &MCPToolUsageSummary{ - Summary: summaries, - Servers: servers, - ToolCalls: allToolCalls, + Summary: summaries, + Servers: servers, + ToolCalls: allToolCalls, + FilteredEvents: allFilteredEvents, } }