From 90c6704746d4587e53eb26de717c199043d081e9 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Wed, 4 Feb 2026 12:55:04 -0500 Subject: [PATCH 01/11] Save REST perf summary approach --- bugzooka/analysis/perf_summary_analyzer.py | 444 ++++++++++++++++++ .../integrations/slack_socket_listener.py | 102 +++- 2 files changed, 545 insertions(+), 1 deletion(-) create mode 100644 bugzooka/analysis/perf_summary_analyzer.py diff --git a/bugzooka/analysis/perf_summary_analyzer.py b/bugzooka/analysis/perf_summary_analyzer.py new file mode 100644 index 0000000..57a1044 --- /dev/null +++ b/bugzooka/analysis/perf_summary_analyzer.py @@ -0,0 +1,444 @@ +""" +Performance Summary Analyzer. +Provides performance metrics summary via simple REST API calls to orion-mcp server. + +Uses direct HTTP calls to /api/* endpoints instead of MCP protocol, +which avoids the parameter-dropping bugs in langchain-mcp-adapters. +""" +import hashlib +import logging +import os +import random +from typing import Any, List, Optional + +import httpx + +logger = logging.getLogger(__name__) + +# Orion MCP REST API base URL +ORION_API_BASE_URL = os.getenv("ORION_API_BASE_URL", "http://localhost:3030") + +# Test mode: bypass OpenSearch by returning mock data +_TEST_MODE = os.getenv("PERF_SUMMARY_TEST_MODE", "false").lower() in ( + "1", + "true", + "yes", +) +_TEST_VERSIONS = { + v.strip() + for v in os.getenv("PERF_SUMMARY_TEST_VERSIONS", "4.19,4.20").split(",") + if v.strip() +} + +_MOCK_CONFIGS = [ + "small-scale-udn-l3.yaml", + "med-scale-udn-l3.yaml", +] +_DEFAULT_MOCK_METRICS = [ + "podReadyLatency_P99", + "podReadyLatency_P50", + "ovnCPU_avg", + "ovnCPU_P99", + "kubeAPIServerThroughput", +] +_MOCK_METRICS_BY_CONFIG = { + "small-scale-udn-l3.yaml": [ + "podReadyLatency_P99", + "podReadyLatency_P50", + "ovnCPU_avg", + "kubeAPIServerThroughput", + ], + "med-scale-udn-l3.yaml": [ + "podReadyLatency_P99", + "podReadyLatency_P50", + "ovnCPU_avg", + "ovnCPU_P99", + "kubeAPIServerThroughput", + ], +} + + +def _mock_metrics_for_config(config: str) -> List[str]: + return _MOCK_METRICS_BY_CONFIG.get(config, _DEFAULT_MOCK_METRICS) + + +def _mock_meta_for_config(config: str) -> dict: + metrics = _mock_metrics_for_config(config) + + def _infer_moi(metric_name: str) -> str: + if "_" in metric_name: + suffix = metric_name.split("_")[-1] + suffix_upper = suffix.upper() + if suffix_upper.startswith("P") and suffix_upper[1:].isdigit(): + return suffix_upper + if suffix in ("avg", "max", "min", "value"): + return suffix + return "value" + + return { + metric: { + "direction": 1, + "threshold": 10.0, + "metric_of_interest": _infer_moi(metric), + } + for metric in metrics + } + + +def _stable_seed(*parts: str) -> int: + joined = "::".join(parts).encode("utf-8") + digest = hashlib.sha256(joined).hexdigest() + return int(digest[:8], 16) + + +def _mock_series(config: str, metric: str, version: str, lookback: int) -> List[float]: + seed = _stable_seed(config, metric, version) + rng = random.Random(seed) + base = rng.uniform(10.0, 100.0) + trend = rng.uniform(-0.5, 0.5) + values = [] + for i in range(lookback): + noise = rng.uniform(-1.5, 1.5) + values.append(round(base + (trend * i) + noise, 4)) + return values + + +def _calculate_stats(values: List[float]) -> dict: + if not values: + return {"min": None, "max": None, "avg": None} + return { + "min": round(min(values), 4), + "max": round(max(values), 4), + "avg": round(sum(values) / len(values), 4), + } + + +def _calculate_weekly_change(values: List[float]) -> Optional[float]: + """ + Compare last 7 values vs prior 7 values. + Returns percent change, or None if insufficient data. + """ + if len(values) < 14: + return None + week1 = values[:7] + week2 = values[7:14] + avg1 = sum(week1) / len(week1) + avg2 = sum(week2) / len(week2) + if avg1 == 0: + return None + return round(((avg2 - avg1) / avg1) * 100, 2) + + +def _weekly_hint(change: Optional[float], meta: dict) -> str: + if change is None: + return "n/a" + try: + change_val = float(change) + except (TypeError, ValueError): + return "n/a" + + direction = meta.get("direction") + threshold = meta.get("threshold") + if direction is None or threshold is None: + return f" {change_val:+.2f}" + + try: + threshold_val = float(threshold) + except (TypeError, ValueError): + threshold_val = 0.0 + + if abs(change_val) < threshold_val: + return f" {change_val:+.2f}" + + regression = (direction == 1 and change_val > 0) or ( + direction == -1 and change_val < 0 + ) + if regression: + return f"{change_val:+.2f} 🆘" + return f"{change_val:+.2f} 🟢" + + +def _format_table( + config: str, + version: str, + rows: List[dict], + total_metrics: int, + meta_map: dict, +) -> str: + headers = ["Metric", "Metric_of_interest", "Min", "Max", "Avg", "Weekly Change (%)"] + formatted_rows: List[List[str]] = [] + for row in rows: + weekly = row.get("weekly_change") + metric_name = row.get("metric", "") + meta = meta_map.get(metric_name, {}) + moi = meta.get("metric_of_interest") or meta.get("agg_type") or "value" + moi_value = row.get("avg") if row.get("avg") is not None else "n/a" + moi_display = f"{moi} = {moi_value}" + weekly_display = _weekly_hint(weekly, meta_map.get(metric_name, {})) + formatted_rows.append( + [ + str(row.get("metric", "n/a")), + str(moi_display), + str(row.get("min", "n/a")), + str(row.get("max", "n/a")), + str(row.get("avg", "n/a")), + str(weekly_display), + ] + ) + + col_widths = [len(h) for h in headers] + for formatted_row in formatted_rows: + for i, value in enumerate(formatted_row): + col_widths[i] = max(col_widths[i], len(str(value))) + + header_line = " | ".join(h.ljust(col_widths[i]) for i, h in enumerate(headers)) + sep_line = "-+-".join("-" * col_widths[i] for i in range(len(headers))) + row_lines = [ + " | ".join( + str(value).ljust(col_widths[i]) for i, value in enumerate(formatted_row) + ) + for formatted_row in formatted_rows + ] + + metrics_note = f"showing {len(rows)} of {total_metrics} metrics" + return "\n".join( + [ + f"*Config: {config}* (Version: {version}, {metrics_note})", + "```", + header_line, + sep_line, + *row_lines, + "```", + ] + ) + + +async def get_configs() -> List[str]: + """ + Get list of available Orion configuration files via REST API. + + :return: List of config file names (e.g., ['small-scale-udn-l3.yaml', ...]) + """ + if _TEST_MODE: + logger.info("Test mode enabled: returning mock configs") + return _MOCK_CONFIGS + + logger.info("Fetching available configs from orion-mcp REST API") + + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.get(f"{ORION_API_BASE_URL}/api/configs") + response.raise_for_status() + data = response.json() + return data.get("configs", []) + + +async def get_metrics( + config: str, version: Optional[str] = None +) -> tuple[List[str], dict]: + """ + Get list of available metrics for a specific config via REST API. + + :param config: Config file name (e.g., 'small-scale-udn-l3.yaml') + :return: List of metric names + """ + if _TEST_MODE: + logger.info("Test mode enabled: returning mock metrics") + return _mock_metrics_for_config(config), _mock_meta_for_config(config) + + logger.info(f"Fetching metrics for config '{config}' from orion-mcp REST API") + + async with httpx.AsyncClient(timeout=60.0) as client: + response = await client.get( + f"{ORION_API_BASE_URL}/api/metrics", + params={ + "config": config, + "include_meta": "1", + "version": version or "4.19", + }, + ) + response.raise_for_status() + data = response.json() + meta_map: dict[str, Any] = ( + data.get("meta", {}) if isinstance(data, dict) else {} + ) + metrics_data: Any = data.get("metrics", []) if isinstance(data, dict) else data + if isinstance(metrics_data, list): + return metrics_data, meta_map + + # The metrics endpoint returns {metrics: {config_path: [metric1, metric2, ...]}} + if isinstance(metrics_data, dict): + for config_path, metrics_list in metrics_data.items(): + if isinstance(metrics_list, list): + return metrics_list, meta_map + + return [], meta_map + + +async def get_performance_data( + config: str, metric: str, version: str = "4.19", lookback: int = 14 +) -> dict: + """ + Get performance data for a specific config/metric/version via REST API. + + :param config: Config file name + :param metric: Metric name + :param version: OpenShift version + :param lookback: Number of days to look back + :return: Dict with 'values' list and metadata + """ + if _TEST_MODE: + if version not in _TEST_VERSIONS: + logger.info( + "Test mode enabled: no mock data for version '%s' (allowed: %s)", + version, + sorted(_TEST_VERSIONS), + ) + return { + "config": config, + "metric": metric, + "version": version, + "lookback": str(lookback), + "values": [], + "count": 0, + "source": "mock", + "error": "no data for version", + } + logger.info("Test mode enabled: returning mock performance data") + values = _mock_series(config, metric, version, lookback) + return { + "config": config, + "metric": metric, + "version": version, + "lookback": str(lookback), + "values": values, + "count": len(values), + "source": "mock", + } + + logger.info( + f"Fetching performance data: config={config}, metric={metric}, version={version}, lookback={lookback}" + ) + + async with httpx.AsyncClient(timeout=120.0) as client: + response = await client.get( + f"{ORION_API_BASE_URL}/api/performance", + params={ + "config": config, + "metric": metric, + "version": version, + "lookback": str(lookback), + }, + ) + if response.status_code == 404: + return { + "config": config, + "metric": metric, + "version": version, + "lookback": str(lookback), + "values": [], + "count": 0, + "error": "no data found", + } + response.raise_for_status() + return response.json() + + +def _normalize_list(value: Optional[Any]) -> List[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(v) for v in value if str(v).strip()] + if isinstance(value, tuple): + return [str(v) for v in value if str(v).strip()] + text = str(value).strip() + if not text: + return [] + if "," in text: + return [part.strip() for part in text.split(",") if part.strip()] + return [text] + + +async def analyze_performance( + config: Optional[Any] = None, version: Optional[Any] = None +) -> dict: + """ + Analyze performance metrics for the specified config and version. + + :param config: Optional config file name (e.g., 'small-scale-udn-l3.yaml') + If None, analyzes all available configs. + :param version: Optional OpenShift version (e.g., '4.19') + If None, uses default version. + :return: Dict with 'success' boolean and 'message' string + """ + logger.info(f"analyze_performance called with config={config}, version={version}") + + try: + # Step 2: Get configs + config_list = _normalize_list(config) + if config_list: + configs = config_list + else: + configs = await get_configs() + + # Step 3: Get metrics for each config + result_parts: List[str] = [] + versions = _normalize_list(version) + if not versions: + versions = ["4.19"] + lookback_days = 14 + + for cfg in configs: + metrics, meta_map = await get_metrics( + cfg, versions[0] if versions else None + ) + if not metrics: + result_parts.append(f"*{cfg}*: no metrics found") + continue + + # Step 4: Fetch raw data for all metrics and all versions + metrics_to_show = metrics + for ver in versions: + rows: List[dict[str, Any]] = [] + for metric in metrics_to_show: + perf_data = await get_performance_data( + config=cfg, + metric=metric, + version=ver, + lookback=lookback_days, + ) + values = perf_data.get("values", []) + if not values: + rows.append( + { + "metric": metric, + "min": "n/a", + "max": "n/a", + "avg": "n/a", + "weekly_change": None, + } + ) + continue + + stats = _calculate_stats(values) + weekly_change = _calculate_weekly_change(values) + rows.append( + { + "metric": metric, + "min": stats["min"], + "max": stats["max"], + "avg": stats["avg"], + "weekly_change": weekly_change, + } + ) + + result_parts.append( + _format_table(cfg, ver, rows, len(metrics), meta_map) + ) + + if _TEST_MODE: + result_parts.insert(0, "_Using mock data (PERF_SUMMARY_TEST_MODE=true)_") + + return {"success": True, "message": "\n\n".join(result_parts)} + except Exception as e: + logger.error(f"Error in analyze_performance: {e}", exc_info=True) + return {"success": False, "message": f"Error: {str(e)}"} diff --git a/bugzooka/integrations/slack_socket_listener.py b/bugzooka/integrations/slack_socket_listener.py index 9a48360..3ec01b1 100644 --- a/bugzooka/integrations/slack_socket_listener.py +++ b/bugzooka/integrations/slack_socket_listener.py @@ -20,6 +20,7 @@ ) from bugzooka.analysis.pr_analyzer import analyze_pr_with_gemini from bugzooka.analysis.nightly_regression_analyzer import analyze_nightly_regression +from bugzooka.analysis.perf_summary_analyzer import analyze_performance from bugzooka.integrations.slack_client_base import SlackClientBase @@ -61,6 +62,55 @@ def __init__(self, logger: logging.Logger, max_workers: int = 5): self.processing_lock = Lock() self.processing_messages: Set[str] = set() + def _parse_perf_summary_args(self, text: str) -> tuple: + """ + Parse optional configs and versions from performance summary message. + Expected format: "@bot performance summary [config.yaml ...] [version ...]" + + :param text: Message text + :return: Tuple of (configs, versions) - both can be empty lists + """ + import re + + # Remove bot mention and normalize + # Pattern: anything before "performance summary", then optional args + text_lower = text.lower() + + # Find where "performance summary" ends + match = re.search(r"performance\s+summary\s*(.*)", text_lower) + if not match: + return [], [] + + args_text = match.group(1).strip() + if not args_text: + return [], [] + + # Split remaining text into potential args (handle commas) + parts = args_text.replace(",", " ").split() + + configs = [] + versions = [] + seen_configs = set() + seen_versions = set() + + for part in parts: + token = part.strip() + if not token: + continue + # Config files end with .yaml + if token.endswith(".yaml"): + if token not in seen_configs: + configs.append(token) + seen_configs.add(token) + # Config files end with .yaml + # Version looks like X.XX (e.g., 4.19) + elif re.match(r"^\d+\.\d+$", token): + if token not in seen_versions: + versions.append(token) + seen_versions.add(token) + + return configs, versions + def _should_process_message(self, event: Dict[str, Any]) -> bool: """ Determine if a message should be processed. @@ -234,6 +284,55 @@ def _process_mention(self, event: Dict[str, Any]) -> None: ) return + # Check if message contains "performance summary" + if "performance summary" in text.lower(): + try: + # Send initial acknowledgment + self.client.chat_postMessage( + channel=channel, + text="📊 Gathering performance summary... This may take a moment.", + thread_ts=ts, + ) + + # Parse optional configs and versions from the message + # Expected format: "performance summary [config ...] [version ...]" + configs, versions = self._parse_perf_summary_args(text) + + # Run async function in sync context + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + result = loop.run_until_complete( + analyze_performance(configs, versions) + ) + finally: + loop.close() + + # Send the result + self.client.chat_postMessage( + channel=channel, + text=result["message"], + thread_ts=ts, + ) + + if result["success"]: + self.logger.info(f"✅ Sent performance summary to {user}") + else: + self.logger.warning( + f"⚠️ Performance summary failed: {result['message']}" + ) + + except Exception as e: + self.logger.error( + f"Error processing performance summary: {e}", exc_info=True + ) + self.client.chat_postMessage( + channel=channel, + text=f"❌ Unexpected error: {str(e)}", + thread_ts=ts, + ) + return + # Default: Send simple greeting message try: self.client.chat_postMessage( @@ -241,7 +340,8 @@ def _process_mention(self, event: Dict[str, Any]) -> None: text="May the force be with you! :performance_jedi:\n\n" ":bulb: *Tips:*\n" "- `analyze pr: , compare with ` - PR performance analysis\n" - "- `inspect [vs ] [for config ] [for days]` - Nightly regression analysis", + "- `inspect [vs ] [for config ] [for days]` - Nightly regression analysis\n" + "- `performance summary [config.yaml ...] [version ...]` - Performance metrics summary", thread_ts=ts, ) self.logger.info(f"✅ Sent greeting to {user}") From 2edddb9b53d0dc5616b51a19663f2e98f40002d4 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Thu, 5 Feb 2026 16:38:25 -0500 Subject: [PATCH 02/11] add basic working version of perf summary --- bugzooka/analysis/perf_summary_analyzer.py | 440 ++++++++++-------- .../integrations/slack_socket_listener.py | 26 +- 2 files changed, 257 insertions(+), 209 deletions(-) diff --git a/bugzooka/analysis/perf_summary_analyzer.py b/bugzooka/analysis/perf_summary_analyzer.py index 57a1044..dcf71b2 100644 --- a/bugzooka/analysis/perf_summary_analyzer.py +++ b/bugzooka/analysis/perf_summary_analyzer.py @@ -1,106 +1,52 @@ """ Performance Summary Analyzer. -Provides performance metrics summary via simple REST API calls to orion-mcp server. - -Uses direct HTTP calls to /api/* endpoints instead of MCP protocol, -which avoids the parameter-dropping bugs in langchain-mcp-adapters. +Provides performance metrics summary via MCP tools exposed by orion-mcp. """ -import hashlib +import json import logging import os -import random from typing import Any, List, Optional -import httpx +from bugzooka.integrations.mcp_client import ( + get_mcp_tool, + initialize_global_resources_async, +) logger = logging.getLogger(__name__) -# Orion MCP REST API base URL -ORION_API_BASE_URL = os.getenv("ORION_API_BASE_URL", "http://localhost:3030") - -# Test mode: bypass OpenSearch by returning mock data -_TEST_MODE = os.getenv("PERF_SUMMARY_TEST_MODE", "false").lower() in ( - "1", - "true", - "yes", -) -_TEST_VERSIONS = { - v.strip() - for v in os.getenv("PERF_SUMMARY_TEST_VERSIONS", "4.19,4.20").split(",") - if v.strip() -} - -_MOCK_CONFIGS = [ - "small-scale-udn-l3.yaml", - "med-scale-udn-l3.yaml", +# Default control plane configs used when user doesn't specify a config +_DEFAULT_CONTROL_PLANE_CONFIGS = [ + "okd-control-plane-cluster-density.yaml", + "okd-control-plane-node-density.yaml", + "okd-control-plane-node-density-cni.yaml", + "okd-control-plane-crd-scale.yaml", ] -_DEFAULT_MOCK_METRICS = [ - "podReadyLatency_P99", - "podReadyLatency_P50", - "ovnCPU_avg", - "ovnCPU_P99", - "kubeAPIServerThroughput", -] -_MOCK_METRICS_BY_CONFIG = { - "small-scale-udn-l3.yaml": [ - "podReadyLatency_P99", - "podReadyLatency_P50", - "ovnCPU_avg", - "kubeAPIServerThroughput", - ], - "med-scale-udn-l3.yaml": [ - "podReadyLatency_P99", - "podReadyLatency_P50", - "ovnCPU_avg", - "ovnCPU_P99", - "kubeAPIServerThroughput", - ], -} - - -def _mock_metrics_for_config(config: str) -> List[str]: - return _MOCK_METRICS_BY_CONFIG.get(config, _DEFAULT_MOCK_METRICS) - - -def _mock_meta_for_config(config: str) -> dict: - metrics = _mock_metrics_for_config(config) - - def _infer_moi(metric_name: str) -> str: - if "_" in metric_name: - suffix = metric_name.split("_")[-1] - suffix_upper = suffix.upper() - if suffix_upper.startswith("P") and suffix_upper[1:].isdigit(): - return suffix_upper - if suffix in ("avg", "max", "min", "value"): - return suffix - return "value" - return { - metric: { - "direction": 1, - "threshold": 10.0, - "metric_of_interest": _infer_moi(metric), - } - for metric in metrics - } +# Slack message size limit (actual is ~4000, use 3500 to be safe) +_SLACK_MESSAGE_LIMIT = int(os.getenv("PERF_SUMMARY_SLACK_MSG_LIMIT", "3500")) -def _stable_seed(*parts: str) -> int: - joined = "::".join(parts).encode("utf-8") - digest = hashlib.sha256(joined).hexdigest() - return int(digest[:8], 16) +def _coerce_mcp_result(result: Any) -> Any: + if isinstance(result, tuple) and len(result) == 2: + result = result[0] + if isinstance(result, str): + try: + return json.loads(result) + except json.JSONDecodeError: + return result + return result -def _mock_series(config: str, metric: str, version: str, lookback: int) -> List[float]: - seed = _stable_seed(config, metric, version) - rng = random.Random(seed) - base = rng.uniform(10.0, 100.0) - trend = rng.uniform(-0.5, 0.5) - values = [] - for i in range(lookback): - noise = rng.uniform(-1.5, 1.5) - values.append(round(base + (trend * i) + noise, 4)) - return values +async def _call_mcp_tool(tool_name: str, args: dict[str, Any]) -> Any: + await initialize_global_resources_async() + tool = get_mcp_tool(tool_name) + if tool is None: + raise RuntimeError(f"MCP tool '{tool_name}' not found") + if hasattr(tool, "ainvoke"): + result = await tool.ainvoke(args) + else: + result = tool.invoke(args) + return _coerce_mcp_result(result) def _calculate_stats(values: List[float]) -> dict: @@ -113,20 +59,27 @@ def _calculate_stats(values: List[float]) -> dict: } -def _calculate_weekly_change(values: List[float]) -> Optional[float]: +def _calculate_weekly_change_calendar( + this_week_values: List[float], last_week_values: List[float] +) -> Optional[float]: """ - Compare last 7 values vs prior 7 values. - Returns percent change, or None if insufficient data. + Calculate weekly change using calendar-based data. + Compares average of this week (last 7 days) vs last week (prior 7 days). + + :param this_week_values: Values from the last 7 calendar days + :param last_week_values: Values from the prior 7 calendar days + :return: Percent change, or None if insufficient data """ - if len(values) < 14: + if not this_week_values or not last_week_values: return None - week1 = values[:7] - week2 = values[7:14] - avg1 = sum(week1) / len(week1) - avg2 = sum(week2) / len(week2) - if avg1 == 0: + + this_week_avg = sum(this_week_values) / len(this_week_values) + last_week_avg = sum(last_week_values) / len(last_week_values) + + if last_week_avg == 0: return None - return round(((avg2 - avg1) / avg1) * 100, 2) + + return round(((this_week_avg - last_week_avg) / last_week_avg) * 100, 2) def _weekly_hint(change: Optional[float], meta: dict) -> str: @@ -158,6 +111,28 @@ def _weekly_hint(change: Optional[float], meta: dict) -> str: return f"{change_val:+.2f} 🟢" +def _select_metric_of_interest(meta: dict) -> str: + moi = meta.get("metric_of_interest") + agg = meta.get("agg_type") + if isinstance(moi, str) and moi.lower() != "value": + return moi + if agg: + return str(agg) + if moi: + return str(moi) + return "value" + + +def _truncate_text(text: str, max_len: int) -> str: + if max_len <= 0: + return text + if len(text) <= max_len: + return text + if max_len <= 3: + return text[:max_len] + return f"{text[: max_len - 3]}..." + + def _format_table( config: str, version: str, @@ -167,18 +142,31 @@ def _format_table( ) -> str: headers = ["Metric", "Metric_of_interest", "Min", "Max", "Avg", "Weekly Change (%)"] formatted_rows: List[List[str]] = [] + max_metric_len = int(os.getenv("PERF_SUMMARY_MAX_METRIC_LEN", "40")) + max_moi_len = int(os.getenv("PERF_SUMMARY_MAX_MOI_LEN", "24")) for row in rows: weekly = row.get("weekly_change") metric_name = row.get("metric", "") meta = meta_map.get(metric_name, {}) - moi = meta.get("metric_of_interest") or meta.get("agg_type") or "value" - moi_value = row.get("avg") if row.get("avg") is not None else "n/a" + moi = _select_metric_of_interest(meta) + # Dynamically select value based on metric of interest + moi_key = moi.lower() + # Check if moi is a simple stat type that exists in row + if moi_key in ("min", "max", "avg") and row.get(moi_key) is not None: + moi_value = row.get(moi_key) + elif row.get("avg") is not None: + # For complex moi (e.g., p99Latency), show the avg of those values + moi_value = row.get("avg") + else: + moi_value = "n/a" moi_display = f"{moi} = {moi_value}" weekly_display = _weekly_hint(weekly, meta_map.get(metric_name, {})) + metric_label = _truncate_text(str(metric_name), max_metric_len) + moi_label = _truncate_text(str(moi_display), max_moi_len) formatted_rows.append( [ - str(row.get("metric", "n/a")), - str(moi_display), + metric_label, + moi_label, str(row.get("min", "n/a")), str(row.get("max", "n/a")), str(row.get("avg", "n/a")), @@ -215,70 +203,70 @@ def _format_table( async def get_configs() -> List[str]: """ - Get list of available Orion configuration files via REST API. + Get list of available Orion configuration files via MCP tool. :return: List of config file names (e.g., ['small-scale-udn-l3.yaml', ...]) """ - if _TEST_MODE: - logger.info("Test mode enabled: returning mock configs") - return _MOCK_CONFIGS - - logger.info("Fetching available configs from orion-mcp REST API") - - async with httpx.AsyncClient(timeout=30.0) as client: - response = await client.get(f"{ORION_API_BASE_URL}/api/configs") - response.raise_for_status() - data = response.json() - return data.get("configs", []) + logger.info("Fetching available configs from orion-mcp MCP tool") + result = await _call_mcp_tool("get_orion_configs", {}) + if isinstance(result, list): + return result + if isinstance(result, dict): + configs = result.get("configs") + if isinstance(configs, list): + return configs + return [] async def get_metrics( config: str, version: Optional[str] = None ) -> tuple[List[str], dict]: """ - Get list of available metrics for a specific config via REST API. + Get list of available metrics for a specific config via MCP tool. :param config: Config file name (e.g., 'small-scale-udn-l3.yaml') :return: List of metric names """ - if _TEST_MODE: - logger.info("Test mode enabled: returning mock metrics") - return _mock_metrics_for_config(config), _mock_meta_for_config(config) - - logger.info(f"Fetching metrics for config '{config}' from orion-mcp REST API") - - async with httpx.AsyncClient(timeout=60.0) as client: - response = await client.get( - f"{ORION_API_BASE_URL}/api/metrics", - params={ - "config": config, - "include_meta": "1", - "version": version or "4.19", - }, + logger.info(f"Fetching metrics for config '{config}' from orion-mcp MCP tool") + meta_map: dict[str, Any] = {} + metrics_data: Any = [] + try: + result = await _call_mcp_tool( + "get_orion_metrics_with_meta", + {"config_name": config, "version": version or "4.19"}, + ) + except Exception as e: + logger.warning( + "get_orion_metrics_with_meta unavailable, falling back to get_orion_metrics: %s", + e, ) - response.raise_for_status() - data = response.json() - meta_map: dict[str, Any] = ( - data.get("meta", {}) if isinstance(data, dict) else {} + result = await _call_mcp_tool("get_orion_metrics", {"config_name": config}) + + if isinstance(result, dict) and "metrics" in result: + meta_map = ( + result.get("meta", {}) if isinstance(result.get("meta"), dict) else {} ) - metrics_data: Any = data.get("metrics", []) if isinstance(data, dict) else data - if isinstance(metrics_data, list): - return metrics_data, meta_map + metrics_data = result.get("metrics", []) + else: + metrics_data = result + + if isinstance(metrics_data, list): + return metrics_data, meta_map - # The metrics endpoint returns {metrics: {config_path: [metric1, metric2, ...]}} - if isinstance(metrics_data, dict): - for config_path, metrics_list in metrics_data.items(): - if isinstance(metrics_list, list): - return metrics_list, meta_map + # The metrics tool may return {config_path: [metric1, metric2, ...]} + if isinstance(metrics_data, dict): + for _config_path, metrics_list in metrics_data.items(): + if isinstance(metrics_list, list): + return metrics_list, meta_map - return [], meta_map + return [], meta_map async def get_performance_data( config: str, metric: str, version: str = "4.19", lookback: int = 14 ) -> dict: """ - Get performance data for a specific config/metric/version via REST API. + Get performance data for a specific config/metric/version via MCP tool. :param config: Config file name :param metric: Metric name @@ -286,61 +274,69 @@ async def get_performance_data( :param lookback: Number of days to look back :return: Dict with 'values' list and metadata """ - if _TEST_MODE: - if version not in _TEST_VERSIONS: - logger.info( - "Test mode enabled: no mock data for version '%s' (allowed: %s)", - version, - sorted(_TEST_VERSIONS), - ) - return { - "config": config, - "metric": metric, - "version": version, - "lookback": str(lookback), - "values": [], - "count": 0, - "source": "mock", - "error": "no data for version", - } - logger.info("Test mode enabled: returning mock performance data") - values = _mock_series(config, metric, version, lookback) - return { - "config": config, - "metric": metric, - "version": version, - "lookback": str(lookback), - "values": values, - "count": len(values), - "source": "mock", - } - logger.info( - f"Fetching performance data: config={config}, metric={metric}, version={version}, lookback={lookback}" + "Fetching performance data via MCP: config=%s, metric=%s, version=%s, lookback=%s", + config, + metric, + version, + lookback, ) - async with httpx.AsyncClient(timeout=120.0) as client: - response = await client.get( - f"{ORION_API_BASE_URL}/api/performance", - params={ - "config": config, + try: + result = await _call_mcp_tool( + "get_orion_performance_data", + { + "config_name": config, "metric": metric, "version": version, "lookback": str(lookback), }, ) - if response.status_code == 404: - return { - "config": config, + except Exception as e: + logger.warning( + "get_orion_performance_data unavailable, falling back to openshift_report_on: %s", + e, + ) + result = await _call_mcp_tool( + "openshift_report_on", + { + "versions": version, "metric": metric, - "version": version, + "config_name": config, "lookback": str(lookback), - "values": [], - "count": 0, - "error": "no data found", - } - response.raise_for_status() - return response.json() + "options": "json", + }, + ) + + if isinstance(result, dict) and "values" in result: + return result + + if isinstance(result, dict) and "data" in result: + data = result.get("data", {}) + version_data = data.get(version) + if isinstance(version_data, dict): + metric_data = version_data.get(metric, {}) + values = metric_data.get("value", []) + if isinstance(values, list): + values = [v for v in values if v is not None] + return { + "config": config, + "metric": metric, + "version": version, + "lookback": str(lookback), + "values": values, + "count": len(values), + } + + return { + "config": config, + "metric": metric, + "version": version, + "lookback": str(lookback), + "values": [], + "count": 0, + "error": "no data found", + } def _normalize_list(value: Optional[Any]) -> List[str]: @@ -373,19 +369,22 @@ async def analyze_performance( logger.info(f"analyze_performance called with config={config}, version={version}") try: - # Step 2: Get configs + # Step 2: Get configs (use default control plane configs if not specified) config_list = _normalize_list(config) if config_list: configs = config_list else: - configs = await get_configs() + # Use default control plane configs as fallback + configs = _DEFAULT_CONTROL_PLANE_CONFIGS + logger.info( + f"No config specified, using default control plane configs: {configs}" + ) # Step 3: Get metrics for each config result_parts: List[str] = [] versions = _normalize_list(version) if not versions: versions = ["4.19"] - lookback_days = 14 for cfg in configs: metrics, meta_map = await get_metrics( @@ -400,14 +399,16 @@ async def analyze_performance( for ver in versions: rows: List[dict[str, Any]] = [] for metric in metrics_to_show: - perf_data = await get_performance_data( + # Fetch this week's data (last 7 days) for stats display + this_week_data = await get_performance_data( config=cfg, metric=metric, version=ver, - lookback=lookback_days, + lookback=7, ) - values = perf_data.get("values", []) - if not values: + this_week_values = this_week_data.get("values", []) + + if not this_week_values: rows.append( { "metric": metric, @@ -419,8 +420,27 @@ async def analyze_performance( ) continue - stats = _calculate_stats(values) - weekly_change = _calculate_weekly_change(values) + # Fetch last 14 days data to calculate weekly change + two_weeks_data = await get_performance_data( + config=cfg, + metric=metric, + version=ver, + lookback=14, + ) + two_weeks_values = two_weeks_data.get("values", []) + + # Derive last week's values (14 days minus this week) + # Values are ordered, so last_week = values beyond this_week count + this_week_count = len(this_week_values) + if len(two_weeks_values) > this_week_count: + last_week_values = two_weeks_values[this_week_count:] + else: + last_week_values = [] + + stats = _calculate_stats(this_week_values) + weekly_change = _calculate_weekly_change_calendar( + this_week_values, last_week_values + ) rows.append( { "metric": metric, @@ -435,10 +455,30 @@ async def analyze_performance( _format_table(cfg, ver, rows, len(metrics), meta_map) ) - if _TEST_MODE: - result_parts.insert(0, "_Using mock data (PERF_SUMMARY_TEST_MODE=true)_") - - return {"success": True, "message": "\n\n".join(result_parts)} + # Split result_parts into multiple messages to avoid Slack's size limit + messages: List[str] = [] + current_message_parts: List[str] = [] + current_length = 0 + + for part in result_parts: + part_length = len(part) + 2 # +2 for "\n\n" separator + if ( + current_length + part_length > _SLACK_MESSAGE_LIMIT + and current_message_parts + ): + # Current message would exceed limit, start a new message + messages.append("\n\n".join(current_message_parts)) + current_message_parts = [part] + current_length = len(part) + else: + current_message_parts.append(part) + current_length += part_length + + # Don't forget the last message + if current_message_parts: + messages.append("\n\n".join(current_message_parts)) + + return {"success": True, "messages": messages} except Exception as e: logger.error(f"Error in analyze_performance: {e}", exc_info=True) return {"success": False, "message": f"Error: {str(e)}"} diff --git a/bugzooka/integrations/slack_socket_listener.py b/bugzooka/integrations/slack_socket_listener.py index 3ec01b1..70831ba 100644 --- a/bugzooka/integrations/slack_socket_listener.py +++ b/bugzooka/integrations/slack_socket_listener.py @@ -308,18 +308,26 @@ def _process_mention(self, event: Dict[str, Any]) -> None: finally: loop.close() - # Send the result - self.client.chat_postMessage( - channel=channel, - text=result["message"], - thread_ts=ts, - ) - + # Send the result(s) - may be multiple messages to avoid Slack limit if result["success"]: - self.logger.info(f"✅ Sent performance summary to {user}") + messages = result.get("messages", []) + for msg in messages: + self.client.chat_postMessage( + channel=channel, + text=msg, + thread_ts=ts, + ) + self.logger.info( + f"✅ Sent performance summary to {user} ({len(messages)} message(s))" + ) else: + self.client.chat_postMessage( + channel=channel, + text=result.get("message", "Unknown error"), + thread_ts=ts, + ) self.logger.warning( - f"⚠️ Performance summary failed: {result['message']}" + f"⚠️ Performance summary failed: {result.get('message')}" ) except Exception as e: From 6a05c93f721e1f23a8fee3999cd02c50ede1a662 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Tue, 10 Feb 2026 11:53:45 -0500 Subject: [PATCH 03/11] refine presentation style of perf sum feat --- bugzooka/analysis/perf_summary_analyzer.py | 427 ++++++++++++++---- bugzooka/integrations/slack_fetcher.py | 81 ++++ .../integrations/slack_socket_listener.py | 75 +-- 3 files changed, 429 insertions(+), 154 deletions(-) diff --git a/bugzooka/analysis/perf_summary_analyzer.py b/bugzooka/analysis/perf_summary_analyzer.py index dcf71b2..c594dcd 100644 --- a/bugzooka/analysis/perf_summary_analyzer.py +++ b/bugzooka/analysis/perf_summary_analyzer.py @@ -5,6 +5,7 @@ import json import logging import os +import re from typing import Any, List, Optional from bugzooka.integrations.mcp_client import ( @@ -22,6 +23,50 @@ "okd-control-plane-crd-scale.yaml", ] +# Fallback list used when MCP config list is unavailable and ALL is requested +_ALL_CONFIGS_FALLBACK = [ + "chaos_tests.yaml", + "label-small-scale-cluster-density.yaml", + "med-scale-udn-l2.yaml", + "med-scale-udn-l3.yaml", + "metal-perfscale-cpt-data-path.yaml", + "metal-perfscale-cpt-node-density-heavy.yaml", + "metal-perfscale-cpt-node-density.yaml", + "metal-perfscale-cpt-virt-density.yaml", + "metal-perfscale-cpt-virt-udn-density.yaml", + "netobserv-diff-meta-index.yaml", + "node_scenarios.yaml", + "okd-control-plane-cluster-density.yaml", + "okd-control-plane-crd-scale.yaml", + "okd-control-plane-node-density-cni.yaml", + "okd-control-plane-node-density.yaml", + "okd-data-plane-data-path.yaml", + "olmv1.yaml", + "ols-load-generator.yaml", + "payload-scale.yaml", + "pod_disruption_scenarios.yaml", + "quay-load-test-stable-stage.yaml", + "quay-load-test-stable.yaml", + "readout-control-plane-cdv2.yaml", + "readout-control-plane-node-density.yaml", + "readout-ingress.yaml", + "readout-netperf-tcp.yaml", + "rhbok.yaml", + "servicemesh-ingress.yaml", + "servicemesh-netperf-tcp.yaml", + "small-rosa-control-plane-node-density.yaml", + "small-scale-cluster-density-report.yaml", + "small-scale-cluster-density.yaml", + "small-scale-node-density-cni.yaml", + "small-scale-udn-l2.yaml", + "small-scale-udn-l3.yaml", + "trt-external-payload-cluster-density.yaml", + "trt-external-payload-cpu.yaml", + "trt-external-payload-crd-scale.yaml", + "trt-external-payload-node-density-cni.yaml", + "trt-external-payload-node-density.yaml", +] + # Slack message size limit (actual is ~4000, use 3500 to be safe) _SLACK_MESSAGE_LIMIT = int(os.getenv("PERF_SUMMARY_SLACK_MSG_LIMIT", "3500")) @@ -59,30 +104,29 @@ def _calculate_stats(values: List[float]) -> dict: } -def _calculate_weekly_change_calendar( - this_week_values: List[float], last_week_values: List[float] +def _calculate_period_change( + current_values: List[float], previous_values: List[float] ) -> Optional[float]: """ - Calculate weekly change using calendar-based data. - Compares average of this week (last 7 days) vs last week (prior 7 days). + Calculate percent change between two periods. - :param this_week_values: Values from the last 7 calendar days - :param last_week_values: Values from the prior 7 calendar days + :param current_values: Values from the most recent period + :param previous_values: Values from the prior period :return: Percent change, or None if insufficient data """ - if not this_week_values or not last_week_values: + if not current_values or not previous_values: return None - this_week_avg = sum(this_week_values) / len(this_week_values) - last_week_avg = sum(last_week_values) / len(last_week_values) + current_avg = sum(current_values) / len(current_values) + previous_avg = sum(previous_values) / len(previous_values) - if last_week_avg == 0: + if previous_avg == 0: return None - return round(((this_week_avg - last_week_avg) / last_week_avg) * 100, 2) + return round(((current_avg - previous_avg) / previous_avg) * 100, 2) -def _weekly_hint(change: Optional[float], meta: dict) -> str: +def _change_hint(change: Optional[float], meta: dict) -> str: if change is None: return "n/a" try: @@ -111,18 +155,6 @@ def _weekly_hint(change: Optional[float], meta: dict) -> str: return f"{change_val:+.2f} 🟢" -def _select_metric_of_interest(meta: dict) -> str: - moi = meta.get("metric_of_interest") - agg = meta.get("agg_type") - if isinstance(moi, str) and moi.lower() != "value": - return moi - if agg: - return str(agg) - if moi: - return str(moi) - return "value" - - def _truncate_text(text: str, max_len: int) -> str: if max_len <= 0: return text @@ -133,44 +165,26 @@ def _truncate_text(text: str, max_len: int) -> str: return f"{text[: max_len - 3]}..." -def _format_table( +def _format_config_table( config: str, version: str, rows: List[dict], total_metrics: int, - meta_map: dict, + lookback_days: int, ) -> str: - headers = ["Metric", "Metric_of_interest", "Min", "Max", "Avg", "Weekly Change (%)"] + headers = ["Metric", "Min", "Max", "Avg", "Change (%)"] formatted_rows: List[List[str]] = [] max_metric_len = int(os.getenv("PERF_SUMMARY_MAX_METRIC_LEN", "40")) - max_moi_len = int(os.getenv("PERF_SUMMARY_MAX_MOI_LEN", "24")) for row in rows: - weekly = row.get("weekly_change") - metric_name = row.get("metric", "") - meta = meta_map.get(metric_name, {}) - moi = _select_metric_of_interest(meta) - # Dynamically select value based on metric of interest - moi_key = moi.lower() - # Check if moi is a simple stat type that exists in row - if moi_key in ("min", "max", "avg") and row.get(moi_key) is not None: - moi_value = row.get(moi_key) - elif row.get("avg") is not None: - # For complex moi (e.g., p99Latency), show the avg of those values - moi_value = row.get("avg") - else: - moi_value = "n/a" - moi_display = f"{moi} = {moi_value}" - weekly_display = _weekly_hint(weekly, meta_map.get(metric_name, {})) - metric_label = _truncate_text(str(metric_name), max_metric_len) - moi_label = _truncate_text(str(moi_display), max_moi_len) + change_display = _change_hint(row.get("change"), row.get("meta", {})) + metric_label = _truncate_text(str(row.get("metric", "")), max_metric_len) formatted_rows.append( [ metric_label, - moi_label, str(row.get("min", "n/a")), str(row.get("max", "n/a")), str(row.get("avg", "n/a")), - str(weekly_display), + str(change_display), ] ) @@ -188,7 +202,10 @@ def _format_table( for formatted_row in formatted_rows ] - metrics_note = f"showing {len(rows)} of {total_metrics} metrics" + metrics_note = ( + f"showing {len(rows)} of {total_metrics} metrics, " + f"last {lookback_days}d vs prior {lookback_days}d" + ) return "\n".join( [ f"*Config: {config}* (Version: {version}, {metrics_note})", @@ -201,6 +218,62 @@ def _format_table( ) +def _format_summary_table( + version: str, + rows: List[dict], + total_metrics: int, + lookback_days: int, + limit: int, +) -> str: + headers = ["Config", "Metric", "Min", "Max", "Avg", "Change (%)"] + formatted_rows: List[List[str]] = [] + max_metric_len = int(os.getenv("PERF_SUMMARY_MAX_METRIC_LEN", "40")) + max_config_len = int(os.getenv("PERF_SUMMARY_MAX_CONFIG_LEN", "25")) + for row in rows: + change_display = _change_hint(row.get("change"), row.get("meta", {})) + config_label = _truncate_text(str(row.get("config", "")), max_config_len) + metric_label = _truncate_text(str(row.get("metric", "")), max_metric_len) + formatted_rows.append( + [ + config_label, + metric_label, + str(row.get("min", "n/a")), + str(row.get("max", "n/a")), + str(row.get("avg", "n/a")), + str(change_display), + ] + ) + + col_widths = [len(h) for h in headers] + for formatted_row in formatted_rows: + for i, value in enumerate(formatted_row): + col_widths[i] = max(col_widths[i], len(str(value))) + + header_line = " | ".join(h.ljust(col_widths[i]) for i, h in enumerate(headers)) + sep_line = "-+-".join("-" * col_widths[i] for i in range(len(headers))) + row_lines = [ + " | ".join( + str(value).ljust(col_widths[i]) for i, value in enumerate(formatted_row) + ) + for formatted_row in formatted_rows + ] + + metrics_note = ( + f"top {len(rows)} of {total_metrics} metrics, " + f"last {lookback_days}d vs prior {lookback_days}d" + ) + return "\n".join( + [ + f"*Top {limit} Changes* (Version: {version}, {metrics_note})", + "```", + header_line, + sep_line, + *row_lines, + "```", + ] + ) + + async def get_configs() -> List[str]: """ Get list of available Orion configuration files via MCP tool. @@ -354,8 +427,91 @@ def _normalize_list(value: Optional[Any]) -> List[str]: return [text] +def parse_perf_summary_args( + text: str, +) -> tuple[List[str], List[str], Optional[int], bool, bool]: + """ + Parse configs, versions, lookback days, and verbose flag from a message. + Expected format: "performance summary [config.yaml,...] [version ...] [verbose]" + """ + text_lower = text.lower() + match = re.search(r"performance\s+summary\s*(.*)", text_lower) + if not match: + return [], [], None, False, False + + args_text = match.group(1).strip() + if not args_text: + return [], [], None, False, False + + parts = args_text.split() + configs: List[str] = [] + versions: List[str] = [] + seen_configs: set[str] = set() + seen_versions: set[str] = set() + lookback_days: Optional[int] = None + verbose = False + all_configs = False + comma_present = "," in args_text + + def _add_config(value: str) -> None: + if value.endswith(".yaml") and value not in seen_configs: + configs.append(value) + seen_configs.add(value) + + def _add_version(value: str) -> None: + if re.match(r"^\d+\.\d+$", value) and value not in seen_versions: + versions.append(value) + seen_versions.add(value) + + for part in parts: + token = part.strip() + if not token: + continue + + token_lower = token.lower().strip(",") + if token_lower == "verbose": + verbose = True + continue + if token_lower in {"all", "all-configs", "allconfigs"}: + all_configs = True + continue + + if re.match(r"^\d+d$", token_lower): + lookback_days = int(token_lower[:-1]) + continue + + if token_lower.isdigit(): + lookback_days = int(token_lower) + continue + + if ".yaml" in token_lower: + for cfg in token.split(","): + cfg = cfg.strip() + if cfg: + _add_config(cfg) + continue + + if "," in token: + for ver in token.split(","): + ver = ver.strip() + if ver: + _add_version(ver) + continue + + _add_version(token_lower) + + if not comma_present and len(configs) > 1: + configs = configs[:1] + + return configs, versions, lookback_days, verbose, all_configs + + async def analyze_performance( - config: Optional[Any] = None, version: Optional[Any] = None + config: Optional[Any] = None, + version: Optional[Any] = None, + lookback_days: Optional[int] = None, + verbose: bool = False, + use_all_configs: bool = False, ) -> dict: """ Analyze performance metrics for the specified config and version. @@ -364,96 +520,169 @@ async def analyze_performance( If None, analyzes all available configs. :param version: Optional OpenShift version (e.g., '4.19') If None, uses default version. + :param lookback_days: Lookback period in days for stats and change calculation + :param verbose: If True, show per-config tables; otherwise show top changes summary :return: Dict with 'success' boolean and 'message' string """ - logger.info(f"analyze_performance called with config={config}, version={version}") + logger.info( + "analyze_performance called with config=%s, version=%s, lookback_days=%s, verbose=%s, use_all_configs=%s", + config, + version, + lookback_days, + verbose, + use_all_configs, + ) try: - # Step 2: Get configs (use default control plane configs if not specified) + # Step 2: Get configs (default to control plane configs unless ALL requested) config_list = _normalize_list(config) - if config_list: + if use_all_configs: + configs = await get_configs() + if configs: + logger.info("Using ALL configs from MCP: %s", configs) + else: + configs = _ALL_CONFIGS_FALLBACK + logger.info( + "No configs returned from MCP, using fallback ALL list: %s", + configs, + ) + elif config_list: configs = config_list else: - # Use default control plane configs as fallback configs = _DEFAULT_CONTROL_PLANE_CONFIGS logger.info( - f"No config specified, using default control plane configs: {configs}" + "No config specified, using default control plane configs: %s", + configs, ) - # Step 3: Get metrics for each config + # Step 3: Normalize lookback days + if lookback_days is None: + lookback_days = int(os.getenv("PERF_SUMMARY_LOOKBACK_DAYS", "14")) + if lookback_days <= 0: + lookback_days = 14 + lookback_period = lookback_days + lookback_window = lookback_days * 2 + + # Step 4: Get metrics for each config result_parts: List[str] = [] versions = _normalize_list(version) if not versions: versions = ["4.19"] + missing_configs: List[str] = [] + aggregated_rows: dict[str, List[dict[str, Any]]] = {ver: [] for ver in versions} + total_metrics = 0 for cfg in configs: metrics, meta_map = await get_metrics( cfg, versions[0] if versions else None ) if not metrics: - result_parts.append(f"*{cfg}*: no metrics found") + missing_configs.append(cfg) continue - # Step 4: Fetch raw data for all metrics and all versions - metrics_to_show = metrics + total_metrics += len(metrics) + + # Fetch raw data for all metrics and all versions for ver in versions: rows: List[dict[str, Any]] = [] - for metric in metrics_to_show: - # Fetch this week's data (last 7 days) for stats display - this_week_data = await get_performance_data( + for metric in metrics: + this_period_data = await get_performance_data( config=cfg, metric=metric, version=ver, - lookback=7, + lookback=lookback_period, ) - this_week_values = this_week_data.get("values", []) - - if not this_week_values: - rows.append( - { - "metric": metric, - "min": "n/a", - "max": "n/a", - "avg": "n/a", - "weekly_change": None, - } - ) + this_period_values = this_period_data.get("values", []) + + if not this_period_values: + row = { + "config": cfg, + "metric": metric, + "min": "n/a", + "max": "n/a", + "avg": "n/a", + "change": None, + "meta": meta_map.get(metric, {}), + } + rows.append(row) + aggregated_rows[ver].append(row) continue - # Fetch last 14 days data to calculate weekly change - two_weeks_data = await get_performance_data( + two_period_data = await get_performance_data( config=cfg, metric=metric, version=ver, - lookback=14, + lookback=lookback_window, ) - two_weeks_values = two_weeks_data.get("values", []) + two_period_values = two_period_data.get("values", []) - # Derive last week's values (14 days minus this week) - # Values are ordered, so last_week = values beyond this_week count - this_week_count = len(this_week_values) - if len(two_weeks_values) > this_week_count: - last_week_values = two_weeks_values[this_week_count:] + this_period_count = len(this_period_values) + if len(two_period_values) > this_period_count: + previous_period_values = two_period_values[this_period_count:] else: - last_week_values = [] + previous_period_values = [] - stats = _calculate_stats(this_week_values) - weekly_change = _calculate_weekly_change_calendar( - this_week_values, last_week_values + stats = _calculate_stats(this_period_values) + change = _calculate_period_change( + this_period_values, previous_period_values ) - rows.append( - { - "metric": metric, - "min": stats["min"], - "max": stats["max"], - "avg": stats["avg"], - "weekly_change": weekly_change, - } + row = { + "config": cfg, + "metric": metric, + "min": stats["min"], + "max": stats["max"], + "avg": stats["avg"], + "change": change, + "meta": meta_map.get(metric, {}), + } + rows.append(row) + aggregated_rows[ver].append(row) + + if verbose: + result_parts.append( + _format_config_table( + cfg, ver, rows, len(metrics), lookback_days + ) ) - result_parts.append( - _format_table(cfg, ver, rows, len(metrics), meta_map) - ) + if not verbose: + limit = 15 + + def _change_sort_key(row: dict[str, Any]) -> float: + change_val = row.get("change") + if isinstance(change_val, (int, float)): + return abs(change_val) + return -1 + + def _include_in_summary(row: dict[str, Any]) -> bool: + keys = ("change", "min", "max", "avg") + for key in keys: + val = row.get(key, "n/a") + if isinstance(val, (int, float)): + return True + if isinstance(val, str) and val.lower() != "n/a": + return True + return False + + for ver in versions: + rows = [ + row + for row in aggregated_rows.get(ver, []) + if _include_in_summary(row) + ] + sorted_rows = sorted(rows, key=_change_sort_key, reverse=True) + top_rows = sorted_rows[:limit] + if top_rows: + result_parts.append( + _format_summary_table( + ver, top_rows, total_metrics, lookback_days, limit + ) + ) + else: + result_parts.append(f"*Version {ver}*: no performance data found") + + if missing_configs: + result_parts.append(f"*No metrics found for:* {', '.join(missing_configs)}") # Split result_parts into multiple messages to avoid Slack's size limit messages: List[str] = [] diff --git a/bugzooka/integrations/slack_fetcher.py b/bugzooka/integrations/slack_fetcher.py index e4b4bd1..1206542 100644 --- a/bugzooka/integrations/slack_fetcher.py +++ b/bugzooka/integrations/slack_fetcher.py @@ -1,3 +1,4 @@ +import asyncio import io import time import re @@ -18,6 +19,10 @@ filter_errors_with_llm, run_agent_analysis, ) +from bugzooka.analysis.perf_summary_analyzer import ( + analyze_performance, + parse_perf_summary_args, +) from bugzooka.analysis.log_summarizer import ( classify_failure_type, build_summary_sections, @@ -441,6 +446,82 @@ def _process_message(self, msg, enable_inference): # No weekly trigger; dynamic summarize only + # Performance summary trigger (polling mode) + if "performance summary" in text_lower: + try: + self.client.chat_postMessage( + channel=self.channel_id, + text="📊 Gathering performance summary... This may take a moment.", + thread_ts=ts, + ) + + ( + configs, + versions, + lookback_days, + verbose, + use_all_configs, + ) = parse_perf_summary_args(text) + self.logger.info( + "Triggering performance summary via polling: lookback_days=%s, versions=%s, configs=%s, verbose=%s, use_all_configs=%s", + lookback_days, + versions, + configs, + verbose, + use_all_configs, + ) + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + result = loop.run_until_complete( + analyze_performance( + configs, + versions, + lookback_days=lookback_days, + verbose=verbose, + use_all_configs=use_all_configs, + ) + ) + finally: + loop.close() + + if result.get("success"): + messages = result.get("messages", []) + for msg in messages: + self.client.chat_postMessage( + channel=self.channel_id, + text=msg, + thread_ts=ts, + ) + self.logger.info( + "✅ Sent performance summary via polling (%s message(s))", + len(messages), + ) + else: + self.client.chat_postMessage( + channel=self.channel_id, + text=result.get("message", "Unknown error"), + thread_ts=ts, + ) + self.logger.warning( + "⚠️ Performance summary failed: %s", + result.get("message"), + ) + + except Exception as e: + self.logger.error( + "Error processing performance summary in polling mode: %s", + e, + exc_info=True, + ) + self.client.chat_postMessage( + channel=self.channel_id, + text=f"❌ Unexpected error: {str(e)}", + thread_ts=ts, + ) + return ts + if "failure" not in text_lower: self.logger.info("Not a failure job, skipping") return ts diff --git a/bugzooka/integrations/slack_socket_listener.py b/bugzooka/integrations/slack_socket_listener.py index 70831ba..54e8e7e 100644 --- a/bugzooka/integrations/slack_socket_listener.py +++ b/bugzooka/integrations/slack_socket_listener.py @@ -20,7 +20,10 @@ ) from bugzooka.analysis.pr_analyzer import analyze_pr_with_gemini from bugzooka.analysis.nightly_regression_analyzer import analyze_nightly_regression -from bugzooka.analysis.perf_summary_analyzer import analyze_performance +from bugzooka.analysis.perf_summary_analyzer import ( + analyze_performance, + parse_perf_summary_args, +) from bugzooka.integrations.slack_client_base import SlackClientBase @@ -62,55 +65,6 @@ def __init__(self, logger: logging.Logger, max_workers: int = 5): self.processing_lock = Lock() self.processing_messages: Set[str] = set() - def _parse_perf_summary_args(self, text: str) -> tuple: - """ - Parse optional configs and versions from performance summary message. - Expected format: "@bot performance summary [config.yaml ...] [version ...]" - - :param text: Message text - :return: Tuple of (configs, versions) - both can be empty lists - """ - import re - - # Remove bot mention and normalize - # Pattern: anything before "performance summary", then optional args - text_lower = text.lower() - - # Find where "performance summary" ends - match = re.search(r"performance\s+summary\s*(.*)", text_lower) - if not match: - return [], [] - - args_text = match.group(1).strip() - if not args_text: - return [], [] - - # Split remaining text into potential args (handle commas) - parts = args_text.replace(",", " ").split() - - configs = [] - versions = [] - seen_configs = set() - seen_versions = set() - - for part in parts: - token = part.strip() - if not token: - continue - # Config files end with .yaml - if token.endswith(".yaml"): - if token not in seen_configs: - configs.append(token) - seen_configs.add(token) - # Config files end with .yaml - # Version looks like X.XX (e.g., 4.19) - elif re.match(r"^\d+\.\d+$", token): - if token not in seen_versions: - versions.append(token) - seen_versions.add(token) - - return configs, versions - def _should_process_message(self, event: Dict[str, Any]) -> bool: """ Determine if a message should be processed. @@ -294,16 +248,27 @@ def _process_mention(self, event: Dict[str, Any]) -> None: thread_ts=ts, ) - # Parse optional configs and versions from the message - # Expected format: "performance summary [config ...] [version ...]" - configs, versions = self._parse_perf_summary_args(text) + # Parse configs, versions, lookback days, and verbose flag + ( + configs, + versions, + lookback_days, + verbose, + use_all_configs, + ) = parse_perf_summary_args(text) # Run async function in sync context loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: result = loop.run_until_complete( - analyze_performance(configs, versions) + analyze_performance( + configs, + versions, + lookback_days=lookback_days, + verbose=verbose, + use_all_configs=use_all_configs, + ) ) finally: loop.close() @@ -349,7 +314,7 @@ def _process_mention(self, event: Dict[str, Any]) -> None: ":bulb: *Tips:*\n" "- `analyze pr: , compare with ` - PR performance analysis\n" "- `inspect [vs ] [for config ] [for days]` - Nightly regression analysis\n" - "- `performance summary [config.yaml ...] [version ...]` - Performance metrics summary", + "- `performance summary [ALL|config1.yaml,config2.yaml] [version ...] [verbose]` - Performance metrics summary", thread_ts=ts, ) self.logger.info(f"✅ Sent greeting to {user}") From f5418a5e9b0d6c051c498755bce8b20305abdd34 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Thu, 12 Feb 2026 17:37:53 -0500 Subject: [PATCH 04/11] minor fix --- bugzooka/analysis/perf_summary_analyzer.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bugzooka/analysis/perf_summary_analyzer.py b/bugzooka/analysis/perf_summary_analyzer.py index c594dcd..45ee59a 100644 --- a/bugzooka/analysis/perf_summary_analyzer.py +++ b/bugzooka/analysis/perf_summary_analyzer.py @@ -17,10 +17,13 @@ # Default control plane configs used when user doesn't specify a config _DEFAULT_CONTROL_PLANE_CONFIGS = [ - "okd-control-plane-cluster-density.yaml", - "okd-control-plane-node-density.yaml", - "okd-control-plane-node-density-cni.yaml", - "okd-control-plane-crd-scale.yaml", + "metal-perfscale-cpt-virt-udn-density.yaml", + "trt-external-payload-cluster-density.yaml", + "trt-external-payload-node-density.yaml", + "trt-external-payload-node-density-cni.yaml", + "trt-external-payload-crd-scale.yaml", + "small-scale-udn-l3.yaml", + "med-scale-udn-l3.yaml", ] # Fallback list used when MCP config list is unavailable and ALL is requested @@ -104,7 +107,7 @@ def _calculate_stats(values: List[float]) -> dict: } -def _calculate_period_change( +def _calculate_percentage_change( current_values: List[float], previous_values: List[float] ) -> Optional[float]: """ @@ -623,7 +626,7 @@ async def analyze_performance( previous_period_values = [] stats = _calculate_stats(this_period_values) - change = _calculate_period_change( + change = _calculate_percentage_change( this_period_values, previous_period_values ) row = { From a0b4f6f27c69efb7c2c2b032dc483e057123099c Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Mon, 16 Feb 2026 18:01:18 -0500 Subject: [PATCH 05/11] resovle duplicated trigerring issue --- bugzooka/entrypoint.py | 5 ++++- bugzooka/integrations/slack_fetcher.py | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bugzooka/entrypoint.py b/bugzooka/entrypoint.py index 86e86dc..a4c5945 100644 --- a/bugzooka/entrypoint.py +++ b/bugzooka/entrypoint.py @@ -52,7 +52,10 @@ def main() -> None: } fetcher = SlackMessageFetcher( - channel_id=SLACK_CHANNEL_ID, logger=logger, poll_interval=SLACK_POLL_INTERVAL + channel_id=SLACK_CHANNEL_ID, + logger=logger, + poll_interval=SLACK_POLL_INTERVAL, + enable_socket_mode=args.enable_socket_mode, ) listener = None diff --git a/bugzooka/integrations/slack_fetcher.py b/bugzooka/integrations/slack_fetcher.py index 1206542..59b0537 100644 --- a/bugzooka/integrations/slack_fetcher.py +++ b/bugzooka/integrations/slack_fetcher.py @@ -47,13 +47,14 @@ class SlackMessageFetcher(SlackClientBase): """Continuously fetches new messages from a Slack channel and logs them.""" - def __init__(self, channel_id, logger, poll_interval=600): + def __init__(self, channel_id, logger, poll_interval=600, enable_socket_mode=False): """Initialize Slack client and channel details.""" # Initialize base class (handles WebClient, logger, channel_id, running flag, signal handler) super().__init__(logger, channel_id) self.poll_interval = poll_interval # How often to fetch messages self.last_seen_timestamp = None # Track the latest message timestamp + self.enable_socket_mode = enable_socket_mode def _sanitize_job_text(self, text: str) -> str: """ @@ -448,6 +449,11 @@ def _process_message(self, msg, enable_inference): # Performance summary trigger (polling mode) if "performance summary" in text_lower: + if self.enable_socket_mode: + self.logger.info( + "Socket mode enabled; skipping performance summary in polling" + ) + return ts try: self.client.chat_postMessage( channel=self.channel_id, From 1af44f239528ec100a8768dc4d8cdc1cd4a0b2ff Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Mon, 16 Feb 2026 20:06:59 -0500 Subject: [PATCH 06/11] refine perf summary formatting and defaults --- bugzooka/analysis/perf_summary_analyzer.py | 130 +++++++++------------ 1 file changed, 54 insertions(+), 76 deletions(-) diff --git a/bugzooka/analysis/perf_summary_analyzer.py b/bugzooka/analysis/perf_summary_analyzer.py index 45ee59a..959f871 100644 --- a/bugzooka/analysis/perf_summary_analyzer.py +++ b/bugzooka/analysis/perf_summary_analyzer.py @@ -70,6 +70,10 @@ "trt-external-payload-node-density.yaml", ] +# Default OpenShift versions used when none provided +_DEFAULT_VERSION = "4.19" +_DEFAULT_VERSIONS = [_DEFAULT_VERSION] + # Slack message size limit (actual is ~4000, use 3500 to be safe) _SLACK_MESSAGE_LIMIT = int(os.getenv("PERF_SUMMARY_SLACK_MSG_LIMIT", "3500")) @@ -132,10 +136,7 @@ def _calculate_percentage_change( def _change_hint(change: Optional[float], meta: dict) -> str: if change is None: return "n/a" - try: - change_val = float(change) - except (TypeError, ValueError): - return "n/a" + change_val = change direction = meta.get("direction") threshold = meta.get("threshold") @@ -168,6 +169,23 @@ def _truncate_text(text: str, max_len: int) -> str: return f"{text[: max_len - 3]}..." +def _render_table(headers: List[str], formatted_rows: List[List[str]]) -> str: + col_widths = [len(h) for h in headers] + for formatted_row in formatted_rows: + for i, value in enumerate(formatted_row): + col_widths[i] = max(col_widths[i], len(str(value))) + + header_line = " | ".join(h.ljust(col_widths[i]) for i, h in enumerate(headers)) + sep_line = "-+-".join("-" * col_widths[i] for i in range(len(headers))) + row_lines = [ + " | ".join( + str(value).ljust(col_widths[i]) for i, value in enumerate(formatted_row) + ) + for formatted_row in formatted_rows + ] + return "\n".join([header_line, sep_line, *row_lines]) + + def _format_config_table( config: str, version: str, @@ -191,20 +209,7 @@ def _format_config_table( ] ) - col_widths = [len(h) for h in headers] - for formatted_row in formatted_rows: - for i, value in enumerate(formatted_row): - col_widths[i] = max(col_widths[i], len(str(value))) - - header_line = " | ".join(h.ljust(col_widths[i]) for i, h in enumerate(headers)) - sep_line = "-+-".join("-" * col_widths[i] for i in range(len(headers))) - row_lines = [ - " | ".join( - str(value).ljust(col_widths[i]) for i, value in enumerate(formatted_row) - ) - for formatted_row in formatted_rows - ] - + table_body = _render_table(headers, formatted_rows) metrics_note = ( f"showing {len(rows)} of {total_metrics} metrics, " f"last {lookback_days}d vs prior {lookback_days}d" @@ -213,9 +218,7 @@ def _format_config_table( [ f"*Config: {config}* (Version: {version}, {metrics_note})", "```", - header_line, - sep_line, - *row_lines, + table_body, "```", ] ) @@ -247,20 +250,7 @@ def _format_summary_table( ] ) - col_widths = [len(h) for h in headers] - for formatted_row in formatted_rows: - for i, value in enumerate(formatted_row): - col_widths[i] = max(col_widths[i], len(str(value))) - - header_line = " | ".join(h.ljust(col_widths[i]) for i, h in enumerate(headers)) - sep_line = "-+-".join("-" * col_widths[i] for i in range(len(headers))) - row_lines = [ - " | ".join( - str(value).ljust(col_widths[i]) for i, value in enumerate(formatted_row) - ) - for formatted_row in formatted_rows - ] - + table_body = _render_table(headers, formatted_rows) metrics_note = ( f"top {len(rows)} of {total_metrics} metrics, " f"last {lookback_days}d vs prior {lookback_days}d" @@ -269,9 +259,7 @@ def _format_summary_table( [ f"*Top {limit} Changes* (Version: {version}, {metrics_note})", "```", - header_line, - sep_line, - *row_lines, + table_body, "```", ] ) @@ -306,17 +294,21 @@ async def get_metrics( logger.info(f"Fetching metrics for config '{config}' from orion-mcp MCP tool") meta_map: dict[str, Any] = {} metrics_data: Any = [] + effective_version = version or _DEFAULT_VERSION try: result = await _call_mcp_tool( "get_orion_metrics_with_meta", - {"config_name": config, "version": version or "4.19"}, + {"config_name": config, "version": effective_version}, ) except Exception as e: logger.warning( "get_orion_metrics_with_meta unavailable, falling back to get_orion_metrics: %s", e, ) - result = await _call_mcp_tool("get_orion_metrics", {"config_name": config}) + result = await _call_mcp_tool( + "get_orion_metrics", + {"config_name": config, "version": effective_version}, + ) if isinstance(result, dict) and "metrics" in result: meta_map = ( @@ -339,7 +331,7 @@ async def get_metrics( async def get_performance_data( - config: str, metric: str, version: str = "4.19", lookback: int = 14 + config: str, metric: str, version: str = _DEFAULT_VERSION, lookback: int = 14 ) -> dict: """ Get performance data for a specific config/metric/version via MCP tool. @@ -415,21 +407,6 @@ async def get_performance_data( } -def _normalize_list(value: Optional[Any]) -> List[str]: - if value is None: - return [] - if isinstance(value, list): - return [str(v) for v in value if str(v).strip()] - if isinstance(value, tuple): - return [str(v) for v in value if str(v).strip()] - text = str(value).strip() - if not text: - return [] - if "," in text: - return [part.strip() for part in text.split(",") if part.strip()] - return [text] - - def parse_perf_summary_args( text: str, ) -> tuple[List[str], List[str], Optional[int], bool, bool]: @@ -510,8 +487,8 @@ def _add_version(value: str) -> None: async def analyze_performance( - config: Optional[Any] = None, - version: Optional[Any] = None, + configs: Optional[List[str]] = None, + versions: Optional[List[str]] = None, lookback_days: Optional[int] = None, verbose: bool = False, use_all_configs: bool = False, @@ -519,18 +496,20 @@ async def analyze_performance( """ Analyze performance metrics for the specified config and version. - :param config: Optional config file name (e.g., 'small-scale-udn-l3.yaml') - If None, analyzes all available configs. - :param version: Optional OpenShift version (e.g., '4.19') - If None, uses default version. + :param configs: Optional list of config file names + (e.g., ['small-scale-udn-l3.yaml']). + If None or empty, uses defaults (or ALL if requested). + :param versions: Optional list of OpenShift versions + (defaults to [_DEFAULT_VERSION] when empty). + If None or empty, uses default version. :param lookback_days: Lookback period in days for stats and change calculation :param verbose: If True, show per-config tables; otherwise show top changes summary :return: Dict with 'success' boolean and 'message' string """ logger.info( - "analyze_performance called with config=%s, version=%s, lookback_days=%s, verbose=%s, use_all_configs=%s", - config, - version, + "analyze_performance called with configs=%s, versions=%s, lookback_days=%s, verbose=%s, use_all_configs=%s", + configs, + versions, lookback_days, verbose, use_all_configs, @@ -538,7 +517,7 @@ async def analyze_performance( try: # Step 2: Get configs (default to control plane configs unless ALL requested) - config_list = _normalize_list(config) + requested_configs = list(configs or []) if use_all_configs: configs = await get_configs() if configs: @@ -549,8 +528,8 @@ async def analyze_performance( "No configs returned from MCP, using fallback ALL list: %s", configs, ) - elif config_list: - configs = config_list + elif requested_configs: + configs = requested_configs else: configs = _DEFAULT_CONTROL_PLANE_CONFIGS logger.info( @@ -563,22 +542,21 @@ async def analyze_performance( lookback_days = int(os.getenv("PERF_SUMMARY_LOOKBACK_DAYS", "14")) if lookback_days <= 0: lookback_days = 14 - lookback_period = lookback_days lookback_window = lookback_days * 2 # Step 4: Get metrics for each config result_parts: List[str] = [] - versions = _normalize_list(version) - if not versions: - versions = ["4.19"] + requested_versions = list(versions or []) + if not requested_versions: + versions = list(_DEFAULT_VERSIONS) + else: + versions = requested_versions missing_configs: List[str] = [] aggregated_rows: dict[str, List[dict[str, Any]]] = {ver: [] for ver in versions} total_metrics = 0 for cfg in configs: - metrics, meta_map = await get_metrics( - cfg, versions[0] if versions else None - ) + metrics, meta_map = await get_metrics(cfg, versions[0]) if not metrics: missing_configs.append(cfg) continue @@ -593,7 +571,7 @@ async def analyze_performance( config=cfg, metric=metric, version=ver, - lookback=lookback_period, + lookback=lookback_days, ) this_period_values = this_period_data.get("values", []) From b88f9f9ef19c635085b3d42888abf08b476721b8 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Mon, 23 Feb 2026 17:33:16 -0500 Subject: [PATCH 07/11] Add PerformanceData dataclass and refactor metrics table formatting - Introduced a new `PerformanceData` dataclass to encapsulate performance metrics. - Refactored `_format_metrics_table` and `_format_config_table` functions for improved clarity and functionality. - Updated `get_performance_data` to return `PerformanceData` instances instead of dictionaries. - Enhanced table formatting to conditionally include configuration details based on parameters. --- bugzooka/analysis/perf_summary_analyzer.py | 166 +++++++++++++-------- 1 file changed, 100 insertions(+), 66 deletions(-) diff --git a/bugzooka/analysis/perf_summary_analyzer.py b/bugzooka/analysis/perf_summary_analyzer.py index 959f871..682a254 100644 --- a/bugzooka/analysis/perf_summary_analyzer.py +++ b/bugzooka/analysis/perf_summary_analyzer.py @@ -6,6 +6,7 @@ import logging import os import re +from dataclasses import dataclass from typing import Any, List, Optional from bugzooka.integrations.mcp_client import ( @@ -78,6 +79,20 @@ _SLACK_MESSAGE_LIMIT = int(os.getenv("PERF_SUMMARY_SLACK_MSG_LIMIT", "3500")) +@dataclass(frozen=True) +class PerformanceData: + config: str + metric: str + version: str + lookback: int + values: List[float] + error: Optional[str] = None + + @property + def count(self) -> int: + return len(self.values) + + def _coerce_mcp_result(result: Any) -> Any: if isinstance(result, tuple) and len(result) == 2: result = result[0] @@ -186,37 +201,46 @@ def _render_table(headers: List[str], formatted_rows: List[List[str]]) -> str: return "\n".join([header_line, sep_line, *row_lines]) -def _format_config_table( - config: str, +def _format_metrics_table( + *, + title: str, version: str, rows: List[dict], total_metrics: int, lookback_days: int, + include_config: bool, + note_prefix: str, ) -> str: headers = ["Metric", "Min", "Max", "Avg", "Change (%)"] + if include_config: + headers = ["Config", *headers] + formatted_rows: List[List[str]] = [] max_metric_len = int(os.getenv("PERF_SUMMARY_MAX_METRIC_LEN", "40")) + max_config_len = int(os.getenv("PERF_SUMMARY_MAX_CONFIG_LEN", "25")) for row in rows: change_display = _change_hint(row.get("change"), row.get("meta", {})) metric_label = _truncate_text(str(row.get("metric", "")), max_metric_len) - formatted_rows.append( - [ - metric_label, - str(row.get("min", "n/a")), - str(row.get("max", "n/a")), - str(row.get("avg", "n/a")), - str(change_display), - ] - ) + row_values = [ + metric_label, + str(row.get("min", "n/a")), + str(row.get("max", "n/a")), + str(row.get("avg", "n/a")), + str(change_display), + ] + if include_config: + config_label = _truncate_text(str(row.get("config", "")), max_config_len) + row_values = [config_label, *row_values] + formatted_rows.append(row_values) table_body = _render_table(headers, formatted_rows) metrics_note = ( - f"showing {len(rows)} of {total_metrics} metrics, " + f"{note_prefix} {len(rows)} of {total_metrics} metrics, " f"last {lookback_days}d vs prior {lookback_days}d" ) return "\n".join( [ - f"*Config: {config}* (Version: {version}, {metrics_note})", + f"{title} (Version: {version}, {metrics_note})", "```", table_body, "```", @@ -224,6 +248,24 @@ def _format_config_table( ) +def _format_config_table( + config: str, + version: str, + rows: List[dict], + total_metrics: int, + lookback_days: int, +) -> str: + return _format_metrics_table( + title=f"*Config: {config}*", + version=version, + rows=rows, + total_metrics=total_metrics, + lookback_days=lookback_days, + include_config=False, + note_prefix="showing", + ) + + def _format_summary_table( version: str, rows: List[dict], @@ -231,37 +273,14 @@ def _format_summary_table( lookback_days: int, limit: int, ) -> str: - headers = ["Config", "Metric", "Min", "Max", "Avg", "Change (%)"] - formatted_rows: List[List[str]] = [] - max_metric_len = int(os.getenv("PERF_SUMMARY_MAX_METRIC_LEN", "40")) - max_config_len = int(os.getenv("PERF_SUMMARY_MAX_CONFIG_LEN", "25")) - for row in rows: - change_display = _change_hint(row.get("change"), row.get("meta", {})) - config_label = _truncate_text(str(row.get("config", "")), max_config_len) - metric_label = _truncate_text(str(row.get("metric", "")), max_metric_len) - formatted_rows.append( - [ - config_label, - metric_label, - str(row.get("min", "n/a")), - str(row.get("max", "n/a")), - str(row.get("avg", "n/a")), - str(change_display), - ] - ) - - table_body = _render_table(headers, formatted_rows) - metrics_note = ( - f"top {len(rows)} of {total_metrics} metrics, " - f"last {lookback_days}d vs prior {lookback_days}d" - ) - return "\n".join( - [ - f"*Top {limit} Changes* (Version: {version}, {metrics_note})", - "```", - table_body, - "```", - ] + return _format_metrics_table( + title=f"*Top {limit} Changes*", + version=version, + rows=rows, + total_metrics=total_metrics, + lookback_days=lookback_days, + include_config=True, + note_prefix="top", ) @@ -332,7 +351,7 @@ async def get_metrics( async def get_performance_data( config: str, metric: str, version: str = _DEFAULT_VERSION, lookback: int = 14 -) -> dict: +) -> PerformanceData: """ Get performance data for a specific config/metric/version via MCP tool. @@ -340,7 +359,7 @@ async def get_performance_data( :param metric: Metric name :param version: OpenShift version :param lookback: Number of days to look back - :return: Dict with 'values' list and metadata + :return: PerformanceData with values and optional error """ logger.info( "Fetching performance data via MCP: config=%s, metric=%s, version=%s, lookback=%s", @@ -377,7 +396,24 @@ async def get_performance_data( ) if isinstance(result, dict) and "values" in result: - return result + values = result.get("values", []) + if isinstance(values, list): + return PerformanceData( + config=config, + metric=metric, + version=version, + lookback=lookback, + values=[v for v in values if v is not None], + error=result.get("error"), + ) + return PerformanceData( + config=config, + metric=metric, + version=version, + lookback=lookback, + values=[], + error="Unexpected data format for metric values", + ) if isinstance(result, dict) and "data" in result: data = result.get("data", {}) @@ -387,24 +423,22 @@ async def get_performance_data( values = metric_data.get("value", []) if isinstance(values, list): values = [v for v in values if v is not None] - return { - "config": config, - "metric": metric, - "version": version, - "lookback": str(lookback), - "values": values, - "count": len(values), - } + return PerformanceData( + config=config, + metric=metric, + version=version, + lookback=lookback, + values=values, + ) - return { - "config": config, - "metric": metric, - "version": version, - "lookback": str(lookback), - "values": [], - "count": 0, - "error": "no data found", - } + return PerformanceData( + config=config, + metric=metric, + version=version, + lookback=lookback, + values=[], + error="no data found", + ) def parse_perf_summary_args( @@ -573,7 +607,7 @@ async def analyze_performance( version=ver, lookback=lookback_days, ) - this_period_values = this_period_data.get("values", []) + this_period_values = this_period_data.values if not this_period_values: row = { @@ -595,7 +629,7 @@ async def analyze_performance( version=ver, lookback=lookback_window, ) - two_period_values = two_period_data.get("values", []) + two_period_values = two_period_data.values this_period_count = len(this_period_values) if len(two_period_values) > this_period_count: From 5c9ffbb9c6e23038e8ab69caa901ecc6376dd8c7 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Mon, 23 Feb 2026 18:18:49 -0500 Subject: [PATCH 08/11] Enhance README with Performance Summary feature details - Added a new section detailing the usage of the Performance Summary feature in BugZooka. - Included examples and notes on configuration and behavior. - Updated mandatory and optional fields to include `JEDI_BOT_SLACK_USER_ID` and performance summary settings. --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index 80b4005..a677ae7 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,29 @@ Mention the bot in Slack with the following format: ``` For local testing, see [Orion-MCP](https://github.com/jtaleric/orion-mcp) for instructions on how to run orion-mcp. +#### **Performance Summary (Orion-MCP)** + +BugZooka can generate a configurable performance summary across metrics for one or more configs and versions. This feature requires the **orion-mcp server** to be reachable via `mcp_config.json`. + +**Usage:** +``` +@BugZooka performance summary [ALL|config1.yaml,config2.yaml] [version ...] [verbose] +``` + +**Examples:** +``` +@BugZooka performance summary 14d +@BugZooka performance summary 30d ALL verbose +@BugZooka performance summary 7d trt-external-payload-node-density.yaml 4.19 +``` + +**Notes:** +- If no config is provided, defaults to a curated control-plane config list. +- `ALL` uses all available Orion configs (fallback list is used if MCP is unavailable). +- Multiple versions can be provided (e.g. `4.19 4.20`). +- When Socket Mode is enabled, the polling handler skips `performance summary` to avoid duplicate replies. +- Without the verbose option, the default is ranking the top 15 most influential metrics with visual hints. + ## **Configurables** This tool monitors a slack channel and uses AI to provide replies to CI failure messages. Also it operates as a singleton instance. @@ -108,6 +131,7 @@ All secrets are passed using a `.env` file which is located in the root director ### Mandatory fields SLACK_BOT_TOKEN="YOUR_SLACK_BOT_TOKEN" SLACK_CHANNEL_ID="YOUR_SLACK_CHANNEL_ID" +JEDI_BOT_SLACK_USER_ID="YOUR_BOT_USER_ID" ### Optional for Socket Mode (required only when using --enable-socket-mode) SLACK_APP_TOKEN="YOUR_SLACK_APP_TOKEN" # App-level token (xapp-*) for WebSocket mode @@ -129,6 +153,12 @@ INFERENCE_API_RETRY_MAX_ATTEMPTS="3" # Max retry attempts (default: 3) INFERENCE_API_RETRY_DELAY="5.0" # Initial retry delay in seconds (default: 5.0) INFERENCE_API_RETRY_BACKOFF_MULTIPLIER="2.0" # Exponential backoff multiplier (default: 2.0) INFERENCE_API_RETRY_MAX_DELAY="60.0" # Max retry delay in seconds (default: 60.0) + +### Performance Summary (optional) +PERF_SUMMARY_LOOKBACK_DAYS="14" # Default lookback window for perf summary +PERF_SUMMARY_MAX_METRIC_LEN="40" # Truncation length for metric labels +PERF_SUMMARY_MAX_CONFIG_LEN="25" # Truncation length for config labels +PERF_SUMMARY_SLACK_MSG_LIMIT="3500" # Slack message size safety cap ``` **Note**: The inference client works with any OpenAI-compatible API endpoint. Make sure to provide the mandatory Slack and inference configuration. From 910e7c43133be343bce718324bc19546370f8785 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Wed, 25 Feb 2026 15:33:14 -0500 Subject: [PATCH 09/11] Removed the performance summary polling logic and improved code clarity --- bugzooka/analysis/perf_summary_analyzer.py | 56 ++++---------- bugzooka/integrations/slack_fetcher.py | 87 ---------------------- 2 files changed, 16 insertions(+), 127 deletions(-) diff --git a/bugzooka/analysis/perf_summary_analyzer.py b/bugzooka/analysis/perf_summary_analyzer.py index 682a254..a5ef71c 100644 --- a/bugzooka/analysis/perf_summary_analyzer.py +++ b/bugzooka/analysis/perf_summary_analyzer.py @@ -248,42 +248,6 @@ def _format_metrics_table( ) -def _format_config_table( - config: str, - version: str, - rows: List[dict], - total_metrics: int, - lookback_days: int, -) -> str: - return _format_metrics_table( - title=f"*Config: {config}*", - version=version, - rows=rows, - total_metrics=total_metrics, - lookback_days=lookback_days, - include_config=False, - note_prefix="showing", - ) - - -def _format_summary_table( - version: str, - rows: List[dict], - total_metrics: int, - lookback_days: int, - limit: int, -) -> str: - return _format_metrics_table( - title=f"*Top {limit} Changes*", - version=version, - rows=rows, - total_metrics=total_metrics, - lookback_days=lookback_days, - include_config=True, - note_prefix="top", - ) - - async def get_configs() -> List[str]: """ Get list of available Orion configuration files via MCP tool. @@ -655,8 +619,14 @@ async def analyze_performance( if verbose: result_parts.append( - _format_config_table( - cfg, ver, rows, len(metrics), lookback_days + _format_metrics_table( + title=f"*Config: {cfg}*", + version=ver, + rows=rows, + total_metrics=len(metrics), + lookback_days=lookback_days, + include_config=False, + note_prefix="showing", ) ) @@ -689,8 +659,14 @@ def _include_in_summary(row: dict[str, Any]) -> bool: top_rows = sorted_rows[:limit] if top_rows: result_parts.append( - _format_summary_table( - ver, top_rows, total_metrics, lookback_days, limit + _format_metrics_table( + title=f"*Top {limit} Changes*", + version=ver, + rows=top_rows, + total_metrics=total_metrics, + lookback_days=lookback_days, + include_config=True, + note_prefix="top", ) ) else: diff --git a/bugzooka/integrations/slack_fetcher.py b/bugzooka/integrations/slack_fetcher.py index 59b0537..ffd3635 100644 --- a/bugzooka/integrations/slack_fetcher.py +++ b/bugzooka/integrations/slack_fetcher.py @@ -1,4 +1,3 @@ -import asyncio import io import time import re @@ -19,10 +18,6 @@ filter_errors_with_llm, run_agent_analysis, ) -from bugzooka.analysis.perf_summary_analyzer import ( - analyze_performance, - parse_perf_summary_args, -) from bugzooka.analysis.log_summarizer import ( classify_failure_type, build_summary_sections, @@ -446,88 +441,6 @@ def _process_message(self, msg, enable_inference): return ts # No weekly trigger; dynamic summarize only - - # Performance summary trigger (polling mode) - if "performance summary" in text_lower: - if self.enable_socket_mode: - self.logger.info( - "Socket mode enabled; skipping performance summary in polling" - ) - return ts - try: - self.client.chat_postMessage( - channel=self.channel_id, - text="📊 Gathering performance summary... This may take a moment.", - thread_ts=ts, - ) - - ( - configs, - versions, - lookback_days, - verbose, - use_all_configs, - ) = parse_perf_summary_args(text) - self.logger.info( - "Triggering performance summary via polling: lookback_days=%s, versions=%s, configs=%s, verbose=%s, use_all_configs=%s", - lookback_days, - versions, - configs, - verbose, - use_all_configs, - ) - - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - result = loop.run_until_complete( - analyze_performance( - configs, - versions, - lookback_days=lookback_days, - verbose=verbose, - use_all_configs=use_all_configs, - ) - ) - finally: - loop.close() - - if result.get("success"): - messages = result.get("messages", []) - for msg in messages: - self.client.chat_postMessage( - channel=self.channel_id, - text=msg, - thread_ts=ts, - ) - self.logger.info( - "✅ Sent performance summary via polling (%s message(s))", - len(messages), - ) - else: - self.client.chat_postMessage( - channel=self.channel_id, - text=result.get("message", "Unknown error"), - thread_ts=ts, - ) - self.logger.warning( - "⚠️ Performance summary failed: %s", - result.get("message"), - ) - - except Exception as e: - self.logger.error( - "Error processing performance summary in polling mode: %s", - e, - exc_info=True, - ) - self.client.chat_postMessage( - channel=self.channel_id, - text=f"❌ Unexpected error: {str(e)}", - thread_ts=ts, - ) - return ts - if "failure" not in text_lower: self.logger.info("Not a failure job, skipping") return ts From f0c0b0d189735e74546fc0404e45e99f3a2f00b4 Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Fri, 27 Feb 2026 10:33:45 -0500 Subject: [PATCH 10/11] minor fix and refinement --- README.md | 15 +++------------ bugzooka/entrypoint.py | 1 - bugzooka/integrations/slack_fetcher.py | 3 +-- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 7876927..e38e779 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,6 @@ Mention the bot in Slack with the following format: ``` For local testing, see [Orion-MCP](https://github.com/jtaleric/orion-mcp) for instructions on how to run orion-mcp. -<<<<<<< option2-config-alias #### **Performance Summary (Orion-MCP)** BugZooka can generate a configurable performance summary across metrics for one or more configs and versions. This feature requires the **orion-mcp server** to be reachable via `mcp_config.json`. @@ -114,15 +113,14 @@ BugZooka can generate a configurable performance summary across metrics for one @BugZooka performance summary 14d @BugZooka performance summary 30d ALL verbose @BugZooka performance summary 7d trt-external-payload-node-density.yaml 4.19 +@BugZooka performance summary 7d trt-external-payload-node-density.yaml 4.19,4.20,4.22 ``` **Notes:** - If no config is provided, defaults to a curated control-plane config list. -- `ALL` uses all available Orion configs (fallback list is used if MCP is unavailable). -- Multiple versions can be provided (e.g. `4.19 4.20`). -- When Socket Mode is enabled, the polling handler skips `performance summary` to avoid duplicate replies. +- `ALL` uses all 41 available Orion configs (fallback list is used if MCP is unavailable). +- Socket Mode has to be enabled. - Without the verbose option, the default is ranking the top 15 most influential metrics with visual hints. -======= ### **Supported Bot Triggers** #### **Job Summary** * `summarize Nd` - Job summary for N number of days. @@ -136,7 +134,6 @@ BugZooka can generate a configurable performance summary across metrics for one * `@PerfScale Jedi inspect 4.22.0-0.nightly-2026-01-05-203335 for config trt-external-payload-node-density.yaml` - Calls `has_nightly_regressed` tool in [orion-mcp](https://github.com/cloud-bulldozer/orion-mcp) for a given nightly checks regression only for a given orion configuration file instead of the [default](https://github.com/cloud-bulldozer/orion-mcp/blob/main/orion_mcp.py#L470). **Note**: All the triggers that start with a bot mention (.i.e. `@PerfScale Jedi`) run in socket mode. All socket mode features can be used in any slack channel without needing to host your own on premise openshift deployment. ->>>>>>> main ## **Configurables** This tool monitors a slack channel and uses AI to provide replies to CI failure messages. Also it operates as a singleton instance. @@ -169,12 +166,6 @@ INFERENCE_API_RETRY_MAX_ATTEMPTS="3" # Max retry attempts (default: 3) INFERENCE_API_RETRY_DELAY="5.0" # Initial retry delay in seconds (default: 5.0) INFERENCE_API_RETRY_BACKOFF_MULTIPLIER="2.0" # Exponential backoff multiplier (default: 2.0) INFERENCE_API_RETRY_MAX_DELAY="60.0" # Max retry delay in seconds (default: 60.0) - -### Performance Summary (optional) -PERF_SUMMARY_LOOKBACK_DAYS="14" # Default lookback window for perf summary -PERF_SUMMARY_MAX_METRIC_LEN="40" # Truncation length for metric labels -PERF_SUMMARY_MAX_CONFIG_LEN="25" # Truncation length for config labels -PERF_SUMMARY_SLACK_MSG_LIMIT="3500" # Slack message size safety cap ``` **Note**: The inference client works with any OpenAI-compatible API endpoint. Make sure to provide the mandatory Slack and inference configuration. diff --git a/bugzooka/entrypoint.py b/bugzooka/entrypoint.py index a4c5945..69819de 100644 --- a/bugzooka/entrypoint.py +++ b/bugzooka/entrypoint.py @@ -55,7 +55,6 @@ def main() -> None: channel_id=SLACK_CHANNEL_ID, logger=logger, poll_interval=SLACK_POLL_INTERVAL, - enable_socket_mode=args.enable_socket_mode, ) listener = None diff --git a/bugzooka/integrations/slack_fetcher.py b/bugzooka/integrations/slack_fetcher.py index ffd3635..e62e131 100644 --- a/bugzooka/integrations/slack_fetcher.py +++ b/bugzooka/integrations/slack_fetcher.py @@ -42,14 +42,13 @@ class SlackMessageFetcher(SlackClientBase): """Continuously fetches new messages from a Slack channel and logs them.""" - def __init__(self, channel_id, logger, poll_interval=600, enable_socket_mode=False): + def __init__(self, channel_id, logger, poll_interval=600): """Initialize Slack client and channel details.""" # Initialize base class (handles WebClient, logger, channel_id, running flag, signal handler) super().__init__(logger, channel_id) self.poll_interval = poll_interval # How often to fetch messages self.last_seen_timestamp = None # Track the latest message timestamp - self.enable_socket_mode = enable_socket_mode def _sanitize_job_text(self, text: str) -> str: """ From a1ea8d4c1df2af28c8274dd7bad7f57f4313176e Mon Sep 17 00:00:00 2001 From: ArthurChenCoding Date: Mon, 2 Mar 2026 21:26:06 -0500 Subject: [PATCH 11/11] add greating message --- tests/test_slack_socket_listener.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_slack_socket_listener.py b/tests/test_slack_socket_listener.py index 27c0aa6..9383c4f 100644 --- a/tests/test_slack_socket_listener.py +++ b/tests/test_slack_socket_listener.py @@ -164,7 +164,8 @@ def test_process_mention_sends_greeting( "May the force be with you! :performance_jedi:\n\n" ":bulb: *Tips:*\n" "- `analyze pr: , compare with ` - PR performance analysis\n" - "- `inspect [vs ] [for config ] [for days]` - Nightly regression analysis" + "- `inspect [vs ] [for config ] [for days]` - Nightly regression analysis\n" + "- `performance summary [ALL|config1.yaml,config2.yaml] [version ...] [verbose]` - Performance metrics summary" ), thread_ts="1234567890.123456", )