From 6fe2e4077260702ee32d46d5a939a3197f3d2163 Mon Sep 17 00:00:00 2001 From: Drake Date: Wed, 11 Mar 2026 15:41:32 -0400 Subject: [PATCH 01/58] feat: expand desktop parity and foundry support --- README.md | 58 +- agent/__main__.py | 101 ++- agent/builder.py | 117 +++- agent/config.py | 146 +++- agent/credentials.py | 29 + agent/engine.py | 61 +- agent/model.py | 233 ++++++- agent/settings.py | 6 + agent/tool_defs.py | 2 +- agent/tools.py | 127 ++++ agent/tui.py | 26 +- openplanter-desktop/Cargo.lock | 1 - .../crates/op-core/src/builder.rs | 201 ++++-- .../crates/op-core/src/config.rs | 347 +++++++-- .../crates/op-core/src/credentials.rs | 41 +- .../crates/op-core/src/engine/curator.rs | 52 +- .../crates/op-core/src/engine/judge.rs | 12 +- .../crates/op-core/src/engine/mod.rs | 225 ++++-- .../crates/op-core/src/events.rs | 4 + openplanter-desktop/crates/op-core/src/lib.rs | 10 +- .../crates/op-core/src/model/anthropic.rs | 174 +++-- .../crates/op-core/src/model/mod.rs | 29 +- .../crates/op-core/src/model/openai.rs | 657 ++++++++++++++++-- .../crates/op-core/src/session/mod.rs | 2 +- .../crates/op-core/src/session/replay.rs | 22 +- .../crates/op-core/src/settings.rs | 67 +- .../crates/op-core/src/tools/defs.rs | 26 +- .../crates/op-core/src/tools/filesystem.rs | 21 +- .../crates/op-core/src/tools/mod.rs | 51 +- .../crates/op-core/src/tools/patching.rs | 122 +--- .../crates/op-core/src/tools/shell.rs | 35 +- .../crates/op-core/src/tools/web.rs | 592 +++++++++++++--- .../crates/op-core/src/wiki/matching.rs | 11 +- .../crates/op-core/src/wiki/mod.rs | 2 +- .../crates/op-core/src/wiki/parser.rs | 4 +- .../crates/op-core/src/wiki/watcher.rs | 57 +- .../op-core/tests/test_model_streaming.rs | 373 ++++++++-- .../crates/op-tauri/Cargo.toml | 1 - .../crates/op-tauri/src/bridge.rs | 64 +- .../crates/op-tauri/src/commands/agent.rs | 11 +- .../crates/op-tauri/src/commands/config.rs | 183 +++-- .../crates/op-tauri/src/commands/session.rs | 47 +- .../crates/op-tauri/src/commands/wiki.rs | 314 ++++++--- .../crates/op-tauri/src/main.rs | 6 +- .../crates/op-tauri/src/state.rs | 127 +++- .../crates/op-tauri/tauri.conf.json | 2 +- .../frontend/package-lock.json | 3 + .../frontend/src/api/invoke.test.ts | 44 +- openplanter-desktop/frontend/src/api/types.ts | 7 + .../src/commands/completionRegistry.test.ts | 23 + .../src/commands/completionRegistry.ts | 21 + .../frontend/src/commands/model.test.ts | 46 +- .../frontend/src/commands/model.ts | 81 ++- .../frontend/src/commands/reasoning.test.ts | 8 + .../frontend/src/commands/reasoning.ts | 6 +- .../frontend/src/commands/slash.test.ts | 28 + .../frontend/src/commands/slash.ts | 16 + .../frontend/src/commands/webSearch.test.ts | 76 ++ .../frontend/src/commands/webSearch.ts | 58 ++ .../frontend/src/commands/zaiPlan.test.ts | 79 +++ .../frontend/src/commands/zaiPlan.ts | 62 ++ .../frontend/src/components/App.test.ts | 18 +- .../frontend/src/components/App.ts | 4 +- .../frontend/src/components/StatusBar.test.ts | 13 + .../frontend/src/components/StatusBar.ts | 6 + openplanter-desktop/frontend/src/main.ts | 4 + .../frontend/src/state/store.ts | 4 + openplanter-desktop/package.json | 10 + tests/test_coverage_gaps.py | 116 +++- tests/test_credentials.py | 6 + tests/test_engine_complex.py | 113 ++- tests/test_model.py | 197 +++++- tests/test_settings.py | 24 + tests/test_streaming.py | 82 +++ tests/test_tools.py | 59 ++ tests/test_tools_complex.py | 21 + 76 files changed, 5031 insertions(+), 1003 deletions(-) create mode 100644 openplanter-desktop/frontend/src/commands/webSearch.test.ts create mode 100644 openplanter-desktop/frontend/src/commands/webSearch.ts create mode 100644 openplanter-desktop/frontend/src/commands/zaiPlan.test.ts create mode 100644 openplanter-desktop/frontend/src/commands/zaiPlan.ts create mode 100644 openplanter-desktop/package.json diff --git a/README.md b/README.md index 9fa92c1a..53a01029 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,9 @@ cd openplanter-desktop # Install frontend dependencies cd frontend && npm install && cd .. +# Install the Tauri Cargo subcommand +cargo install tauri-cli --version "^2" + # Run in development mode cargo tauri dev @@ -43,7 +46,7 @@ cargo tauri dev cargo tauri build ``` -Requires: Rust stable, Node.js 20+, and platform-specific Tauri dependencies ([see Tauri prerequisites](https://v2.tauri.app/start/prerequisites/)). +Requires: Rust stable, Node.js 20+, the Tauri CLI, and platform-specific Tauri dependencies ([see Tauri prerequisites](https://v2.tauri.app/start/prerequisites/)). ## CLI Agent @@ -81,12 +84,18 @@ The container mounts `./workspace` as the agent's working directory. | Provider | Default Model | Env Var | |----------|---------------|---------| -| OpenAI | `gpt-5.2` | `OPENAI_API_KEY` | -| Anthropic | `claude-opus-4-6` | `ANTHROPIC_API_KEY` | +| OpenAI | `azure-foundry/gpt-5.3-codex` | `OPENAI_API_KEY` | +| Anthropic | `anthropic-foundry/claude-opus-4-6` | `ANTHROPIC_API_KEY` | | OpenRouter | `anthropic/claude-sonnet-4-5` | `OPENROUTER_API_KEY` | | Cerebras | `qwen-3-235b-a22b-instruct-2507` | `CEREBRAS_API_KEY` | +| Z.AI | `glm-5` | `ZAI_API_KEY` | | Ollama | `llama3.2` | (none — local) | +OpenAI-compatible requests now default to the Azure Foundry proxy at +`https://foundry-proxy.cheetah-koi.ts.net/openai/v1`, and Anthropic requests +default to the Anthropic Foundry proxy at +`https://foundry-proxy.cheetah-koi.ts.net/anthropic/v1`. + ### Local Models (Ollama) [Ollama](https://ollama.com) runs models locally with no API key. Install Ollama, pull a model (`ollama pull llama3.2`), then: @@ -99,6 +108,46 @@ openplanter-agent --provider ollama --list-models The base URL defaults to `http://localhost:11434/v1` and can be overridden with `OPENPLANTER_OLLAMA_BASE_URL` or `--base-url`. The first request may be slow while Ollama loads the model into memory; a 120-second first-byte timeout is used automatically. +### Z.AI Endpoint Plans + +Z.AI has two distinct endpoint plans: + +- PAYGO endpoint: `https://api.z.ai/api/paas/v4` +- Coding plan endpoint: `https://api.z.ai/api/coding/paas/v4` + +Choose the plan explicitly: + +```bash +export OPENPLANTER_ZAI_PLAN=paygo # or coding +``` + +Or per run: + +```bash +openplanter-agent --provider zai --model glm-5 --zai-plan coding +``` + +Advanced overrides: + +```bash +export OPENPLANTER_ZAI_PAYGO_BASE_URL=https://api.z.ai/api/paas/v4 +export OPENPLANTER_ZAI_CODING_BASE_URL=https://api.z.ai/api/coding/paas/v4 +``` + +`OPENPLANTER_ZAI_BASE_URL` still overrides both plans when set. + +### Z.AI Reliability Tuning + +Z.AI rate limits (`HTTP 429`, code `1302`) are retried with capped backoff and jitter. For Z.AI streaming connection issues, OpenPlanter also retries up to `OPENPLANTER_ZAI_STREAM_MAX_RETRIES` times. + +```bash +export OPENPLANTER_RATE_LIMIT_MAX_RETRIES=12 +export OPENPLANTER_RATE_LIMIT_BACKOFF_BASE_SEC=1.0 +export OPENPLANTER_RATE_LIMIT_BACKOFF_MAX_SEC=60.0 +export OPENPLANTER_RATE_LIMIT_RETRY_AFTER_CAP_SEC=120.0 +export OPENPLANTER_ZAI_STREAM_MAX_RETRIES=10 +``` + Additional service keys: `EXA_API_KEY` (web search), `VOYAGE_API_KEY` (embeddings). All keys can also be set with an `OPENPLANTER_` prefix (e.g. `OPENPLANTER_OPENAI_API_KEY`), via `.env` files in the workspace, or via CLI flags. @@ -136,8 +185,9 @@ openplanter-agent [options] | Flag | Description | |------|-------------| -| `--provider NAME` | `auto`, `openai`, `anthropic`, `openrouter`, `cerebras`, `ollama` | +| `--provider NAME` | `auto`, `openai`, `anthropic`, `openrouter`, `cerebras`, `zai`, `ollama` | | `--model NAME` | Model name or `newest` to auto-select | +| `--zai-plan PLAN` | Z.AI endpoint plan: `paygo` or `coding` | | `--reasoning-effort LEVEL` | `low`, `medium`, `high`, or `none` | | `--list-models` | Fetch available models from the provider API | diff --git a/agent/__main__.py b/agent/__main__.py index 8ba38df4..f3c29eca 100644 --- a/agent/__main__.py +++ b/agent/__main__.py @@ -6,7 +6,13 @@ from datetime import datetime, timezone from .builder import _fetch_models_for_provider, build_engine, infer_provider_for_model -from .config import AgentConfig +from .config import ( + AgentConfig, + normalize_zai_plan, + resolve_anthropic_api_key, + resolve_openai_api_key, + resolve_zai_base_url, +) from .credentials import ( CredentialBundle, CredentialStore, @@ -33,7 +39,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--provider", default=None, - choices=["auto", "openai", "anthropic", "openrouter", "cerebras", "ollama", "all"], + choices=["auto", "openai", "anthropic", "openrouter", "cerebras", "zai", "ollama", "all"], help="Model provider. Use 'all' only with --list-models.", ) parser.add_argument("--model", help="Model name (use 'newest' to auto-select latest from API).") @@ -67,6 +73,10 @@ def build_parser() -> argparse.ArgumentParser: "--default-model-cerebras", help="Persist workspace default model for Cerebras provider.", ) + parser.add_argument( + "--default-model-zai", + help="Persist workspace default model for Z.AI provider.", + ) parser.add_argument( "--default-model-ollama", help="Persist workspace default model for Ollama provider.", @@ -82,7 +92,19 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--anthropic-api-key", help="Anthropic API key override.") parser.add_argument("--openrouter-api-key", help="OpenRouter API key override.") parser.add_argument("--cerebras-api-key", help="Cerebras API key override.") + parser.add_argument("--zai-api-key", help="Z.AI API key override.") + parser.add_argument( + "--zai-plan", + choices=["paygo", "coding"], + help="Z.AI endpoint plan: paygo uses /api/paas/v4, coding uses /api/coding/paas/v4.", + ) parser.add_argument("--exa-api-key", help="Exa API key override.") + parser.add_argument("--firecrawl-api-key", help="Firecrawl API key override.") + parser.add_argument( + "--web-search-provider", + choices=["exa", "firecrawl"], + help="Web search backend provider.", + ) parser.add_argument("--voyage-api-key", help="Voyage API key override.") parser.add_argument( "--configure-keys", @@ -112,6 +134,11 @@ def build_parser() -> argparse.ArgumentParser: "--session-id", help="Session id to use. If omitted, a new id is generated unless --resume is used.", ) + parser.add_argument( + "session_id_positional", + nargs="?", + help=argparse.SUPPRESS, + ) parser.add_argument( "--resume", action="store_true", @@ -153,7 +180,7 @@ def _format_ts(ts: int) -> str: def _resolve_provider(requested: str, creds: CredentialBundle) -> str: requested = requested.strip().lower() - if requested in {"openai", "anthropic", "openrouter", "cerebras", "ollama"}: + if requested in {"openai", "anthropic", "openrouter", "cerebras", "zai", "ollama"}: return requested if requested == "all": return "all" @@ -165,15 +192,17 @@ def _resolve_provider(requested: str, creds: CredentialBundle) -> str: return "openrouter" if creds.cerebras_api_key: return "cerebras" - return "openai" + if creds.zai_api_key: + return "zai" + return "anthropic" def _print_models(cfg: AgentConfig, requested_provider: str) -> int: providers: list[str] if requested_provider == "all": - providers = ["openai", "anthropic", "openrouter", "cerebras", "ollama"] + providers = ["openai", "anthropic", "openrouter", "cerebras", "zai", "ollama"] elif requested_provider == "auto": - providers = ["openai", "anthropic", "openrouter", "cerebras", "ollama"] + providers = ["openai", "anthropic", "openrouter", "cerebras", "zai", "ollama"] else: providers = [requested_provider] @@ -209,7 +238,9 @@ def _load_credentials( anthropic_api_key=user_creds.anthropic_api_key, openrouter_api_key=user_creds.openrouter_api_key, cerebras_api_key=user_creds.cerebras_api_key, + zai_api_key=user_creds.zai_api_key, exa_api_key=user_creds.exa_api_key, + firecrawl_api_key=user_creds.firecrawl_api_key, voyage_api_key=user_creds.voyage_api_key, ) @@ -223,8 +254,12 @@ def _load_credentials( creds.openrouter_api_key = stored.openrouter_api_key if stored.cerebras_api_key: creds.cerebras_api_key = stored.cerebras_api_key + if stored.zai_api_key: + creds.zai_api_key = stored.zai_api_key if stored.exa_api_key: creds.exa_api_key = stored.exa_api_key + if stored.firecrawl_api_key: + creds.firecrawl_api_key = stored.firecrawl_api_key if stored.voyage_api_key: creds.voyage_api_key = stored.voyage_api_key @@ -237,8 +272,12 @@ def _load_credentials( creds.openrouter_api_key = env_creds.openrouter_api_key if env_creds.cerebras_api_key: creds.cerebras_api_key = env_creds.cerebras_api_key + if env_creds.zai_api_key: + creds.zai_api_key = env_creds.zai_api_key if env_creds.exa_api_key: creds.exa_api_key = env_creds.exa_api_key + if env_creds.firecrawl_api_key: + creds.firecrawl_api_key = env_creds.firecrawl_api_key if env_creds.voyage_api_key: creds.voyage_api_key = env_creds.voyage_api_key @@ -256,8 +295,12 @@ def _load_credentials( creds.openrouter_api_key = args.openrouter_api_key.strip() or creds.openrouter_api_key if args.cerebras_api_key: creds.cerebras_api_key = args.cerebras_api_key.strip() or creds.cerebras_api_key + if args.zai_api_key: + creds.zai_api_key = args.zai_api_key.strip() or creds.zai_api_key if args.exa_api_key: creds.exa_api_key = args.exa_api_key.strip() or creds.exa_api_key + if args.firecrawl_api_key: + creds.firecrawl_api_key = args.firecrawl_api_key.strip() or creds.firecrawl_api_key if args.voyage_api_key: creds.voyage_api_key = args.voyage_api_key.strip() or creds.voyage_api_key @@ -296,14 +339,27 @@ def _apply_runtime_overrides(cfg: AgentConfig, args: argparse.Namespace, creds: cfg.provider = args.provider cfg.provider = _resolve_provider(cfg.provider, creds) - cfg.openai_api_key = creds.openai_api_key - cfg.anthropic_api_key = creds.anthropic_api_key + cfg.openai_api_key = resolve_openai_api_key(creds.openai_api_key, cfg.openai_base_url) + cfg.anthropic_api_key = resolve_anthropic_api_key( + creds.anthropic_api_key, + cfg.anthropic_base_url, + ) cfg.openrouter_api_key = creds.openrouter_api_key cfg.cerebras_api_key = creds.cerebras_api_key + cfg.zai_api_key = creds.zai_api_key cfg.exa_api_key = creds.exa_api_key + cfg.firecrawl_api_key = creds.firecrawl_api_key cfg.voyage_api_key = creds.voyage_api_key cfg.api_key = cfg.openai_api_key + if args.zai_plan: + cfg.zai_plan = normalize_zai_plan(args.zai_plan) + cfg.zai_base_url = resolve_zai_base_url( + cfg.zai_plan, + paygo_base_url=cfg.zai_paygo_base_url, + coding_base_url=cfg.zai_coding_base_url, + ) + if args.base_url: if cfg.provider == "openai": cfg.openai_base_url = args.base_url @@ -313,12 +369,25 @@ def _apply_runtime_overrides(cfg: AgentConfig, args: argparse.Namespace, creds: cfg.openrouter_base_url = args.base_url elif cfg.provider == "cerebras": cfg.cerebras_base_url = args.base_url + elif cfg.provider == "zai": + cfg.zai_base_url = args.base_url elif cfg.provider == "ollama": cfg.ollama_base_url = args.base_url cfg.base_url = args.base_url + cfg.openai_api_key = resolve_openai_api_key(cfg.openai_api_key, cfg.openai_base_url) + cfg.anthropic_api_key = resolve_anthropic_api_key( + cfg.anthropic_api_key, + cfg.anthropic_base_url, + ) + cfg.api_key = resolve_openai_api_key(cfg.api_key, cfg.base_url) + if args.model: cfg.model = args.model + if args.web_search_provider: + cfg.web_search_provider = args.web_search_provider + if cfg.web_search_provider not in {"exa", "firecrawl"}: + cfg.web_search_provider = "exa" if args.reasoning_effort: cfg.reasoning_effort = None if args.reasoning_effort == "none" else args.reasoning_effort if args.recursive: @@ -390,6 +459,9 @@ def _apply_persistent_settings( if args.default_model_cerebras is not None: settings.default_model_cerebras = args.default_model_cerebras.strip() or None changed = True + if args.default_model_zai is not None: + settings.default_model_zai = args.default_model_zai.strip() or None + changed = True if args.default_model_ollama is not None: settings.default_model_ollama = args.default_model_ollama.strip() or None changed = True @@ -423,6 +495,7 @@ def _print_settings(settings: PersistentSettings) -> None: print(f" default_model_anthropic: {settings.default_model_anthropic or '(unset)'}") print(f" default_model_openrouter: {settings.default_model_openrouter or '(unset)'}") print(f" default_model_cerebras: {settings.default_model_cerebras or '(unset)'}") + print(f" default_model_zai: {settings.default_model_zai or '(unset)'}") print(f" default_model_ollama: {settings.default_model_ollama or '(unset)'}") @@ -449,6 +522,8 @@ def _has_non_interactive_command(args: argparse.Namespace) -> bool: return True if args.default_model_cerebras is not None: return True + if args.default_model_zai is not None: + return True if args.default_model_ollama is not None: return True return False @@ -458,6 +533,11 @@ def main() -> None: parser = build_parser() args = parser.parse_args() + if args.resume and args.session_id is None and args.session_id_positional: + args.session_id = args.session_id_positional + elif args.session_id_positional and not args.resume: + parser.error("Positional session-id is only supported with --resume.") + non_tty = not (sys.stdin.isatty() and sys.stdout.isatty()) if (args.headless or non_tty) and not args.textual: args.no_tui = True @@ -526,6 +606,7 @@ def main() -> None: "anthropic": cfg.anthropic_api_key, "openrouter": cfg.openrouter_api_key, "cerebras": cfg.cerebras_api_key, + "zai": cfg.zai_api_key, "ollama": "ollama", }.get(inferred) if key: @@ -554,7 +635,11 @@ def main() -> None: startup_info: dict[str, str] = { "Provider": cfg.provider, "Model": model_name, + "WebSearch": cfg.web_search_provider, } + if cfg.provider == "zai": + startup_info["ZAIPlan"] = cfg.zai_plan + startup_info["ZAIURL"] = cfg.zai_base_url if cfg.reasoning_effort: startup_info["Reasoning"] = cfg.reasoning_effort startup_info["Mode"] = "recursive" if cfg.recursive else "flat" diff --git a/agent/builder.py b/agent/builder.py index a47d3e31..1a07bf56 100644 --- a/agent/builder.py +++ b/agent/builder.py @@ -9,7 +9,16 @@ import re from pathlib import Path -from .config import PROVIDER_DEFAULT_MODELS, AgentConfig +from .config import ( + ANTHROPIC_FOUNDRY_MODEL_PREFIX, + AZURE_FOUNDRY_MODEL_PREFIX, + PROVIDER_DEFAULT_MODELS, + AgentConfig, + is_foundry_anthropic_base_url, + is_foundry_openai_base_url, + resolve_anthropic_api_key, + resolve_openai_api_key, +) from .engine import RLMEngine from .model import ( AnthropicModel, @@ -27,7 +36,8 @@ # Patterns that unambiguously identify a provider. _ANTHROPIC_RE = re.compile(r"^claude", re.IGNORECASE) _OPENAI_RE = re.compile(r"^(gpt|o[1-4]-|o[1-4]$|chatgpt|dall-e|tts-|whisper)", re.IGNORECASE) -_CEREBRAS_RE = re.compile(r"^(llama.*cerebras|qwen-3|gpt-oss|zai-glm)", re.IGNORECASE) +_CEREBRAS_RE = re.compile(r"^(llama.*cerebras|qwen-3|gpt-oss)", re.IGNORECASE) +_ZAI_RE = re.compile(r"^(glm|zai-glm)", re.IGNORECASE) _OLLAMA_RE = re.compile( r"^(llama|mistral|gemma|phi|codellama|deepseek|vicuna|tinyllama|" r"neural-chat|dolphin|wizardlm|orca|nous-hermes|command-r|qwen(?!-3))", @@ -37,12 +47,19 @@ def infer_provider_for_model(model: str) -> str | None: """Return the likely provider for *model*, or ``None`` if ambiguous.""" + lowered = model.strip().lower() + if lowered.startswith(ANTHROPIC_FOUNDRY_MODEL_PREFIX): + return "anthropic" + if lowered.startswith(AZURE_FOUNDRY_MODEL_PREFIX): + return "openai" if "/" in model: return "openrouter" if _ANTHROPIC_RE.search(model): return "anthropic" if _CEREBRAS_RE.search(model): return "cerebras" + if _ZAI_RE.search(model): + return "zai" if _OPENAI_RE.search(model): return "openai" if _OLLAMA_RE.search(model): @@ -66,13 +83,37 @@ def _validate_model_provider(model_name: str, provider: str) -> None: def _fetch_models_for_provider(cfg: AgentConfig, provider: str) -> list[dict]: if provider == "openai": - if not cfg.openai_api_key: + api_key = resolve_openai_api_key(cfg.openai_api_key, cfg.openai_base_url) + if not api_key: raise ModelError("OpenAI key not configured.") - return list_openai_models(api_key=cfg.openai_api_key, base_url=cfg.openai_base_url) + models = list_openai_models(api_key=api_key, base_url=cfg.openai_base_url) + if is_foundry_openai_base_url(cfg.openai_base_url): + return [ + { + **row, + "id": row["id"] + if str(row.get("id", "")).lower().startswith(AZURE_FOUNDRY_MODEL_PREFIX) + else f"{AZURE_FOUNDRY_MODEL_PREFIX}{row['id']}", + } + for row in models + ] + return models if provider == "anthropic": - if not cfg.anthropic_api_key: + api_key = resolve_anthropic_api_key(cfg.anthropic_api_key, cfg.anthropic_base_url) + if not api_key: raise ModelError("Anthropic key not configured.") - return list_anthropic_models(api_key=cfg.anthropic_api_key, base_url=cfg.anthropic_base_url) + models = list_anthropic_models(api_key=api_key, base_url=cfg.anthropic_base_url) + if is_foundry_anthropic_base_url(cfg.anthropic_base_url): + return [ + { + **row, + "id": row["id"] + if str(row.get("id", "")).lower().startswith(ANTHROPIC_FOUNDRY_MODEL_PREFIX) + else f"{ANTHROPIC_FOUNDRY_MODEL_PREFIX}{row['id']}", + } + for row in models + ] + return models if provider == "openrouter": if not cfg.openrouter_api_key: raise ModelError("OpenRouter key not configured.") @@ -81,6 +122,10 @@ def _fetch_models_for_provider(cfg: AgentConfig, provider: str) -> list[dict]: if not cfg.cerebras_api_key: raise ModelError("Cerebras key not configured.") return list_openai_models(api_key=cfg.cerebras_api_key, base_url=cfg.cerebras_base_url) + if provider == "zai": + if not cfg.zai_api_key: + raise ModelError("Z.AI key not configured.") + return list_openai_models(api_key=cfg.zai_api_key, base_url=cfg.zai_base_url) if provider == "ollama": return list_ollama_models(base_url=cfg.ollama_base_url) raise ModelError(f"Unknown provider: {provider}") @@ -98,25 +143,28 @@ def _resolve_model_name(cfg: AgentConfig) -> str: if not models: raise ModelError(f"No models returned for provider '{cfg.provider}'.") return str(models[0]["id"]) - return PROVIDER_DEFAULT_MODELS.get(cfg.provider, "claude-opus-4-6") + return PROVIDER_DEFAULT_MODELS.get(cfg.provider, "anthropic-foundry/claude-opus-4-6") def build_model_factory(cfg: AgentConfig) -> ModelFactory | None: """Return a factory that creates models by name + optional reasoning effort.""" + openai_api_key = resolve_openai_api_key(cfg.openai_api_key, cfg.openai_base_url) + anthropic_api_key = resolve_anthropic_api_key(cfg.anthropic_api_key, cfg.anthropic_base_url) + def _factory(model_name: str, reasoning_effort: str | None = None) -> AnthropicModel | OpenAICompatibleModel: provider = infer_provider_for_model(model_name) effort = reasoning_effort or cfg.reasoning_effort - if provider == "anthropic" and cfg.anthropic_api_key: + if provider == "anthropic" and anthropic_api_key: return AnthropicModel( model=model_name, - api_key=cfg.anthropic_api_key, + api_key=anthropic_api_key, base_url=cfg.anthropic_base_url, reasoning_effort=effort, ) - if provider in ("openai", None) and cfg.openai_api_key: + if provider in ("openai", None) and openai_api_key: return OpenAICompatibleModel( model=model_name, - api_key=cfg.openai_api_key, + api_key=openai_api_key, base_url=cfg.openai_base_url, reasoning_effort=effort, ) @@ -138,6 +186,18 @@ def _factory(model_name: str, reasoning_effort: str | None = None) -> AnthropicM base_url=cfg.cerebras_base_url, reasoning_effort=effort, ) + if provider == "zai" and cfg.zai_api_key: + thinking_type = "disabled" if effort in (None, "", "none") else "enabled" + return OpenAICompatibleModel( + model=model_name, + api_key=cfg.zai_api_key, + base_url=cfg.zai_base_url, + reasoning_effort=effort, + thinking_type=thinking_type, + extra_headers={"Accept-Language": "en-US,en"}, + provider="zai", + stream_max_retries=cfg.zai_stream_max_retries, + ) if provider == "ollama": return OpenAICompatibleModel( model=model_name, @@ -149,7 +209,14 @@ def _factory(model_name: str, reasoning_effort: str | None = None) -> AnthropicM ) raise ModelError(f"No API key available for model '{model_name}' (provider={provider})") - if cfg.anthropic_api_key or cfg.openai_api_key or cfg.openrouter_api_key or cfg.cerebras_api_key or cfg.ollama_base_url: + if ( + anthropic_api_key + or openai_api_key + or cfg.openrouter_api_key + or cfg.cerebras_api_key + or cfg.zai_api_key + or cfg.ollama_base_url + ): return _factory return None @@ -163,8 +230,11 @@ def build_engine(cfg: AgentConfig) -> RLMEngine: max_file_chars=cfg.max_file_chars, max_files_listed=cfg.max_files_listed, max_search_hits=cfg.max_search_hits, + web_search_provider=cfg.web_search_provider, exa_api_key=cfg.exa_api_key, exa_base_url=cfg.exa_base_url, + firecrawl_api_key=cfg.firecrawl_api_key, + firecrawl_base_url=cfg.firecrawl_base_url, ) try: @@ -175,10 +245,13 @@ def build_engine(cfg: AgentConfig) -> RLMEngine: _validate_model_provider(model_name, cfg.provider) - if cfg.provider == "openai" and cfg.openai_api_key: + openai_api_key = resolve_openai_api_key(cfg.openai_api_key, cfg.openai_base_url) + anthropic_api_key = resolve_anthropic_api_key(cfg.anthropic_api_key, cfg.anthropic_base_url) + + if cfg.provider == "openai" and openai_api_key: model = OpenAICompatibleModel( model=model_name, - api_key=cfg.openai_api_key, + api_key=openai_api_key, base_url=cfg.openai_base_url, reasoning_effort=cfg.reasoning_effort, ) @@ -200,6 +273,18 @@ def build_engine(cfg: AgentConfig) -> RLMEngine: base_url=cfg.cerebras_base_url, reasoning_effort=cfg.reasoning_effort, ) + elif cfg.provider == "zai" and cfg.zai_api_key: + thinking_type = "disabled" if cfg.reasoning_effort in (None, "", "none") else "enabled" + model = OpenAICompatibleModel( + model=model_name, + api_key=cfg.zai_api_key, + base_url=cfg.zai_base_url, + reasoning_effort=cfg.reasoning_effort, + thinking_type=thinking_type, + extra_headers={"Accept-Language": "en-US,en"}, + provider="zai", + stream_max_retries=cfg.zai_stream_max_retries, + ) elif cfg.provider == "ollama": model = OpenAICompatibleModel( model=model_name, @@ -209,10 +294,10 @@ def build_engine(cfg: AgentConfig) -> RLMEngine: first_byte_timeout=120, strict_tools=False, ) - elif cfg.provider == "anthropic" and cfg.anthropic_api_key: + elif cfg.provider == "anthropic" and anthropic_api_key: model = AnthropicModel( model=model_name, - api_key=cfg.anthropic_api_key, + api_key=anthropic_api_key, base_url=cfg.anthropic_base_url, reasoning_effort=cfg.reasoning_effort, ) diff --git a/agent/config.py b/agent/config.py index 86d368c4..527c0d2c 100644 --- a/agent/config.py +++ b/agent/config.py @@ -4,34 +4,114 @@ from dataclasses import dataclass from pathlib import Path +AZURE_FOUNDRY_MODEL_PREFIX = "azure-foundry/" +ANTHROPIC_FOUNDRY_MODEL_PREFIX = "anthropic-foundry/" +FOUNDRY_OPENAI_BASE_URL = "https://foundry-proxy.cheetah-koi.ts.net/openai/v1" +FOUNDRY_ANTHROPIC_BASE_URL = "https://foundry-proxy.cheetah-koi.ts.net/anthropic/v1" +FOUNDRY_OPENAI_API_KEY_PLACEHOLDER = "dont-worry-this-key-will-be-auto-injected" +FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER = "dont-worry-it-will-be-injected" +ZAI_PAYGO_BASE_URL = "https://api.z.ai/api/paas/v4" +ZAI_CODING_BASE_URL = "https://api.z.ai/api/coding/paas/v4" + PROVIDER_DEFAULT_MODELS: dict[str, str] = { - "openai": "gpt-5.2", - "anthropic": "claude-opus-4-6", + "openai": "azure-foundry/gpt-5.3-codex", + "anthropic": "anthropic-foundry/claude-opus-4-6", "openrouter": "anthropic/claude-sonnet-4-5", "cerebras": "qwen-3-235b-a22b-instruct-2507", + "zai": "glm-5", "ollama": "llama3.2", } +def normalize_zai_plan(value: str | None) -> str: + text = (value or "").strip().lower() + if text in {"paygo", "coding"}: + return text + return "paygo" + + +def resolve_zai_base_url( + plan: str, + *, + paygo_base_url: str = ZAI_PAYGO_BASE_URL, + coding_base_url: str = ZAI_CODING_BASE_URL, +) -> str: + return coding_base_url if normalize_zai_plan(plan) == "coding" else paygo_base_url + + +def _normalize_base_url(url: str) -> str: + return url.strip().rstrip("/") + + +def is_foundry_openai_base_url(url: str) -> bool: + return _normalize_base_url(url) == FOUNDRY_OPENAI_BASE_URL + + +def is_foundry_anthropic_base_url(url: str) -> bool: + return _normalize_base_url(url) == FOUNDRY_ANTHROPIC_BASE_URL + + +def resolve_openai_api_key(api_key: str | None, base_url: str) -> str | None: + key = (api_key or "").strip() or None + if key == FOUNDRY_OPENAI_API_KEY_PLACEHOLDER and not is_foundry_openai_base_url(base_url): + return None + if key: + return key + if is_foundry_openai_base_url(base_url): + return FOUNDRY_OPENAI_API_KEY_PLACEHOLDER + return None + + +def resolve_anthropic_api_key(api_key: str | None, base_url: str) -> str | None: + key = (api_key or "").strip() or None + if ( + key == FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER + and not is_foundry_anthropic_base_url(base_url) + ): + return None + if key: + return key + if is_foundry_anthropic_base_url(base_url): + return FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER + return None + + +def strip_foundry_model_prefix(model: str) -> str: + text = model.strip() + lower = text.lower() + if lower.startswith(AZURE_FOUNDRY_MODEL_PREFIX): + return text[len(AZURE_FOUNDRY_MODEL_PREFIX):] + if lower.startswith(ANTHROPIC_FOUNDRY_MODEL_PREFIX): + return text[len(ANTHROPIC_FOUNDRY_MODEL_PREFIX):] + return text + @dataclass(slots=True) class AgentConfig: workspace: Path provider: str = "auto" - model: str = "claude-opus-4-6" + model: str = "anthropic-foundry/claude-opus-4-6" reasoning_effort: str | None = "high" - base_url: str = "https://api.openai.com/v1" # Legacy alias for OpenAI-compatible base URL. + base_url: str = FOUNDRY_OPENAI_BASE_URL # Legacy alias for OpenAI-compatible base URL. api_key: str | None = None # Legacy alias for OpenAI key. - openai_base_url: str = "https://api.openai.com/v1" - anthropic_base_url: str = "https://api.anthropic.com/v1" + openai_base_url: str = FOUNDRY_OPENAI_BASE_URL + anthropic_base_url: str = FOUNDRY_ANTHROPIC_BASE_URL openrouter_base_url: str = "https://openrouter.ai/api/v1" cerebras_base_url: str = "https://api.cerebras.ai/v1" + zai_plan: str = "paygo" + zai_paygo_base_url: str = ZAI_PAYGO_BASE_URL + zai_coding_base_url: str = ZAI_CODING_BASE_URL + zai_base_url: str = ZAI_PAYGO_BASE_URL ollama_base_url: str = "http://localhost:11434/v1" exa_base_url: str = "https://api.exa.ai" + firecrawl_base_url: str = "https://api.firecrawl.dev/v1" openai_api_key: str | None = None anthropic_api_key: str | None = None openrouter_api_key: str | None = None cerebras_api_key: str | None = None + zai_api_key: str | None = None exa_api_key: str | None = None + firecrawl_api_key: str | None = None + web_search_provider: str = "exa" voyage_api_key: str | None = None max_depth: int = 4 max_steps_per_call: int = 100 @@ -45,6 +125,11 @@ class AgentConfig: session_root_dir: str = ".openplanter" max_persisted_observations: int = 400 max_solve_seconds: int = 0 + rate_limit_max_retries: int = 12 + zai_stream_max_retries: int = 10 + rate_limit_backoff_base_sec: float = 1.0 + rate_limit_backoff_max_sec: float = 60.0 + rate_limit_retry_after_cap_sec: float = 120.0 recursive: bool = True min_subtask_depth: int = 0 acceptance_criteria: bool = True @@ -52,6 +137,13 @@ class AgentConfig: max_turn_summaries: int = 50 demo: bool = False + def __post_init__(self) -> None: + self.openai_api_key = resolve_openai_api_key(self.openai_api_key, self.openai_base_url) + self.anthropic_api_key = resolve_anthropic_api_key( + self.anthropic_api_key, self.anthropic_base_url + ) + self.api_key = resolve_openai_api_key(self.api_key, self.base_url) + @classmethod def from_env(cls, workspace: str | Path) -> "AgentConfig": ws = Path(workspace).expanduser().resolve() @@ -62,30 +154,61 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": anthropic_api_key = os.getenv("OPENPLANTER_ANTHROPIC_API_KEY") or os.getenv("ANTHROPIC_API_KEY") openrouter_api_key = os.getenv("OPENPLANTER_OPENROUTER_API_KEY") or os.getenv("OPENROUTER_API_KEY") cerebras_api_key = os.getenv("OPENPLANTER_CEREBRAS_API_KEY") or os.getenv("CEREBRAS_API_KEY") + zai_api_key = os.getenv("OPENPLANTER_ZAI_API_KEY") or os.getenv("ZAI_API_KEY") exa_api_key = os.getenv("OPENPLANTER_EXA_API_KEY") or os.getenv("EXA_API_KEY") + firecrawl_api_key = os.getenv("OPENPLANTER_FIRECRAWL_API_KEY") or os.getenv("FIRECRAWL_API_KEY") voyage_api_key = os.getenv("OPENPLANTER_VOYAGE_API_KEY") or os.getenv("VOYAGE_API_KEY") openai_base_url = os.getenv("OPENPLANTER_OPENAI_BASE_URL") or os.getenv( "OPENPLANTER_BASE_URL", - "https://api.openai.com/v1", + FOUNDRY_OPENAI_BASE_URL, + ) + anthropic_base_url = os.getenv( + "OPENPLANTER_ANTHROPIC_BASE_URL", + FOUNDRY_ANTHROPIC_BASE_URL, + ) + openai_api_key = resolve_openai_api_key(openai_api_key, openai_base_url) + anthropic_api_key = resolve_anthropic_api_key(anthropic_api_key, anthropic_base_url) + zai_plan = normalize_zai_plan(os.getenv("OPENPLANTER_ZAI_PLAN", "paygo")) + zai_paygo_base_url = os.getenv("OPENPLANTER_ZAI_PAYGO_BASE_URL", ZAI_PAYGO_BASE_URL) + zai_coding_base_url = os.getenv("OPENPLANTER_ZAI_CODING_BASE_URL", ZAI_CODING_BASE_URL) + zai_base_url_override = (os.getenv("OPENPLANTER_ZAI_BASE_URL", "") or "").strip() + zai_base_url = ( + zai_base_url_override + or resolve_zai_base_url( + zai_plan, + paygo_base_url=zai_paygo_base_url, + coding_base_url=zai_coding_base_url, + ) ) + web_search_provider = (os.getenv("OPENPLANTER_WEB_SEARCH_PROVIDER", "exa").strip().lower() or "exa") + if web_search_provider not in {"exa", "firecrawl"}: + web_search_provider = "exa" return cls( workspace=ws, provider=os.getenv("OPENPLANTER_PROVIDER", "auto").strip().lower() or "auto", - model=os.getenv("OPENPLANTER_MODEL", "claude-opus-4-6"), + model=os.getenv("OPENPLANTER_MODEL", PROVIDER_DEFAULT_MODELS["anthropic"]), reasoning_effort=(os.getenv("OPENPLANTER_REASONING_EFFORT", "high").strip().lower() or None), base_url=openai_base_url, api_key=openai_api_key, openai_base_url=openai_base_url, - anthropic_base_url=os.getenv("OPENPLANTER_ANTHROPIC_BASE_URL", "https://api.anthropic.com/v1"), + anthropic_base_url=anthropic_base_url, openrouter_base_url=os.getenv("OPENPLANTER_OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1"), cerebras_base_url=os.getenv("OPENPLANTER_CEREBRAS_BASE_URL", "https://api.cerebras.ai/v1"), + zai_plan=zai_plan, + zai_paygo_base_url=zai_paygo_base_url, + zai_coding_base_url=zai_coding_base_url, + zai_base_url=zai_base_url, ollama_base_url=os.getenv("OPENPLANTER_OLLAMA_BASE_URL", "http://localhost:11434/v1"), exa_base_url=os.getenv("OPENPLANTER_EXA_BASE_URL", "https://api.exa.ai"), + firecrawl_base_url=os.getenv("OPENPLANTER_FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v1"), openai_api_key=openai_api_key, anthropic_api_key=anthropic_api_key, openrouter_api_key=openrouter_api_key, cerebras_api_key=cerebras_api_key, + zai_api_key=zai_api_key, exa_api_key=exa_api_key, + firecrawl_api_key=firecrawl_api_key, + web_search_provider=web_search_provider, voyage_api_key=voyage_api_key, max_depth=int(os.getenv("OPENPLANTER_MAX_DEPTH", "4")), max_steps_per_call=int(os.getenv("OPENPLANTER_MAX_STEPS", "100")), @@ -99,6 +222,11 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": session_root_dir=os.getenv("OPENPLANTER_SESSION_DIR", ".openplanter"), max_persisted_observations=int(os.getenv("OPENPLANTER_MAX_PERSISTED_OBS", "400")), max_solve_seconds=int(os.getenv("OPENPLANTER_MAX_SOLVE_SECONDS", "0")), + rate_limit_max_retries=int(os.getenv("OPENPLANTER_RATE_LIMIT_MAX_RETRIES", "12")), + zai_stream_max_retries=int(os.getenv("OPENPLANTER_ZAI_STREAM_MAX_RETRIES", "10")), + rate_limit_backoff_base_sec=float(os.getenv("OPENPLANTER_RATE_LIMIT_BACKOFF_BASE_SEC", "1.0")), + rate_limit_backoff_max_sec=float(os.getenv("OPENPLANTER_RATE_LIMIT_BACKOFF_MAX_SEC", "60.0")), + rate_limit_retry_after_cap_sec=float(os.getenv("OPENPLANTER_RATE_LIMIT_RETRY_AFTER_CAP_SEC", "120.0")), recursive=os.getenv("OPENPLANTER_RECURSIVE", "true").strip().lower() in ("1", "true", "yes"), min_subtask_depth=int(os.getenv("OPENPLANTER_MIN_SUBTASK_DEPTH", "0")), acceptance_criteria=os.getenv("OPENPLANTER_ACCEPTANCE_CRITERIA", "true").strip().lower() in ("1", "true", "yes"), diff --git a/agent/credentials.py b/agent/credentials.py index 3a387a59..275a8106 100644 --- a/agent/credentials.py +++ b/agent/credentials.py @@ -15,7 +15,9 @@ class CredentialBundle: anthropic_api_key: str | None = None openrouter_api_key: str | None = None cerebras_api_key: str | None = None + zai_api_key: str | None = None exa_api_key: str | None = None + firecrawl_api_key: str | None = None voyage_api_key: str | None = None def has_any(self) -> bool: @@ -24,7 +26,9 @@ def has_any(self) -> bool: or (self.anthropic_api_key and self.anthropic_api_key.strip()) or (self.openrouter_api_key and self.openrouter_api_key.strip()) or (self.cerebras_api_key and self.cerebras_api_key.strip()) + or (self.zai_api_key and self.zai_api_key.strip()) or (self.exa_api_key and self.exa_api_key.strip()) + or (self.firecrawl_api_key and self.firecrawl_api_key.strip()) or (self.voyage_api_key and self.voyage_api_key.strip()) ) @@ -37,8 +41,12 @@ def merge_missing(self, other: "CredentialBundle") -> None: self.openrouter_api_key = other.openrouter_api_key if not self.cerebras_api_key and other.cerebras_api_key: self.cerebras_api_key = other.cerebras_api_key + if not self.zai_api_key and other.zai_api_key: + self.zai_api_key = other.zai_api_key if not self.exa_api_key and other.exa_api_key: self.exa_api_key = other.exa_api_key + if not self.firecrawl_api_key and other.firecrawl_api_key: + self.firecrawl_api_key = other.firecrawl_api_key if not self.voyage_api_key and other.voyage_api_key: self.voyage_api_key = other.voyage_api_key @@ -52,8 +60,12 @@ def to_json(self) -> dict[str, str]: out["openrouter_api_key"] = self.openrouter_api_key if self.cerebras_api_key: out["cerebras_api_key"] = self.cerebras_api_key + if self.zai_api_key: + out["zai_api_key"] = self.zai_api_key if self.exa_api_key: out["exa_api_key"] = self.exa_api_key + if self.firecrawl_api_key: + out["firecrawl_api_key"] = self.firecrawl_api_key if self.voyage_api_key: out["voyage_api_key"] = self.voyage_api_key return out @@ -67,7 +79,9 @@ def from_json(cls, payload: dict[str, str] | None) -> "CredentialBundle": anthropic_api_key=(payload.get("anthropic_api_key") or "").strip() or None, openrouter_api_key=(payload.get("openrouter_api_key") or "").strip() or None, cerebras_api_key=(payload.get("cerebras_api_key") or "").strip() or None, + zai_api_key=(payload.get("zai_api_key") or "").strip() or None, exa_api_key=(payload.get("exa_api_key") or "").strip() or None, + firecrawl_api_key=(payload.get("firecrawl_api_key") or "").strip() or None, voyage_api_key=(payload.get("voyage_api_key") or "").strip() or None, ) @@ -109,7 +123,10 @@ def parse_env_file(path: Path) -> CredentialBundle: or None, cerebras_api_key=(env.get("CEREBRAS_API_KEY") or env.get("OPENPLANTER_CEREBRAS_API_KEY") or "").strip() or None, + zai_api_key=(env.get("ZAI_API_KEY") or env.get("OPENPLANTER_ZAI_API_KEY") or "").strip() or None, exa_api_key=(env.get("EXA_API_KEY") or env.get("OPENPLANTER_EXA_API_KEY") or "").strip() or None, + firecrawl_api_key=(env.get("FIRECRAWL_API_KEY") or env.get("OPENPLANTER_FIRECRAWL_API_KEY") or "").strip() + or None, voyage_api_key=(env.get("VOYAGE_API_KEY") or env.get("OPENPLANTER_VOYAGE_API_KEY") or "").strip() or None, ) @@ -134,7 +151,15 @@ def credentials_from_env() -> CredentialBundle: os.getenv("OPENPLANTER_CEREBRAS_API_KEY") or os.getenv("CEREBRAS_API_KEY") or "" ).strip() or None, + zai_api_key=( + os.getenv("OPENPLANTER_ZAI_API_KEY") or os.getenv("ZAI_API_KEY") or "" + ).strip() + or None, exa_api_key=(os.getenv("OPENPLANTER_EXA_API_KEY") or os.getenv("EXA_API_KEY") or "").strip() or None, + firecrawl_api_key=( + os.getenv("OPENPLANTER_FIRECRAWL_API_KEY") or os.getenv("FIRECRAWL_API_KEY") or "" + ).strip() + or None, voyage_api_key=(os.getenv("OPENPLANTER_VOYAGE_API_KEY") or os.getenv("VOYAGE_API_KEY") or "").strip() or None, ) @@ -229,7 +254,9 @@ def prompt_for_credentials( anthropic_api_key=existing.anthropic_api_key, openrouter_api_key=existing.openrouter_api_key, cerebras_api_key=existing.cerebras_api_key, + zai_api_key=existing.zai_api_key, exa_api_key=existing.exa_api_key, + firecrawl_api_key=existing.firecrawl_api_key, voyage_api_key=existing.voyage_api_key, ) @@ -262,7 +289,9 @@ def _ask(label: str, existing_value: str | None) -> str | None: current.anthropic_api_key = _ask("Anthropic", current.anthropic_api_key) current.openrouter_api_key = _ask("OpenRouter", current.openrouter_api_key) current.cerebras_api_key = _ask("Cerebras", current.cerebras_api_key) + current.zai_api_key = _ask("Z.AI", current.zai_api_key) current.exa_api_key = _ask("Exa", current.exa_api_key) + current.firecrawl_api_key = _ask("Firecrawl", current.firecrawl_api_key) current.voyage_api_key = _ask("Voyage", current.voyage_api_key) if not force and current.has_any() and not existing.has_any(): changed = True diff --git a/agent/engine.py b/agent/engine.py index 06c526ca..422dbf99 100644 --- a/agent/engine.py +++ b/agent/engine.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import random import re import time import threading @@ -12,7 +13,7 @@ from typing import Any, Callable from .config import AgentConfig -from .model import BaseModel, ImageData, ModelError, ModelTurn, ToolCall, ToolResult +from .model import BaseModel, ImageData, ModelError, ModelTurn, RateLimitError, ToolCall, ToolResult from .prompts import build_system_prompt from .replay_log import ReplayLogger from .tool_defs import get_tool_definitions @@ -122,15 +123,16 @@ def summary(self, max_items: int = 12, max_chars: int = 8000) -> str: @dataclass class TurnSummary: - """Lightweight summary of a completed agent turn for session continuity.""" + """Compact, serializable summary for a completed top-level turn.""" + turn_number: int objective: str - result_preview: str # first ~200 chars - timestamp: str # ISO 8601 UTC + result_preview: str + timestamp: str steps_used: int = 0 replay_seq_start: int = 0 - def to_dict(self) -> dict[str, Any]: + def to_dict(self) -> dict[str, int | str]: return { "turn_number": self.turn_number, "objective": self.objective, @@ -141,14 +143,14 @@ def to_dict(self) -> dict[str, Any]: } @classmethod - def from_dict(cls, d: dict[str, Any]) -> "TurnSummary": + def from_dict(cls, payload: dict[str, object]) -> "TurnSummary": return cls( - turn_number=d["turn_number"], - objective=d["objective"], - result_preview=d["result_preview"], - timestamp=d["timestamp"], - steps_used=d.get("steps_used", 0), - replay_seq_start=d.get("replay_seq_start", 0), + turn_number=int(payload["turn_number"]), + objective=str(payload.get("objective", "")), + result_preview=str(payload.get("result_preview", "")), + timestamp=str(payload.get("timestamp", "")), + steps_used=int(payload.get("steps_used", 0) or 0), + replay_seq_start=int(payload.get("replay_seq_start", 0) or 0), ) @@ -377,7 +379,40 @@ def _solve_recursive( if on_content_delta and depth == 0 and hasattr(model, "on_content_delta"): model.on_content_delta = on_content_delta try: - turn = model.complete(conversation) + rate_limit_retries = 0 + while True: + if self._cancel.is_set(): + self._emit(f"[d{depth}] cancelled by user", on_event) + return "Task cancelled." + try: + turn = model.complete(conversation) + break + except RateLimitError as exc: + if rate_limit_retries >= self.config.rate_limit_max_retries: + self._emit(f"[d{depth}/s{step}] model error: {exc}", on_event) + return f"Model error at depth {depth}, step {step}: {exc}" + rate_limit_retries += 1 + delay: float | None = None + if exc.retry_after_sec is not None: + delay = min( + max(exc.retry_after_sec, 0.0), + self.config.rate_limit_retry_after_cap_sec, + ) + if delay is None: + delay = self.config.rate_limit_backoff_base_sec * (2 ** (rate_limit_retries - 1)) + delay += random.uniform(0.0, 0.25) + delay = min(delay, self.config.rate_limit_backoff_max_sec) + if deadline and (time.monotonic() + delay) > deadline: + self._emit(f"[d{depth}] wall-clock limit reached", on_event) + return "Time limit exceeded. Try a more focused objective." + provider_code = f" ({exc.provider_code})" if exc.provider_code is not None else "" + self._emit( + f"[d{depth}/s{step}] rate limited{provider_code}. " + f"Sleeping {delay:.1f}s before retry {rate_limit_retries}/{self.config.rate_limit_max_retries}...", + on_event, + ) + if delay > 0: + time.sleep(delay) except ModelError as exc: self._emit(f"[d{depth}/s{step}] model error: {exc}", on_event) return f"Model error at depth {depth}, step {step}: {exc}" diff --git a/agent/model.py b/agent/model.py index 30bc3ff7..45fca294 100644 --- a/agent/model.py +++ b/agent/model.py @@ -6,8 +6,10 @@ import urllib.request from dataclasses import dataclass, field from datetime import datetime, timezone +from email.utils import parsedate_to_datetime from typing import Any, Callable, Protocol +from .config import strip_foundry_model_prefix from .tool_defs import TOOL_DEFINITIONS, to_anthropic_tools, to_openai_tools @@ -15,6 +17,27 @@ class ModelError(RuntimeError): pass +class HTTPModelError(ModelError): + def __init__( + self, + message: str, + *, + status_code: int | None = None, + provider_code: str | int | None = None, + body: str = "", + retry_after_sec: float | None = None, + ) -> None: + super().__init__(message) + self.status_code = status_code + self.provider_code = provider_code + self.body = body + self.retry_after_sec = retry_after_sec + + +class RateLimitError(HTTPModelError): + pass + + # --------------------------------------------------------------------------- # Core data types # --------------------------------------------------------------------------- @@ -103,6 +126,130 @@ def _extract_content(content: object) -> str: return "" +def _parse_json_object(text: str) -> dict[str, Any] | None: + try: + parsed = json.loads(text) + except json.JSONDecodeError: + return None + if isinstance(parsed, dict): + return parsed + return None + + +def _parse_retry_after_value(value: object) -> float | None: + if value is None: + return None + if isinstance(value, (int, float)): + return max(float(value), 0.0) + if isinstance(value, str): + text = value.strip() + if not text: + return None + try: + return max(float(text), 0.0) + except ValueError: + pass + try: + dt = parsedate_to_datetime(text) + except (TypeError, ValueError, IndexError): + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return max((dt - datetime.now(timezone.utc)).total_seconds(), 0.0) + return None + + +def _parse_retry_after(headers: Any) -> float | None: + if headers is None: + return None + getter = getattr(headers, "get", None) + if not callable(getter): + return None + return _parse_retry_after_value(getter("Retry-After")) + + +def _extract_openai_style_error(payload: dict[str, Any]) -> tuple[str, str | int | None, float | None]: + error = payload.get("error") + if isinstance(error, dict): + message = str(error.get("message", "")).strip() + provider_code = error.get("code") + retry_after = _parse_retry_after_value(error.get("retry_after")) + if retry_after is None: + retry_after = _parse_retry_after_value(payload.get("retry_after")) + return message, provider_code, retry_after + return "", None, _parse_retry_after_value(payload.get("retry_after")) + + +def _is_rate_limit_error( + status_code: int | None, + provider_code: str | int | None, + message: str, +) -> bool: + if status_code == 429: + return True + if provider_code is not None: + code_text = str(provider_code).strip().lower() + if code_text in {"1302", "429", "rate_limit", "rate_limit_exceeded", "too_many_requests"}: + return True + lower = message.lower() + return "rate limit" in lower or "too many requests" in lower + + +def _raise_http_error(url: str, status_code: int, body: str, headers: Any) -> None: + parsed = _parse_json_object(body) + message = "" + provider_code: str | int | None = None + body_retry_after: float | None = None + if parsed is not None: + message, provider_code, body_retry_after = _extract_openai_style_error(parsed) + retry_after = _parse_retry_after(headers) + if retry_after is None: + retry_after = body_retry_after + text = message or body + exc_cls = RateLimitError if _is_rate_limit_error(status_code, provider_code, text) else HTTPModelError + raise exc_cls( + f"HTTP {status_code} calling {url}: {body}", + status_code=status_code, + provider_code=provider_code, + body=body, + retry_after_sec=retry_after, + ) + + +def _raise_sse_error(data_dict: dict[str, Any]) -> None: + if data_dict.get("type") == "error": + err = data_dict.get("error") + if isinstance(err, dict): + err_msg = str(err.get("message", str(data_dict))) + provider_code = err.get("code") + retry_after = _parse_retry_after_value(err.get("retry_after")) + if _is_rate_limit_error(None, provider_code, err_msg): + raise RateLimitError( + f"Stream error: {err_msg}", + status_code=None, + provider_code=provider_code, + body=json.dumps(data_dict, ensure_ascii=True), + retry_after_sec=retry_after, + ) + raise ModelError(f"Stream error: {err_msg}") + raise ModelError(f"Stream error: {data_dict}") + + err = data_dict.get("error") + if isinstance(err, dict): + err_msg = str(err.get("message", str(data_dict))) + provider_code = err.get("code") + retry_after = _parse_retry_after_value(err.get("retry_after")) + if _is_rate_limit_error(None, provider_code, err_msg): + raise RateLimitError( + f"Stream error: {err_msg}", + status_code=None, + provider_code=provider_code, + body=json.dumps(data_dict, ensure_ascii=True), + retry_after_sec=retry_after, + ) + raise ModelError(f"Stream error: {err_msg}") + + def _http_json( url: str, method: str, @@ -121,7 +268,10 @@ def _http_json( raw = resp.read().decode("utf-8", errors="replace") except urllib.error.HTTPError as exc: # pragma: no cover - network path body = exc.read().decode("utf-8", errors="replace") - raise ModelError(f"HTTP {exc.code} calling {url}: {body}") from exc + try: + _raise_http_error(url, exc.code, body, exc.headers) + except ModelError as model_exc: + raise model_exc from exc except urllib.error.URLError as exc: # pragma: no cover - network path raise ModelError(f"Connection error calling {url}: {exc}") from exc except OSError as exc: # pragma: no cover - bare socket.timeout, etc. @@ -176,10 +326,7 @@ def _read_sse_events( except json.JSONDecodeError: data_dict = {"_raw": joined} if isinstance(data_dict, dict): - # Check for Anthropic error events - if data_dict.get("type") == "error": - err_msg = data_dict.get("error", {}).get("message", str(data_dict)) - raise ModelError(f"Stream error: {err_msg}") + _raise_sse_error(data_dict) events.append((current_event, data_dict)) if on_sse_event: try: @@ -198,9 +345,7 @@ def _read_sse_events( except json.JSONDecodeError: data_dict = {"_raw": joined} if isinstance(data_dict, dict): - if data_dict.get("type") == "error": - err_msg = data_dict.get("error", {}).get("message", str(data_dict)) - raise ModelError(f"Stream error: {err_msg}") + _raise_sse_error(data_dict) events.append((current_event, data_dict)) if on_sse_event: try: @@ -231,7 +376,10 @@ def _http_stream_sse( resp = urllib.request.urlopen(req, timeout=first_byte_timeout) except urllib.error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") - raise ModelError(f"HTTP {exc.code} calling {url}: {body}") from exc + try: + _raise_http_error(url, exc.code, body, exc.headers) + except ModelError as model_exc: + raise model_exc from exc except (socket.timeout, urllib.error.URLError, OSError) as exc: # Timeout or connection error — retry last_exc = exc @@ -254,6 +402,7 @@ def _accumulate_openai_stream( ) -> dict[str, Any]: """Reconstruct an OpenAI non-streaming response dict from SSE delta chunks.""" text_parts: list[str] = [] + reasoning_parts: list[str] = [] tool_calls_by_index: dict[int, dict[str, Any]] = {} finish_reason = "" usage: dict[str, Any] = {} @@ -279,6 +428,9 @@ def _accumulate_openai_stream( content = delta.get("content") if content: text_parts.append(content) + reasoning = delta.get("reasoning_content") + if isinstance(reasoning, str) and reasoning: + reasoning_parts.append(reasoning) # Tool calls (streamed incrementally) tc_deltas = delta.get("tool_calls") @@ -305,6 +457,8 @@ def _accumulate_openai_stream( "role": "assistant", "content": "".join(text_parts) if text_parts else None, } + if reasoning_parts: + message["reasoning_content"] = "".join(reasoning_parts) if tool_calls_by_index: message["tool_calls"] = [ tool_calls_by_index[i] for i in sorted(tool_calls_by_index) @@ -633,11 +787,17 @@ class OpenAICompatibleModel: first_byte_timeout: float = 10 strict_tools: bool = True tool_defs: list[dict[str, Any]] | None = None + thinking_type: str | None = None on_content_delta: Callable[[str, str], None] | None = None + provider: str | None = None + stream_max_retries: int = 3 + + def _request_model_name(self) -> str: + return strip_foundry_model_prefix(self.model) def _is_reasoning_model(self) -> bool: """OpenAI reasoning models (o-series, gpt-5 series) have different API constraints.""" - lower = self.model.lower() + lower = self._request_model_name().lower() if ( lower.startswith("o1-") or lower == "o1" or lower.startswith("o3-") or lower == "o3" @@ -660,7 +820,7 @@ def complete(self, conversation: Conversation) -> ModelTurn: is_reasoning = self._is_reasoning_model() payload: dict[str, Any] = { - "model": self.model, + "model": self._request_model_name(), "messages": conversation._provider_messages, "tools": to_openai_tools(defs=self.tool_defs, strict=self.strict_tools), "tool_choice": "auto", @@ -680,8 +840,10 @@ def complete(self, conversation: Conversation) -> ModelTurn: effort = (self.reasoning_effort or "").strip().lower() if effort: payload["reasoning_effort"] = effort + thinking_type = (self.thinking_type or "").strip().lower() + if thinking_type in {"enabled", "disabled"}: + payload["thinking"] = {"type": thinking_type} - url = self.base_url.rstrip("/") + "/chat/completions" headers = { "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", @@ -702,6 +864,15 @@ def _forward_delta(_event_type: str, data: dict[str, Any]) -> None: content = delta.get("content") if content: cb("text", content) + reasoning_content = delta.get("reasoning_content") + if isinstance(reasoning_content, str) and reasoning_content: + cb("thinking", reasoning_content) + reasoning = delta.get("reasoning") + if isinstance(reasoning, str) and reasoning: + cb("thinking", reasoning) + thinking = delta.get("thinking") + if isinstance(thinking, str) and thinking: + cb("thinking", thinking) # Forward tool call argument deltas for live preview tc_deltas = delta.get("tool_calls") if tc_deltas: @@ -716,17 +887,21 @@ def _forward_delta(_event_type: str, data: dict[str, Any]) -> None: sse_cb = _forward_delta if self.on_content_delta else None - try: + def _request_stream(active_payload: dict[str, Any], active_base_url: str) -> dict[str, Any]: events = _http_stream_sse( - url=url, + url=active_base_url.rstrip("/") + "/chat/completions", method="POST", headers=headers, - payload=payload, + payload=active_payload, first_byte_timeout=self.first_byte_timeout, stream_timeout=self.timeout_sec, + max_retries=self.stream_max_retries, on_sse_event=sse_cb, ) - parsed = _accumulate_openai_stream(events) + return _accumulate_openai_stream(events) + + try: + parsed = _request_stream(payload, self.base_url) except ModelError as exc: text = str(exc).lower() unsupported_reasoning = effort and ( @@ -737,16 +912,7 @@ def _forward_delta(_event_type: str, data: dict[str, Any]) -> None: raise payload = dict(payload) payload.pop("reasoning_effort", None) - events = _http_stream_sse( - url=url, - method="POST", - headers=headers, - payload=payload, - first_byte_timeout=self.first_byte_timeout, - stream_timeout=self.timeout_sec, - on_sse_event=sse_cb, - ) - parsed = _accumulate_openai_stream(events) + parsed = _request_stream(payload, self.base_url) try: message = parsed["choices"][0]["message"] @@ -754,6 +920,13 @@ def _forward_delta(_event_type: str, data: dict[str, Any]) -> None: raise ModelError(f"Model response missing content: {parsed}") from exc finish_reason = parsed["choices"][0].get("finish_reason", "") + if finish_reason == "rate_limit": + raise RateLimitError( + "Model finish_reason=rate_limit", + status_code=429, + provider_code="rate_limit", + body=json.dumps(parsed, ensure_ascii=True), + ) # Parse tool calls raw_tool_calls = message.get("tool_calls") @@ -859,6 +1032,9 @@ class AnthropicModel: tool_defs: list[dict[str, Any]] | None = None on_content_delta: Callable[[str, str], None] | None = None + def _request_model_name(self) -> str: + return strip_foundry_model_prefix(self.model) + def create_conversation(self, system_prompt: str, initial_user_message: str) -> Conversation: messages: list[Any] = [ {"role": "user", "content": initial_user_message}, @@ -866,14 +1042,15 @@ def create_conversation(self, system_prompt: str, initial_user_message: str) -> return Conversation(_provider_messages=messages, system_prompt=system_prompt) def _is_opus_46(self) -> bool: - return "opus-4-6" in self.model.lower() or "opus-4.6" in self.model.lower() + lower = self._request_model_name().lower() + return "opus-4-6" in lower or "opus-4.6" in lower def complete(self, conversation: Conversation) -> ModelTurn: effort = (self.reasoning_effort or "").strip().lower() use_thinking = effort in {"low", "medium", "high"} payload: dict[str, Any] = { - "model": self.model, + "model": self._request_model_name(), "max_tokens": self.max_tokens, "messages": conversation._provider_messages, "tools": to_anthropic_tools(defs=self.tool_defs), diff --git a/agent/settings.py b/agent/settings.py index ec2835ee..5b3b4f97 100644 --- a/agent/settings.py +++ b/agent/settings.py @@ -30,6 +30,7 @@ class PersistentSettings: default_model_anthropic: str | None = None default_model_openrouter: str | None = None default_model_cerebras: str | None = None + default_model_zai: str | None = None default_model_ollama: str | None = None def default_model_for_provider(self, provider: str) -> str | None: @@ -38,6 +39,7 @@ def default_model_for_provider(self, provider: str) -> str | None: "anthropic": self.default_model_anthropic, "openrouter": self.default_model_openrouter, "cerebras": self.default_model_cerebras, + "zai": self.default_model_zai, "ollama": self.default_model_ollama, } specific = per_provider.get(provider) @@ -55,6 +57,7 @@ def normalized(self) -> "PersistentSettings": default_model_anthropic=(self.default_model_anthropic or "").strip() or None, default_model_openrouter=(self.default_model_openrouter or "").strip() or None, default_model_cerebras=(self.default_model_cerebras or "").strip() or None, + default_model_zai=(self.default_model_zai or "").strip() or None, default_model_ollama=(self.default_model_ollama or "").strip() or None, ) @@ -72,6 +75,8 @@ def to_json(self) -> dict[str, str]: payload["default_model_openrouter"] = self.default_model_openrouter if self.default_model_cerebras: payload["default_model_cerebras"] = self.default_model_cerebras + if self.default_model_zai: + payload["default_model_zai"] = self.default_model_zai if self.default_model_ollama: payload["default_model_ollama"] = self.default_model_ollama return payload @@ -89,6 +94,7 @@ def from_json(cls, payload: dict | None) -> "PersistentSettings": default_model_anthropic=(str(payload.get("default_model_anthropic", "")).strip() or None), default_model_openrouter=(str(payload.get("default_model_openrouter", "")).strip() or None), default_model_cerebras=(str(payload.get("default_model_cerebras", "")).strip() or None), + default_model_zai=(str(payload.get("default_model_zai", "")).strip() or None), default_model_ollama=(str(payload.get("default_model_ollama", "")).strip() or None), ).normalized() diff --git a/agent/tool_defs.py b/agent/tool_defs.py index 323edbde..79bdb496 100644 --- a/agent/tool_defs.py +++ b/agent/tool_defs.py @@ -63,7 +63,7 @@ }, { "name": "web_search", - "description": "Search the web using the Exa API. Returns URLs, titles, and optional page text.", + "description": "Search the web using the configured provider (Exa or Firecrawl). Returns URLs, titles, and optional page text.", "parameters": { "type": "object", "properties": { diff --git a/agent/tools.py b/agent/tools.py index 86a9e5ce..a9d6d4ef 100644 --- a/agent/tools.py +++ b/agent/tools.py @@ -54,8 +54,11 @@ class WorkspaceTools: max_file_chars: int = 20000 max_files_listed: int = 400 max_search_hits: int = 200 + web_search_provider: str = "exa" exa_api_key: str | None = None exa_base_url: str = "https://api.exa.ai" + firecrawl_api_key: str | None = None + firecrawl_base_url: str = "https://api.firecrawl.dev/v1" def __post_init__(self) -> None: self.root = self.root.expanduser().resolve() @@ -804,6 +807,38 @@ def _exa_request(self, endpoint: str, payload: dict[str, Any]) -> dict[str, Any] raise ToolError(f"Exa API returned non-object response: {type(parsed)!r}") return parsed + def _firecrawl_request(self, endpoint: str, payload: dict[str, Any]) -> dict[str, Any]: + if not (self.firecrawl_api_key and self.firecrawl_api_key.strip()): + raise ToolError("FIRECRAWL_API_KEY not configured") + url = self.firecrawl_base_url.rstrip("/") + endpoint + req = urllib.request.Request( + url=url, + data=json.dumps(payload).encode("utf-8"), + headers={ + "Authorization": f"Bearer {self.firecrawl_api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=self.command_timeout_sec) as resp: + raw = resp.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise ToolError(f"Firecrawl API HTTP {exc.code}: {body}") from exc + except urllib.error.URLError as exc: + raise ToolError(f"Firecrawl API connection error: {exc}") from exc + except OSError as exc: + raise ToolError(f"Firecrawl API network error: {exc}") from exc + + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise ToolError(f"Firecrawl API returned non-JSON payload: {raw[:500]}") from exc + if not isinstance(parsed, dict): + raise ToolError(f"Firecrawl API returned non-object response: {type(parsed)!r}") + return parsed + def web_search( self, query: str, @@ -814,6 +849,59 @@ def web_search( if not query: return "web_search requires non-empty query" clamped_results = max(1, min(int(num_results), 20)) + provider = (self.web_search_provider or "exa").strip().lower() + if provider not in {"exa", "firecrawl"}: + provider = "exa" + + if provider == "firecrawl": + payload: dict[str, Any] = { + "query": query, + "limit": clamped_results, + } + if include_text: + payload["scrapeOptions"] = {"formats": ["markdown"]} + + try: + parsed = self._firecrawl_request("/search", payload) + except Exception as exc: + return f"Web search failed: {exc}" + + data = parsed.get("data") + rows: list[Any] = [] + if isinstance(data, list): + rows = data + elif isinstance(data, dict): + web_rows = data.get("web") + if isinstance(web_rows, list): + rows = web_rows + + out_results: list[dict[str, Any]] = [] + for row in rows: + if not isinstance(row, dict): + continue + metadata = row.get("metadata") + meta_title = "" + if isinstance(metadata, dict): + meta_title = str(metadata.get("title", "")) + item: dict[str, Any] = { + "url": str(row.get("url", "")), + "title": str(row.get("title", "") or meta_title), + "snippet": str(row.get("description", "") or row.get("snippet", "")), + } + if include_text: + text_value = row.get("markdown") or row.get("text") or "" + if isinstance(text_value, str) and text_value: + item["text"] = self._clip(text_value, 4000) + out_results.append(item) + + output = { + "query": query, + "provider": provider, + "results": out_results, + "total": len(out_results), + } + return self._clip(json.dumps(output, indent=2, ensure_ascii=True), self.max_file_chars) + payload: dict[str, Any] = { "query": query, "numResults": clamped_results, @@ -841,6 +929,7 @@ def web_search( output = { "query": query, + "provider": provider, "results": out_results, "total": len(out_results), } @@ -859,6 +948,43 @@ def fetch_url(self, urls: list[str]) -> str: if not normalized: return "fetch_url requires at least one valid URL" normalized = normalized[:10] + provider = (self.web_search_provider or "exa").strip().lower() + if provider not in {"exa", "firecrawl"}: + provider = "exa" + + if provider == "firecrawl": + pages: list[dict[str, Any]] = [] + for url in normalized: + payload: dict[str, Any] = { + "url": url, + "formats": ["markdown"], + } + try: + parsed = self._firecrawl_request("/scrape", payload) + except Exception as exc: + return f"Fetch URL failed: {exc}" + data = parsed.get("data") + if not isinstance(data, dict): + continue + metadata = data.get("metadata") + title = "" + if isinstance(metadata, dict): + title = str(metadata.get("title", "")) + text = data.get("markdown") or data.get("text") or data.get("html") or "" + pages.append( + { + "url": str(data.get("url", "") or url), + "title": title, + "text": self._clip(str(text), 8000), + } + ) + output = { + "provider": provider, + "pages": pages, + "total": len(pages), + } + return self._clip(json.dumps(output, indent=2, ensure_ascii=True), self.max_file_chars) + payload: dict[str, Any] = { "ids": normalized, "text": {"maxCharacters": 8000}, @@ -881,6 +1007,7 @@ def fetch_url(self, urls: list[str]) -> str: ) output = { + "provider": provider, "pages": pages, "total": len(pages), } diff --git a/agent/tui.py b/agent/tui.py index 0d7184ec..c1a63be2 100644 --- a/agent/tui.py +++ b/agent/tui.py @@ -110,14 +110,15 @@ def _build_splash() -> str: ] MODEL_ALIASES: dict[str, str] = { - "opus": "claude-opus-4-6", - "opus4.6": "claude-opus-4-6", - "sonnet": "claude-sonnet-4-5-20250929", - "sonnet4.5": "claude-sonnet-4-5-20250929", - "haiku": "claude-haiku-4-5-20251001", - "haiku4.5": "claude-haiku-4-5-20251001", - "gpt5": "gpt-5.2", - "gpt5.2": "gpt-5.2", + "opus": "anthropic-foundry/claude-opus-4-6", + "opus4.6": "anthropic-foundry/claude-opus-4-6", + "sonnet": "anthropic-foundry/claude-sonnet-4-6", + "sonnet4.6": "anthropic-foundry/claude-sonnet-4-6", + "haiku": "anthropic-foundry/claude-haiku-4-5", + "haiku4.5": "anthropic-foundry/claude-haiku-4-5", + "gpt5": "azure-foundry/gpt-5.3-codex", + "gpt5.3": "azure-foundry/gpt-5.3-codex", + "kimi": "azure-foundry/Kimi-K2.5", "gpt4": "gpt-4.1", "gpt4.1": "gpt-4.1", "gpt4o": "gpt-4o", @@ -128,6 +129,8 @@ def _build_splash() -> str: "cerebras": "qwen-3-235b-a22b-instruct-2507", "qwen235b": "qwen-3-235b-a22b-instruct-2507", "oss120b": "gpt-oss-120b", + "glm5": "glm-5", + "zai": "glm-5", "llama": "llama3.2", "llama3": "llama3.2", "mistral": "mistral", @@ -176,6 +179,7 @@ def _api_key_for_provider(cfg: AgentConfig, provider: str) -> str | None: "anthropic": cfg.anthropic_api_key, "openrouter": cfg.openrouter_api_key, "cerebras": cfg.cerebras_api_key, + "zai": cfg.zai_api_key, "ollama": "ollama", }.get(provider) @@ -191,6 +195,8 @@ def _available_providers(cfg: AgentConfig) -> list[str]: providers.append("openrouter") if cfg.cerebras_api_key: providers.append("cerebras") + if cfg.zai_api_key: + providers.append("zai") providers.append("ollama") return providers @@ -220,7 +226,7 @@ def handle_model_command(args: str, ctx: ChatContext) -> list[str]: list_target = parts[1] if len(parts) > 1 else None if list_target == "all": providers = _available_providers(ctx.cfg) - elif list_target in {"openai", "anthropic", "openrouter", "cerebras", "ollama"}: + elif list_target in {"openai", "anthropic", "openrouter", "cerebras", "zai", "ollama"}: providers = [list_target] else: providers = [ctx.cfg.provider] @@ -280,6 +286,8 @@ def handle_model_command(args: str, ctx: ChatContext) -> list[str]: settings.default_model_openrouter = new_model elif provider == "cerebras": settings.default_model_cerebras = new_model + elif provider == "zai": + settings.default_model_zai = new_model elif provider == "ollama": settings.default_model_ollama = new_model else: diff --git a/openplanter-desktop/Cargo.lock b/openplanter-desktop/Cargo.lock index 39951ed9..503aecf9 100644 --- a/openplanter-desktop/Cargo.lock +++ b/openplanter-desktop/Cargo.lock @@ -2469,7 +2469,6 @@ dependencies = [ "tempfile", "tokio", "tokio-util", - "uuid", ] [[package]] diff --git a/openplanter-desktop/crates/op-core/src/builder.rs b/openplanter-desktop/crates/op-core/src/builder.rs index a0c4e319..1be274c7 100644 --- a/openplanter-desktop/crates/op-core/src/builder.rs +++ b/openplanter-desktop/crates/op-core/src/builder.rs @@ -7,10 +7,13 @@ use std::collections::HashMap; use regex::Regex; use std::sync::LazyLock; -use crate::config::{AgentConfig, PROVIDER_DEFAULT_MODELS}; +use crate::config::{ + ANTHROPIC_FOUNDRY_MODEL_PREFIX, AZURE_FOUNDRY_MODEL_PREFIX, AgentConfig, + PROVIDER_DEFAULT_MODELS, resolve_anthropic_api_key, resolve_openai_api_key, +}; use crate::model::BaseModel; -use crate::model::openai::OpenAIModel; use crate::model::anthropic::AnthropicModel; +use crate::model::openai::{OpenAIModel, ZaiRuntimeConfig}; /// Error type for model/builder operations. #[derive(Debug, thiserror::Error)] @@ -20,14 +23,16 @@ pub enum ModelError { } // Provider inference regexes — order matters (Cerebras `qwen-3` before Ollama `qwen`). -static ANTHROPIC_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)^claude").unwrap()); +static ANTHROPIC_RE: LazyLock = LazyLock::new(|| Regex::new(r"(?i)^claude").unwrap()); -static OPENAI_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)^(gpt|o[1-4]-|o[1-4]$|chatgpt|dall-e|tts-|whisper)").unwrap()); +static OPENAI_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(gpt|o[1-4]-|o[1-4]$|chatgpt|dall-e|tts-|whisper)").unwrap() +}); static CEREBRAS_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)^(llama.*cerebras|qwen-3|gpt-oss|zai-glm)").unwrap()); + LazyLock::new(|| Regex::new(r"(?i)^(llama.*cerebras|qwen-3|gpt-oss)").unwrap()); + +static ZAI_RE: LazyLock = LazyLock::new(|| Regex::new(r"(?i)^(glm|zai-glm)").unwrap()); // Ollama regex: `qwen` without lookahead — Cerebras check runs first, so // `qwen-3*` is already caught before we reach this regex. @@ -40,6 +45,13 @@ static OLLAMA_RE: LazyLock = LazyLock::new(|| { /// Infer the likely provider for a model name, or `None` if ambiguous. pub fn infer_provider_for_model(model: &str) -> Option<&'static str> { + let lowered = model.trim().to_lowercase(); + if lowered.starts_with(ANTHROPIC_FOUNDRY_MODEL_PREFIX) { + return Some("anthropic"); + } + if lowered.starts_with(AZURE_FOUNDRY_MODEL_PREFIX) { + return Some("openai"); + } if model.contains('/') { return Some("openrouter"); } @@ -49,6 +61,9 @@ pub fn infer_provider_for_model(model: &str) -> Option<&'static str> { if CEREBRAS_RE.is_match(model) { return Some("cerebras"); } + if ZAI_RE.is_match(model) { + return Some("zai"); + } if OPENAI_RE.is_match(model) { return Some("openai"); } @@ -86,12 +101,12 @@ pub fn resolve_model_name(cfg: &AgentConfig) -> Result { // For now, fall through to defaults. return Ok(PROVIDER_DEFAULT_MODELS .get(cfg.provider.as_str()) - .unwrap_or(&"claude-opus-4-6") + .unwrap_or(&"anthropic-foundry/claude-opus-4-6") .to_string()); } Ok(PROVIDER_DEFAULT_MODELS .get(cfg.provider.as_str()) - .unwrap_or(&"claude-opus-4-6") + .unwrap_or(&"anthropic-foundry/claude-opus-4-6") .to_string()) } @@ -117,6 +132,7 @@ pub fn resolve_provider(cfg: &AgentConfig) -> Result { ("openai", &cfg.openai_api_key), ("openrouter", &cfg.openrouter_api_key), ("cerebras", &cfg.cerebras_api_key), + ("zai", &cfg.zai_api_key), ("ollama", &None), // ollama is always last — no key needed ]; @@ -131,38 +147,34 @@ pub fn resolve_provider(cfg: &AgentConfig) -> Result { } /// Resolve the base URL and API key for the given provider. -pub fn resolve_endpoint( - cfg: &AgentConfig, - provider: &str, -) -> Result<(String, String), ModelError> { +pub fn resolve_endpoint(cfg: &AgentConfig, provider: &str) -> Result<(String, String), ModelError> { match provider { "anthropic" => { - let key = cfg - .anthropic_api_key - .as_deref() - .or(cfg.api_key.as_deref()) - .filter(|k| !k.is_empty()) - .ok_or_else(|| { + let key = resolve_anthropic_api_key( + cfg.anthropic_api_key.clone().or_else(|| cfg.api_key.clone()), + &cfg.anthropic_base_url, + ) + .ok_or_else(|| { ModelError::Message( "No Anthropic API key. Set ANTHROPIC_API_KEY or OPENPLANTER_ANTHROPIC_API_KEY.".into(), ) })?; // Anthropic base URL does NOT include /v1 suffix for /messages endpoint — // the model adapter appends /messages itself. The config stores it with /v1. - Ok((cfg.anthropic_base_url.clone(), key.to_string())) + Ok((cfg.anthropic_base_url.clone(), key)) } "openai" => { - let key = cfg - .openai_api_key - .as_deref() - .or(cfg.api_key.as_deref()) - .filter(|k| !k.is_empty()) - .ok_or_else(|| { + let key = resolve_openai_api_key( + cfg.openai_api_key.clone().or_else(|| cfg.api_key.clone()), + &cfg.openai_base_url, + ) + .ok_or_else(|| { ModelError::Message( - "No OpenAI API key. Set OPENAI_API_KEY or OPENPLANTER_OPENAI_API_KEY.".into(), + "No OpenAI API key. Set OPENAI_API_KEY or OPENPLANTER_OPENAI_API_KEY." + .into(), ) })?; - Ok((cfg.openai_base_url.clone(), key.to_string())) + Ok((cfg.openai_base_url.clone(), key)) } "openrouter" => { let key = cfg @@ -190,6 +202,19 @@ pub fn resolve_endpoint( })?; Ok((cfg.cerebras_base_url.clone(), key.to_string())) } + "zai" => { + let key = cfg + .zai_api_key + .as_deref() + .or(cfg.api_key.as_deref()) + .filter(|k| !k.is_empty()) + .ok_or_else(|| { + ModelError::Message( + "No Z.AI API key. Set ZAI_API_KEY or OPENPLANTER_ZAI_API_KEY.".into(), + ) + })?; + Ok((cfg.zai_base_url.clone(), key.to_string())) + } "ollama" => { // Ollama doesn't need a real key — use a dummy Ok((cfg.ollama_base_url.clone(), "ollama".to_string())) @@ -212,7 +237,7 @@ pub fn build_model(cfg: &AgentConfig) -> Result, ModelError> api_key, cfg.reasoning_effort.clone(), ))), - _ => { + "openai" | "openrouter" | "cerebras" | "zai" | "ollama" => { // OpenAI-compatible: openai, openrouter, cerebras, ollama let mut extra_headers = HashMap::new(); if provider == "openrouter" { @@ -222,15 +247,29 @@ pub fn build_model(cfg: &AgentConfig) -> Result, ModelError> ); extra_headers.insert("X-Title".to_string(), "OpenPlanter".to_string()); } - Ok(Box::new(OpenAIModel::new( + if provider == "zai" { + extra_headers.insert("Accept-Language".to_string(), "en-US,en".to_string()); + } + let model = OpenAIModel::new( model_name, - provider, + provider.clone(), base_url, api_key, cfg.reasoning_effort.clone(), extra_headers, - ))) + ); + let model = if provider == "zai" { + model.with_zai_runtime(ZaiRuntimeConfig { + paygo_base_url: cfg.zai_paygo_base_url.clone(), + coding_base_url: cfg.zai_coding_base_url.clone(), + stream_max_retries: cfg.zai_stream_max_retries.max(0) as usize, + }) + } else { + model + }; + Ok(Box::new(model)) } + _ => Err(ModelError::Message(format!("Unknown provider: {provider}"))), } } @@ -244,6 +283,10 @@ mod tests { infer_provider_for_model("claude-opus-4-6"), Some("anthropic") ); + assert_eq!( + infer_provider_for_model("anthropic-foundry/claude-opus-4-6"), + Some("anthropic") + ); assert_eq!( infer_provider_for_model("claude-sonnet-4-5"), Some("anthropic") @@ -257,6 +300,10 @@ mod tests { #[test] fn test_infer_openai() { assert_eq!(infer_provider_for_model("gpt-5.2"), Some("openai")); + assert_eq!( + infer_provider_for_model("azure-foundry/gpt-5.3-codex"), + Some("openai") + ); assert_eq!(infer_provider_for_model("o1-preview"), Some("openai")); assert_eq!(infer_provider_for_model("o3"), Some("openai")); assert_eq!(infer_provider_for_model("chatgpt-4o"), Some("openai")); @@ -282,6 +329,12 @@ mod tests { ); } + #[test] + fn test_infer_zai() { + assert_eq!(infer_provider_for_model("glm-5"), Some("zai")); + assert_eq!(infer_provider_for_model("zai-glm-4.6"), Some("zai")); + } + #[test] fn test_infer_ollama() { assert_eq!(infer_provider_for_model("llama3.2"), Some("ollama")); @@ -297,6 +350,7 @@ mod tests { // qwen-3 → cerebras, qwen (no -3) → ollama assert_eq!(infer_provider_for_model("qwen-3"), Some("cerebras")); assert_eq!(infer_provider_for_model("qwen2"), Some("ollama")); + assert_eq!(infer_provider_for_model("zai-glm"), Some("zai")); } #[test] @@ -326,11 +380,11 @@ mod tests { #[test] fn test_resolve_model_name_explicit() { let cfg = AgentConfig { - model: "gpt-5.2".into(), + model: "azure-foundry/gpt-5.3-codex".into(), provider: "openai".into(), ..Default::default() }; - assert_eq!(resolve_model_name(&cfg).unwrap(), "gpt-5.2"); + assert_eq!(resolve_model_name(&cfg).unwrap(), "azure-foundry/gpt-5.3-codex"); } #[test] @@ -340,7 +394,7 @@ mod tests { provider: "openai".into(), ..Default::default() }; - assert_eq!(resolve_model_name(&cfg).unwrap(), "gpt-5.2"); + assert_eq!(resolve_model_name(&cfg).unwrap(), "azure-foundry/gpt-5.3-codex"); } // ── resolve_provider ── @@ -358,7 +412,7 @@ mod tests { fn test_resolve_provider_auto_infers_from_model() { let cfg = AgentConfig { provider: "auto".into(), - model: "claude-opus-4-6".into(), + model: "anthropic-foundry/claude-opus-4-6".into(), ..Default::default() }; assert_eq!(resolve_provider(&cfg).unwrap(), "anthropic"); @@ -366,24 +420,40 @@ mod tests { #[test] fn test_resolve_provider_auto_falls_back_to_key() { + let cfg = AgentConfig { + provider: "auto".into(), + model: "some-unknown-model".into(), + zai_api_key: Some("zai-test".into()), + openai_api_key: None, + anthropic_api_key: None, + openrouter_api_key: None, + cerebras_api_key: None, + ..Default::default() + }; + assert_eq!(resolve_provider(&cfg).unwrap(), "zai"); + } + + #[test] + fn test_resolve_provider_auto_falls_back_to_openai_before_zai() { let cfg = AgentConfig { provider: "auto".into(), model: "some-unknown-model".into(), openai_api_key: Some("sk-test".into()), + anthropic_api_key: None, + zai_api_key: Some("zai-test".into()), ..Default::default() }; - // anthropic checked first but no key, openai has key assert_eq!(resolve_provider(&cfg).unwrap(), "openai"); } #[test] - fn test_resolve_provider_auto_no_keys_defaults_ollama() { + fn test_resolve_provider_auto_no_keys_defaults_to_foundry_anthropic() { let cfg = AgentConfig { provider: "auto".into(), model: "some-unknown-model".into(), ..Default::default() }; - assert_eq!(resolve_provider(&cfg).unwrap(), "ollama"); + assert_eq!(resolve_provider(&cfg).unwrap(), "anthropic"); } #[test] @@ -407,7 +477,7 @@ mod tests { ..Default::default() }; let (url, key) = resolve_endpoint(&cfg, "anthropic").unwrap(); - assert_eq!(url, "https://api.anthropic.com/v1"); + assert_eq!(url, crate::config::FOUNDRY_ANTHROPIC_BASE_URL); assert_eq!(key, "sk-ant-key"); } @@ -415,6 +485,8 @@ mod tests { fn test_resolve_endpoint_anthropic_fallback_to_api_key() { let cfg = AgentConfig { api_key: Some("fallback-key".into()), + anthropic_api_key: None, + anthropic_base_url: "https://api.anthropic.com/v1".into(), ..Default::default() }; let (_, key) = resolve_endpoint(&cfg, "anthropic").unwrap(); @@ -423,10 +495,20 @@ mod tests { #[test] fn test_resolve_endpoint_anthropic_missing_key() { - let cfg = AgentConfig::default(); + let cfg = AgentConfig { + anthropic_api_key: None, + api_key: None, + anthropic_base_url: "https://api.anthropic.com/v1".into(), + ..Default::default() + }; let result = resolve_endpoint(&cfg, "anthropic"); assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("Anthropic API key")); + assert!( + result + .unwrap_err() + .to_string() + .contains("Anthropic API key") + ); } #[test] @@ -436,10 +518,21 @@ mod tests { ..Default::default() }; let (url, key) = resolve_endpoint(&cfg, "openai").unwrap(); - assert_eq!(url, "https://api.openai.com/v1"); + assert_eq!(url, crate::config::FOUNDRY_OPENAI_BASE_URL); assert_eq!(key, "sk-openai"); } + #[test] + fn test_resolve_endpoint_zai() { + let cfg = AgentConfig { + zai_api_key: Some("zai-key".into()), + ..Default::default() + }; + let (url, key) = resolve_endpoint(&cfg, "zai").unwrap(); + assert_eq!(url, "https://api.z.ai/api/paas/v4"); + assert_eq!(key, "zai-key"); + } + #[test] fn test_resolve_endpoint_ollama_dummy_key() { let cfg = AgentConfig::default(); @@ -484,6 +577,19 @@ mod tests { assert_eq!(model.provider_name(), "openai"); } + #[test] + fn test_build_model_zai() { + let cfg = AgentConfig { + provider: "zai".into(), + model: "glm-5".into(), + zai_api_key: Some("zai-key".into()), + ..Default::default() + }; + let model = build_model(&cfg).unwrap(); + assert_eq!(model.model_name(), "glm-5"); + assert_eq!(model.provider_name(), "zai"); + } + #[test] fn test_build_model_ollama_no_key_needed() { let cfg = AgentConfig { @@ -514,7 +620,9 @@ mod tests { let cfg = AgentConfig { provider: "openai".into(), model: "gpt-4o".into(), - // No key set + openai_base_url: "https://api.openai.com/v1".into(), + openai_api_key: None, + api_key: None, ..Default::default() }; let result = build_model(&cfg); @@ -535,7 +643,10 @@ mod tests { Err(e) => e.to_string(), Ok(_) => panic!("expected error"), }; - assert!(err_msg.contains("openai"), "error should mention openai: {err_msg}"); + assert!( + err_msg.contains("openai"), + "error should mention openai: {err_msg}" + ); } #[test] diff --git a/openplanter-desktop/crates/op-core/src/config.rs b/openplanter-desktop/crates/op-core/src/config.rs index f6ff3039..06ff4c86 100644 --- a/openplanter-desktop/crates/op-core/src/config.rs +++ b/openplanter-desktop/crates/op-core/src/config.rs @@ -5,14 +5,28 @@ use std::sync::LazyLock; use serde::{Deserialize, Serialize}; +pub const AZURE_FOUNDRY_MODEL_PREFIX: &str = "azure-foundry/"; +pub const ANTHROPIC_FOUNDRY_MODEL_PREFIX: &str = "anthropic-foundry/"; +pub const FOUNDRY_OPENAI_BASE_URL: &str = + "https://foundry-proxy.cheetah-koi.ts.net/openai/v1"; +pub const FOUNDRY_ANTHROPIC_BASE_URL: &str = + "https://foundry-proxy.cheetah-koi.ts.net/anthropic/v1"; +pub const FOUNDRY_OPENAI_API_KEY_PLACEHOLDER: &str = + "dont-worry-this-key-will-be-auto-injected"; +pub const FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER: &str = + "dont-worry-it-will-be-injected"; +pub const ZAI_PAYGO_BASE_URL: &str = "https://api.z.ai/api/paas/v4"; +pub const ZAI_CODING_BASE_URL: &str = "https://api.z.ai/api/coding/paas/v4"; + /// Default model for each supported provider. pub static PROVIDER_DEFAULT_MODELS: LazyLock> = LazyLock::new(|| { HashMap::from([ - ("openai", "gpt-5.2"), - ("anthropic", "claude-opus-4-6"), + ("openai", "azure-foundry/gpt-5.3-codex"), + ("anthropic", "anthropic-foundry/claude-opus-4-6"), ("openrouter", "anthropic/claude-sonnet-4-5"), ("cerebras", "qwen-3-235b-a22b-instruct-2507"), + ("zai", "glm-5"), ("ollama", "llama3.2"), ]) }); @@ -32,6 +46,13 @@ fn env_int(key: &str, default: i64) -> i64 { .unwrap_or(default) } +fn env_float(key: &str, default: f64) -> f64 { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + fn env_bool(key: &str, default: bool) -> bool { match env::var(key) { Ok(v) => matches!(v.trim().to_lowercase().as_str(), "1" | "true" | "yes"), @@ -39,6 +60,92 @@ fn env_bool(key: &str, default: bool) -> bool { } } +pub fn normalize_zai_plan(value: Option<&str>) -> String { + match value.unwrap_or_default().trim().to_lowercase().as_str() { + "coding" => "coding".to_string(), + _ => "paygo".to_string(), + } +} + +pub fn resolve_zai_base_url(plan: &str, paygo_base_url: &str, coding_base_url: &str) -> String { + if normalize_zai_plan(Some(plan)) == "coding" { + coding_base_url.to_string() + } else { + paygo_base_url.to_string() + } +} + +pub fn normalize_web_search_provider(value: Option<&str>) -> String { + match value.unwrap_or_default().trim().to_lowercase().as_str() { + "firecrawl" => "firecrawl".to_string(), + _ => "exa".to_string(), + } +} + +fn normalize_base_url(value: &str) -> String { + value.trim().trim_end_matches('/').to_string() +} + +pub fn is_foundry_openai_base_url(value: &str) -> bool { + normalize_base_url(value) == FOUNDRY_OPENAI_BASE_URL +} + +pub fn is_foundry_anthropic_base_url(value: &str) -> bool { + normalize_base_url(value) == FOUNDRY_ANTHROPIC_BASE_URL +} + +pub fn resolve_openai_api_key(api_key: Option, base_url: &str) -> Option { + let normalized = api_key + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string); + if normalized.as_deref() == Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER) + && !is_foundry_openai_base_url(base_url) + { + return None; + } + if normalized.is_some() { + return normalized; + } + if is_foundry_openai_base_url(base_url) { + return Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.to_string()); + } + None +} + +pub fn resolve_anthropic_api_key(api_key: Option, base_url: &str) -> Option { + let normalized = api_key + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string); + if normalized.as_deref() == Some(FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER) + && !is_foundry_anthropic_base_url(base_url) + { + return None; + } + if normalized.is_some() { + return normalized; + } + if is_foundry_anthropic_base_url(base_url) { + return Some(FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER.to_string()); + } + None +} + +pub fn strip_foundry_model_prefix(model: &str) -> String { + let trimmed = model.trim(); + let lower = trimmed.to_lowercase(); + if lower.starts_with(AZURE_FOUNDRY_MODEL_PREFIX) { + return trimmed[AZURE_FOUNDRY_MODEL_PREFIX.len()..].to_string(); + } + if lower.starts_with(ANTHROPIC_FOUNDRY_MODEL_PREFIX) { + return trimmed[ANTHROPIC_FOUNDRY_MODEL_PREFIX.len()..].to_string(); + } + trimmed.to_string() +} + /// Central configuration for the OpenPlanter agent. /// /// Mirrors the Python `AgentConfig` dataclass field-for-field. @@ -55,8 +162,13 @@ pub struct AgentConfig { pub anthropic_base_url: String, pub openrouter_base_url: String, pub cerebras_base_url: String, + pub zai_plan: String, + pub zai_paygo_base_url: String, + pub zai_coding_base_url: String, + pub zai_base_url: String, pub ollama_base_url: String, pub exa_base_url: String, + pub firecrawl_base_url: String, // API keys pub api_key: Option, @@ -64,7 +176,10 @@ pub struct AgentConfig { pub anthropic_api_key: Option, pub openrouter_api_key: Option, pub cerebras_api_key: Option, + pub zai_api_key: Option, pub exa_api_key: Option, + pub firecrawl_api_key: Option, + pub web_search_provider: String, pub voyage_api_key: Option, // Limits @@ -80,6 +195,11 @@ pub struct AgentConfig { pub session_root_dir: String, pub max_persisted_observations: i64, pub max_solve_seconds: i64, + pub rate_limit_max_retries: i64, + pub rate_limit_backoff_base_sec: f64, + pub rate_limit_backoff_max_sec: f64, + pub rate_limit_retry_after_cap_sec: f64, + pub zai_stream_max_retries: i64, pub recursive: bool, pub min_subtask_depth: i64, pub acceptance_criteria: bool, @@ -93,21 +213,29 @@ impl Default for AgentConfig { Self { workspace: PathBuf::from("."), provider: "auto".into(), - model: "claude-opus-4-6".into(), + model: "anthropic-foundry/claude-opus-4-6".into(), reasoning_effort: Some("high".into()), - base_url: "https://api.openai.com/v1".into(), - openai_base_url: "https://api.openai.com/v1".into(), - anthropic_base_url: "https://api.anthropic.com/v1".into(), + base_url: FOUNDRY_OPENAI_BASE_URL.into(), + openai_base_url: FOUNDRY_OPENAI_BASE_URL.into(), + anthropic_base_url: FOUNDRY_ANTHROPIC_BASE_URL.into(), openrouter_base_url: "https://openrouter.ai/api/v1".into(), cerebras_base_url: "https://api.cerebras.ai/v1".into(), + zai_plan: "paygo".into(), + zai_paygo_base_url: ZAI_PAYGO_BASE_URL.into(), + zai_coding_base_url: ZAI_CODING_BASE_URL.into(), + zai_base_url: ZAI_PAYGO_BASE_URL.into(), ollama_base_url: "http://localhost:11434/v1".into(), exa_base_url: "https://api.exa.ai".into(), - api_key: None, - openai_api_key: None, - anthropic_api_key: None, + firecrawl_base_url: "https://api.firecrawl.dev/v1".into(), + api_key: Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.into()), + openai_api_key: Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.into()), + anthropic_api_key: Some(FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER.into()), openrouter_api_key: None, cerebras_api_key: None, + zai_api_key: None, exa_api_key: None, + firecrawl_api_key: None, + web_search_provider: "exa".into(), voyage_api_key: None, max_depth: 4, max_steps_per_call: 100, @@ -121,6 +249,11 @@ impl Default for AgentConfig { session_root_dir: ".openplanter".into(), max_persisted_observations: 400, max_solve_seconds: 0, + rate_limit_max_retries: 12, + rate_limit_backoff_base_sec: 1.0, + rate_limit_backoff_max_sec: 60.0, + rate_limit_retry_after_cap_sec: 120.0, + zai_stream_max_retries: 10, recursive: true, min_subtask_depth: 0, acceptance_criteria: true, @@ -136,27 +269,34 @@ impl AgentConfig { pub fn from_env(workspace: impl AsRef) -> Self { let ws = dunce_canonicalize(workspace.as_ref()); - let openai_api_key = env_opt("OPENPLANTER_OPENAI_API_KEY") - .or_else(|| env_opt("OPENAI_API_KEY")); + let openai_api_key = + env_opt("OPENPLANTER_OPENAI_API_KEY").or_else(|| env_opt("OPENAI_API_KEY")); + + let anthropic_api_key = + env_opt("OPENPLANTER_ANTHROPIC_API_KEY").or_else(|| env_opt("ANTHROPIC_API_KEY")); + + let openrouter_api_key = + env_opt("OPENPLANTER_OPENROUTER_API_KEY").or_else(|| env_opt("OPENROUTER_API_KEY")); - let anthropic_api_key = env_opt("OPENPLANTER_ANTHROPIC_API_KEY") - .or_else(|| env_opt("ANTHROPIC_API_KEY")); + let cerebras_api_key = + env_opt("OPENPLANTER_CEREBRAS_API_KEY").or_else(|| env_opt("CEREBRAS_API_KEY")); - let openrouter_api_key = env_opt("OPENPLANTER_OPENROUTER_API_KEY") - .or_else(|| env_opt("OPENROUTER_API_KEY")); + let zai_api_key = env_opt("OPENPLANTER_ZAI_API_KEY").or_else(|| env_opt("ZAI_API_KEY")); - let cerebras_api_key = env_opt("OPENPLANTER_CEREBRAS_API_KEY") - .or_else(|| env_opt("CEREBRAS_API_KEY")); + let exa_api_key = env_opt("OPENPLANTER_EXA_API_KEY").or_else(|| env_opt("EXA_API_KEY")); - let exa_api_key = env_opt("OPENPLANTER_EXA_API_KEY") - .or_else(|| env_opt("EXA_API_KEY")); + let firecrawl_api_key = + env_opt("OPENPLANTER_FIRECRAWL_API_KEY").or_else(|| env_opt("FIRECRAWL_API_KEY")); - let voyage_api_key = env_opt("OPENPLANTER_VOYAGE_API_KEY") - .or_else(|| env_opt("VOYAGE_API_KEY")); + let voyage_api_key = + env_opt("OPENPLANTER_VOYAGE_API_KEY").or_else(|| env_opt("VOYAGE_API_KEY")); let openai_base_url = env_opt("OPENPLANTER_OPENAI_BASE_URL") .or_else(|| env_opt("OPENPLANTER_BASE_URL")) - .unwrap_or_else(|| "https://api.openai.com/v1".into()); + .unwrap_or_else(|| FOUNDRY_OPENAI_BASE_URL.into()); + let anthropic_base_url = env_or("OPENPLANTER_ANTHROPIC_BASE_URL", FOUNDRY_ANTHROPIC_BASE_URL); + let openai_api_key = resolve_openai_api_key(openai_api_key, &openai_base_url); + let anthropic_api_key = resolve_anthropic_api_key(anthropic_api_key, &anthropic_base_url); let reasoning_effort_raw = env_or("OPENPLANTER_REASONING_EFFORT", "high") .trim() @@ -167,27 +307,31 @@ impl AgentConfig { Some(reasoning_effort_raw) }; - let provider_raw = env_or("OPENPLANTER_PROVIDER", "auto") - .trim() - .to_lowercase(); + let provider_raw = env_or("OPENPLANTER_PROVIDER", "auto").trim().to_lowercase(); let provider = if provider_raw.is_empty() { "auto".into() } else { provider_raw }; + let zai_plan = normalize_zai_plan(env_opt("OPENPLANTER_ZAI_PLAN").as_deref()); + let zai_paygo_base_url = env_or("OPENPLANTER_ZAI_PAYGO_BASE_URL", ZAI_PAYGO_BASE_URL); + let zai_coding_base_url = env_or("OPENPLANTER_ZAI_CODING_BASE_URL", ZAI_CODING_BASE_URL); + let zai_base_url = env_opt("OPENPLANTER_ZAI_BASE_URL").unwrap_or_else(|| { + resolve_zai_base_url(&zai_plan, &zai_paygo_base_url, &zai_coding_base_url) + }); + let web_search_provider = + normalize_web_search_provider(env_opt("OPENPLANTER_WEB_SEARCH_PROVIDER").as_deref()); + Self { workspace: ws, provider, - model: env_or("OPENPLANTER_MODEL", "claude-opus-4-6"), + model: env_or("OPENPLANTER_MODEL", PROVIDER_DEFAULT_MODELS["anthropic"]), reasoning_effort, base_url: openai_base_url.clone(), api_key: openai_api_key.clone(), openai_base_url, - anthropic_base_url: env_or( - "OPENPLANTER_ANTHROPIC_BASE_URL", - "https://api.anthropic.com/v1", - ), + anthropic_base_url, openrouter_base_url: env_or( "OPENPLANTER_OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1", @@ -196,16 +340,24 @@ impl AgentConfig { "OPENPLANTER_CEREBRAS_BASE_URL", "https://api.cerebras.ai/v1", ), - ollama_base_url: env_or( - "OPENPLANTER_OLLAMA_BASE_URL", - "http://localhost:11434/v1", - ), + zai_plan, + zai_paygo_base_url, + zai_coding_base_url, + zai_base_url, + ollama_base_url: env_or("OPENPLANTER_OLLAMA_BASE_URL", "http://localhost:11434/v1"), exa_base_url: env_or("OPENPLANTER_EXA_BASE_URL", "https://api.exa.ai"), + firecrawl_base_url: env_or( + "OPENPLANTER_FIRECRAWL_BASE_URL", + "https://api.firecrawl.dev/v1", + ), openai_api_key, anthropic_api_key, openrouter_api_key, cerebras_api_key, + zai_api_key, exa_api_key, + firecrawl_api_key, + web_search_provider, voyage_api_key, max_depth: env_int("OPENPLANTER_MAX_DEPTH", 4), max_steps_per_call: env_int("OPENPLANTER_MAX_STEPS", 100), @@ -219,6 +371,14 @@ impl AgentConfig { session_root_dir: env_or("OPENPLANTER_SESSION_DIR", ".openplanter"), max_persisted_observations: env_int("OPENPLANTER_MAX_PERSISTED_OBS", 400), max_solve_seconds: env_int("OPENPLANTER_MAX_SOLVE_SECONDS", 0), + rate_limit_max_retries: env_int("OPENPLANTER_RATE_LIMIT_MAX_RETRIES", 12), + rate_limit_backoff_base_sec: env_float("OPENPLANTER_RATE_LIMIT_BACKOFF_BASE_SEC", 1.0), + rate_limit_backoff_max_sec: env_float("OPENPLANTER_RATE_LIMIT_BACKOFF_MAX_SEC", 60.0), + rate_limit_retry_after_cap_sec: env_float( + "OPENPLANTER_RATE_LIMIT_RETRY_AFTER_CAP_SEC", + 120.0, + ), + zai_stream_max_retries: env_int("OPENPLANTER_ZAI_STREAM_MAX_RETRIES", 10), recursive: env_bool("OPENPLANTER_RECURSIVE", true), min_subtask_depth: env_int("OPENPLANTER_MIN_SUBTASK_DEPTH", 0), acceptance_criteria: env_bool("OPENPLANTER_ACCEPTANCE_CRITERIA", true), @@ -259,10 +419,27 @@ mod tests { fn test_default_config() { let cfg = AgentConfig::default(); assert_eq!(cfg.provider, "auto"); - assert_eq!(cfg.model, "claude-opus-4-6"); + assert_eq!(cfg.model, "anthropic-foundry/claude-opus-4-6"); assert_eq!(cfg.reasoning_effort, Some("high".into())); + assert_eq!(cfg.openai_base_url, FOUNDRY_OPENAI_BASE_URL); + assert_eq!(cfg.anthropic_base_url, FOUNDRY_ANTHROPIC_BASE_URL); + assert_eq!( + cfg.openai_api_key.as_deref(), + Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER) + ); + assert_eq!( + cfg.anthropic_api_key.as_deref(), + Some(FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER) + ); assert_eq!(cfg.max_depth, 4); assert_eq!(cfg.max_steps_per_call, 100); + assert_eq!(cfg.zai_plan, "paygo"); + assert_eq!(cfg.zai_base_url, ZAI_PAYGO_BASE_URL); + assert_eq!(cfg.web_search_provider, "exa"); + assert_eq!(cfg.rate_limit_max_retries, 12); + assert_eq!(cfg.rate_limit_backoff_base_sec, 1.0); + assert_eq!(cfg.rate_limit_backoff_max_sec, 60.0); + assert_eq!(cfg.rate_limit_retry_after_cap_sec, 120.0); assert!(cfg.recursive); assert!(cfg.acceptance_criteria); assert!(!cfg.demo); @@ -270,10 +447,13 @@ mod tests { #[test] fn test_provider_default_models() { - assert_eq!(PROVIDER_DEFAULT_MODELS.get("openai"), Some(&"gpt-5.2")); + assert_eq!( + PROVIDER_DEFAULT_MODELS.get("openai"), + Some(&"azure-foundry/gpt-5.3-codex") + ); assert_eq!( PROVIDER_DEFAULT_MODELS.get("anthropic"), - Some(&"claude-opus-4-6") + Some(&"anthropic-foundry/claude-opus-4-6") ); assert_eq!( PROVIDER_DEFAULT_MODELS.get("openrouter"), @@ -283,6 +463,7 @@ mod tests { PROVIDER_DEFAULT_MODELS.get("cerebras"), Some(&"qwen-3-235b-a22b-instruct-2507") ); + assert_eq!(PROVIDER_DEFAULT_MODELS.get("zai"), Some(&"glm-5")); assert_eq!(PROVIDER_DEFAULT_MODELS.get("ollama"), Some(&"llama3.2")); } @@ -296,17 +477,27 @@ mod tests { "OPENPLANTER_REASONING_EFFORT", "OPENPLANTER_OPENAI_API_KEY", "OPENAI_API_KEY", + "OPENPLANTER_OPENAI_BASE_URL", + "OPENPLANTER_BASE_URL", "OPENPLANTER_ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY", + "OPENPLANTER_ANTHROPIC_BASE_URL", + "OPENPLANTER_ZAI_API_KEY", + "ZAI_API_KEY", "OPENPLANTER_MAX_DEPTH", "OPENPLANTER_RECURSIVE", "OPENPLANTER_DEMO", + "OPENPLANTER_WEB_SEARCH_PROVIDER", + "OPENPLANTER_ZAI_PLAN", + "OPENPLANTER_ZAI_BASE_URL", + "OPENPLANTER_RATE_LIMIT_MAX_RETRIES", + "OPENPLANTER_RATE_LIMIT_BACKOFF_BASE_SEC", + "OPENPLANTER_RATE_LIMIT_BACKOFF_MAX_SEC", + "OPENPLANTER_RATE_LIMIT_RETRY_AFTER_CAP_SEC", + "OPENPLANTER_ZAI_STREAM_MAX_RETRIES", ]; // Save original values - let saved: Vec<_> = keys - .iter() - .map(|k| (*k, env::var(k).ok())) - .collect(); + let saved: Vec<_> = keys.iter().map(|k| (*k, env::var(k).ok())).collect(); // SAFETY: test-only; combined into one test to avoid parallel env mutation unsafe { @@ -318,33 +509,64 @@ mod tests { let cfg = AgentConfig::from_env("/tmp"); assert_eq!(cfg.provider, "auto"); - assert_eq!(cfg.model, "claude-opus-4-6"); + assert_eq!(cfg.model, "anthropic-foundry/claude-opus-4-6"); assert_eq!(cfg.reasoning_effort, Some("high".into())); assert_eq!(cfg.max_depth, 4); assert!(cfg.recursive); assert!(!cfg.demo); - assert!(cfg.openai_api_key.is_none()); - assert!(cfg.anthropic_api_key.is_none()); + assert_eq!( + cfg.openai_api_key.as_deref(), + Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER) + ); + assert_eq!( + cfg.anthropic_api_key.as_deref(), + Some(FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER) + ); + assert!(cfg.zai_api_key.is_none()); + assert_eq!(cfg.openai_base_url, FOUNDRY_OPENAI_BASE_URL); + assert_eq!(cfg.anthropic_base_url, FOUNDRY_ANTHROPIC_BASE_URL); + assert_eq!(cfg.web_search_provider, "exa"); + assert_eq!(cfg.rate_limit_max_retries, 12); + assert_eq!(cfg.rate_limit_backoff_base_sec, 1.0); + assert_eq!(cfg.rate_limit_backoff_max_sec, 60.0); + assert_eq!(cfg.rate_limit_retry_after_cap_sec, 120.0); unsafe { // --- Phase 2: test custom values --- env::set_var("OPENPLANTER_PROVIDER", "openai"); - env::set_var("OPENPLANTER_MODEL", "gpt-5.2"); + env::set_var("OPENPLANTER_MODEL", "azure-foundry/gpt-5.3-codex"); env::set_var("OPENPLANTER_REASONING_EFFORT", "low"); env::set_var("OPENPLANTER_MAX_DEPTH", "8"); env::set_var("OPENPLANTER_RECURSIVE", "false"); env::set_var("OPENPLANTER_DEMO", "true"); env::set_var("OPENAI_API_KEY", "sk-test123"); + env::set_var("ZAI_API_KEY", "zai-test123"); + env::set_var("OPENPLANTER_WEB_SEARCH_PROVIDER", "firecrawl"); + env::set_var("OPENPLANTER_RATE_LIMIT_MAX_RETRIES", "5"); + env::set_var("OPENPLANTER_RATE_LIMIT_BACKOFF_BASE_SEC", "2.5"); + env::set_var("OPENPLANTER_RATE_LIMIT_BACKOFF_MAX_SEC", "30.0"); + env::set_var("OPENPLANTER_RATE_LIMIT_RETRY_AFTER_CAP_SEC", "90.0"); + env::set_var("OPENPLANTER_ZAI_PLAN", "coding"); + env::set_var("OPENPLANTER_ZAI_STREAM_MAX_RETRIES", "7"); } let cfg = AgentConfig::from_env("/tmp"); assert_eq!(cfg.provider, "openai"); - assert_eq!(cfg.model, "gpt-5.2"); + assert_eq!(cfg.model, "azure-foundry/gpt-5.3-codex"); assert_eq!(cfg.reasoning_effort, Some("low".into())); assert_eq!(cfg.max_depth, 8); assert!(!cfg.recursive); assert!(cfg.demo); assert_eq!(cfg.openai_api_key, Some("sk-test123".into())); + assert_eq!(cfg.zai_api_key, Some("zai-test123".into())); + assert_eq!(cfg.zai_plan, "coding"); + assert_eq!(cfg.zai_base_url, ZAI_CODING_BASE_URL); + assert_eq!(cfg.zai_stream_max_retries, 7); + assert_eq!(cfg.web_search_provider, "firecrawl"); + assert_eq!(cfg.rate_limit_max_retries, 5); + assert_eq!(cfg.rate_limit_backoff_base_sec, 2.5); + assert_eq!(cfg.rate_limit_backoff_max_sec, 30.0); + assert_eq!(cfg.rate_limit_retry_after_cap_sec, 90.0); // Restore original values for (k, v) in saved { @@ -356,4 +578,37 @@ mod tests { } } } + + #[test] + fn test_normalizers() { + assert_eq!(normalize_zai_plan(Some("coding")), "coding"); + assert_eq!(normalize_zai_plan(Some("bad-value")), "paygo"); + assert_eq!( + resolve_zai_base_url("coding", "https://paygo.example", "https://coding.example"), + "https://coding.example" + ); + assert_eq!( + normalize_web_search_provider(Some("firecrawl")), + "firecrawl" + ); + assert_eq!(normalize_web_search_provider(Some("other")), "exa"); + assert!(is_foundry_openai_base_url(FOUNDRY_OPENAI_BASE_URL)); + assert!(is_foundry_anthropic_base_url(FOUNDRY_ANTHROPIC_BASE_URL)); + assert_eq!( + resolve_openai_api_key(None, FOUNDRY_OPENAI_BASE_URL).as_deref(), + Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER) + ); + assert_eq!( + resolve_anthropic_api_key(None, FOUNDRY_ANTHROPIC_BASE_URL).as_deref(), + Some(FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER) + ); + assert_eq!( + strip_foundry_model_prefix("azure-foundry/gpt-5.3-codex"), + "gpt-5.3-codex" + ); + assert_eq!( + strip_foundry_model_prefix("anthropic-foundry/claude-opus-4-6"), + "claude-opus-4-6" + ); + } } diff --git a/openplanter-desktop/crates/op-core/src/credentials.rs b/openplanter-desktop/crates/op-core/src/credentials.rs index 12e7e914..af746019 100644 --- a/openplanter-desktop/crates/op-core/src/credentials.rs +++ b/openplanter-desktop/crates/op-core/src/credentials.rs @@ -16,26 +16,27 @@ pub struct CredentialBundle { pub anthropic_api_key: Option, pub openrouter_api_key: Option, pub cerebras_api_key: Option, + pub zai_api_key: Option, pub exa_api_key: Option, + pub firecrawl_api_key: Option, pub voyage_api_key: Option, } impl CredentialBundle { /// Returns `true` if any key has a non-empty value. pub fn has_any(&self) -> bool { - let keys: [&Option; 6] = [ + let keys: [&Option; 8] = [ &self.openai_api_key, &self.anthropic_api_key, &self.openrouter_api_key, &self.cerebras_api_key, + &self.zai_api_key, &self.exa_api_key, + &self.firecrawl_api_key, &self.voyage_api_key, ]; - keys.iter().any(|k| { - k.as_ref() - .map(|v| !v.trim().is_empty()) - .unwrap_or(false) - }) + keys.iter() + .any(|k| k.as_ref().map(|v| !v.trim().is_empty()).unwrap_or(false)) } /// Fill in missing keys from `other`. @@ -51,7 +52,9 @@ impl CredentialBundle { fill!(anthropic_api_key); fill!(openrouter_api_key); fill!(cerebras_api_key); + fill!(zai_api_key); fill!(exa_api_key); + fill!(firecrawl_api_key); fill!(voyage_api_key); } @@ -69,7 +72,9 @@ impl CredentialBundle { add!(anthropic_api_key, "anthropic_api_key"); add!(openrouter_api_key, "openrouter_api_key"); add!(cerebras_api_key, "cerebras_api_key"); + add!(zai_api_key, "zai_api_key"); add!(exa_api_key, "exa_api_key"); + add!(firecrawl_api_key, "firecrawl_api_key"); add!(voyage_api_key, "voyage_api_key"); out } @@ -87,7 +92,9 @@ impl CredentialBundle { anthropic_api_key: get_str(payload, "anthropic_api_key"), openrouter_api_key: get_str(payload, "openrouter_api_key"), cerebras_api_key: get_str(payload, "cerebras_api_key"), + zai_api_key: get_str(payload, "zai_api_key"), exa_api_key: get_str(payload, "exa_api_key"), + firecrawl_api_key: get_str(payload, "firecrawl_api_key"), voyage_api_key: get_str(payload, "voyage_api_key"), } } @@ -146,12 +153,14 @@ pub fn parse_env_file(path: &Path) -> CredentialBundle { "OPENROUTER_API_KEY", "OPENPLANTER_OPENROUTER_API_KEY", ), - cerebras_api_key: get_key( + cerebras_api_key: get_key(&env_map, "CEREBRAS_API_KEY", "OPENPLANTER_CEREBRAS_API_KEY"), + zai_api_key: get_key(&env_map, "ZAI_API_KEY", "OPENPLANTER_ZAI_API_KEY"), + exa_api_key: get_key(&env_map, "EXA_API_KEY", "OPENPLANTER_EXA_API_KEY"), + firecrawl_api_key: get_key( &env_map, - "CEREBRAS_API_KEY", - "OPENPLANTER_CEREBRAS_API_KEY", + "FIRECRAWL_API_KEY", + "OPENPLANTER_FIRECRAWL_API_KEY", ), - exa_api_key: get_key(&env_map, "EXA_API_KEY", "OPENPLANTER_EXA_API_KEY"), voyage_api_key: get_key(&env_map, "VOYAGE_API_KEY", "OPENPLANTER_VOYAGE_API_KEY"), } } @@ -171,7 +180,9 @@ pub fn credentials_from_env() -> CredentialBundle { anthropic_api_key: env_key("OPENPLANTER_ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY"), openrouter_api_key: env_key("OPENPLANTER_OPENROUTER_API_KEY", "OPENROUTER_API_KEY"), cerebras_api_key: env_key("OPENPLANTER_CEREBRAS_API_KEY", "CEREBRAS_API_KEY"), + zai_api_key: env_key("OPENPLANTER_ZAI_API_KEY", "ZAI_API_KEY"), exa_api_key: env_key("OPENPLANTER_EXA_API_KEY", "EXA_API_KEY"), + firecrawl_api_key: env_key("OPENPLANTER_FIRECRAWL_API_KEY", "FIRECRAWL_API_KEY"), voyage_api_key: env_key("OPENPLANTER_VOYAGE_API_KEY", "VOYAGE_API_KEY"), } } @@ -319,11 +330,13 @@ mod tests { let b = CredentialBundle { openai_api_key: Some("should-not-overwrite".into()), anthropic_api_key: Some("new-key".into()), + zai_api_key: Some("zai-key".into()), ..Default::default() }; a.merge_missing(&b); assert_eq!(a.openai_api_key, Some("existing".into())); assert_eq!(a.anthropic_api_key, Some("new-key".into())); + assert_eq!(a.zai_api_key, Some("zai-key".into())); } #[test] @@ -332,12 +345,14 @@ mod tests { openai_api_key: Some("sk-123".into()), anthropic_api_key: None, openrouter_api_key: Some("or-456".into()), + firecrawl_api_key: Some("fc-789".into()), ..Default::default() }; let json = bundle.to_json(); assert_eq!(json.get("openai_api_key").unwrap(), "sk-123"); assert!(!json.contains_key("anthropic_api_key")); assert_eq!(json.get("openrouter_api_key").unwrap(), "or-456"); + assert_eq!(json.get("firecrawl_api_key").unwrap(), "fc-789"); } #[test] @@ -351,6 +366,8 @@ mod tests { OPENAI_API_KEY=sk-from-env export ANTHROPIC_API_KEY='ant-key' EXA_API_KEY="exa-quoted" +ZAI_API_KEY=zai-from-env +OPENPLANTER_FIRECRAWL_API_KEY="firecrawl-quoted" UNRELATED_VAR=foo "#, ) @@ -360,6 +377,8 @@ UNRELATED_VAR=foo assert_eq!(bundle.openai_api_key, Some("sk-from-env".into())); assert_eq!(bundle.anthropic_api_key, Some("ant-key".into())); assert_eq!(bundle.exa_api_key, Some("exa-quoted".into())); + assert_eq!(bundle.zai_api_key, Some("zai-from-env".into())); + assert_eq!(bundle.firecrawl_api_key, Some("firecrawl-quoted".into())); assert!(bundle.cerebras_api_key.is_none()); } @@ -370,12 +389,14 @@ UNRELATED_VAR=foo let bundle = CredentialBundle { openai_api_key: Some("sk-test".into()), anthropic_api_key: Some("ant-test".into()), + zai_api_key: Some("zai-test".into()), ..Default::default() }; store.save(&bundle).unwrap(); let loaded = store.load(); assert_eq!(loaded.openai_api_key, Some("sk-test".into())); assert_eq!(loaded.anthropic_api_key, Some("ant-test".into())); + assert_eq!(loaded.zai_api_key, Some("zai-test".into())); } #[test] diff --git a/openplanter-desktop/crates/op-core/src/engine/curator.rs b/openplanter-desktop/crates/op-core/src/engine/curator.rs index e0015567..7d50a61b 100644 --- a/openplanter-desktop/crates/op-core/src/engine/curator.rs +++ b/openplanter-desktop/crates/op-core/src/engine/curator.rs @@ -3,14 +3,13 @@ /// Runs as a non-blocking background task after each main agent step. /// Reads the latest step context, decides if wiki updates are needed, /// and writes to `.openplanter/wiki/` using a restricted tool set. - use tokio_util::sync::CancellationToken; use crate::builder::build_model; use crate::config::AgentConfig; use crate::model::Message; -use crate::tools::defs::build_curator_tool_defs; use crate::tools::WorkspaceTools; +use crate::tools::defs::build_curator_tool_defs; /// Result of a curator run. #[derive(Debug, Clone)] @@ -78,7 +77,9 @@ pub fn extract_step_context(messages: &[Message]) -> String { let mut context = String::new(); // Find last Assistant message index - let assistant_idx = messages.iter().rposition(|m| matches!(m, Message::Assistant { .. })); + let assistant_idx = messages + .iter() + .rposition(|m| matches!(m, Message::Assistant { .. })); let start = match assistant_idx { Some(idx) => idx, None => return context, @@ -86,7 +87,10 @@ pub fn extract_step_context(messages: &[Message]) -> String { for msg in &messages[start..] { match msg { - Message::Assistant { content, tool_calls } => { + Message::Assistant { + content, + tool_calls, + } => { context.push_str("=== Assistant ===\n"); context.push_str(content); context.push('\n'); @@ -223,8 +227,10 @@ pub async fn run_curator( let result = tools.execute(&tc.name, &tc.arguments).await; // Track file modifications - if matches!(tc.name.as_str(), "write_file" | "edit_file" | "apply_patch" | "hashline_edit") - && !result.is_error + if matches!( + tc.name.as_str(), + "write_file" | "edit_file" | "apply_patch" | "hashline_edit" + ) && !result.is_error { files_changed += 1; // Extract path for summary @@ -270,8 +276,12 @@ mod tests { #[test] fn test_extract_step_context_no_assistant() { let messages = vec![ - Message::System { content: "sys".into() }, - Message::User { content: "hello".into() }, + Message::System { + content: "sys".into(), + }, + Message::User { + content: "hello".into(), + }, ]; assert_eq!(extract_step_context(&messages), ""); } @@ -279,8 +289,12 @@ mod tests { #[test] fn test_extract_step_context_with_tool_calls() { let messages = vec![ - Message::System { content: "sys".into() }, - Message::User { content: "investigate".into() }, + Message::System { + content: "sys".into(), + }, + Message::User { + content: "investigate".into(), + }, Message::Assistant { content: "I'll search for data.".into(), tool_calls: Some(vec![ToolCall { @@ -319,7 +333,9 @@ mod tests { content: "old step".into(), tool_calls: None, }, - Message::User { content: "continue".into() }, + Message::User { + content: "continue".into(), + }, Message::Assistant { content: "new step".into(), tool_calls: Some(vec![ToolCall { @@ -342,8 +358,18 @@ mod tests { #[test] fn test_curator_tool_names_no_dangerous_tools() { for name in CURATOR_TOOL_NAMES { - assert!(!["web_search", "fetch_url", "run_shell", "run_shell_bg", "check_shell_bg", "kill_shell_bg"] - .contains(name), "Curator should not have access to {name}"); + assert!( + ![ + "web_search", + "fetch_url", + "run_shell", + "run_shell_bg", + "check_shell_bg", + "kill_shell_bg" + ] + .contains(name), + "Curator should not have access to {name}" + ); } } } diff --git a/openplanter-desktop/crates/op-core/src/engine/judge.rs b/openplanter-desktop/crates/op-core/src/engine/judge.rs index 355000cd..0e4be82e 100644 --- a/openplanter-desktop/crates/op-core/src/engine/judge.rs +++ b/openplanter-desktop/crates/op-core/src/engine/judge.rs @@ -86,13 +86,11 @@ impl Default for AcceptanceCriteriaJudge { /// Extract significant terms from criteria text (words >= 4 chars, excluding stop words). fn extract_terms(text: &str) -> Vec<&str> { const STOP_WORDS: &[&str] = &[ - "the", "and", "for", "are", "but", "not", "you", "all", - "can", "has", "her", "was", "one", "our", "out", "with", - "that", "this", "have", "from", "they", "been", "said", - "each", "which", "their", "will", "other", "about", "many", - "then", "them", "these", "some", "would", "make", "like", - "into", "could", "time", "very", "when", "what", "your", - "there", "should", "must", "also", + "the", "and", "for", "are", "but", "not", "you", "all", "can", "has", "her", "was", "one", + "our", "out", "with", "that", "this", "have", "from", "they", "been", "said", "each", + "which", "their", "will", "other", "about", "many", "then", "them", "these", "some", + "would", "make", "like", "into", "could", "time", "very", "when", "what", "your", "there", + "should", "must", "also", ]; text.split_whitespace() diff --git a/openplanter-desktop/crates/op-core/src/engine/mod.rs b/openplanter-desktop/crates/op-core/src/engine/mod.rs index cdf2847e..f19e38b1 100644 --- a/openplanter-desktop/crates/op-core/src/engine/mod.rs +++ b/openplanter-desktop/crates/op-core/src/engine/mod.rs @@ -7,6 +7,9 @@ pub mod context; pub mod curator; pub mod judge; +use std::time::Duration; + +use anyhow::anyhow; use tokio::sync::mpsc; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; @@ -14,12 +17,12 @@ use tokio_util::sync::CancellationToken; use crate::builder::build_model; use crate::config::AgentConfig; use crate::events::{DeltaEvent, DeltaKind, StepEvent, TokenUsage}; -use crate::model::Message; +use crate::model::{BaseModel, Message, ModelTurn, RateLimitError}; use crate::prompts::build_system_prompt; -use crate::tools::defs::build_tool_defs; use crate::tools::WorkspaceTools; +use crate::tools::defs::build_tool_defs; -use self::curator::{extract_step_context, run_curator, CuratorResult}; +use self::curator::{CuratorResult, extract_step_context, run_curator}; /// Outcome from a background curator task (success or error). enum CuratorOutcome { @@ -114,11 +117,7 @@ pub trait SolveEmitter: Send + Sync { // This is a placeholder until the full engine is implemented in Phase 4. // It emits the standard event sequence so the frontend can be developed // and tested against a working backend. -pub async fn demo_solve( - objective: &str, - emitter: &dyn SolveEmitter, - cancel: CancellationToken, -) { +pub async fn demo_solve(objective: &str, emitter: &dyn SolveEmitter, cancel: CancellationToken) { emitter.emit_trace(&format!("Solving: {objective}")); if cancel.is_cancelled() { @@ -176,11 +175,18 @@ fn estimate_tokens(messages: &[Message]) -> usize { .iter() .map(|m| match m { Message::System { content } | Message::User { content } => content.len(), - Message::Assistant { content, tool_calls } => { + Message::Assistant { + content, + tool_calls, + } => { content.len() + tool_calls .as_ref() - .map(|tcs| tcs.iter().map(|tc| tc.arguments.len() + tc.name.len()).sum()) + .map(|tcs| { + tcs.iter() + .map(|tc| tc.arguments.len() + tc.name.len()) + .sum() + }) .unwrap_or(0) } Message::Tool { content, .. } => content.len(), @@ -213,6 +219,75 @@ fn compact_messages(messages: &mut Vec, max_tokens: usize) { } } +fn compute_rate_limit_delay_sec( + config: &AgentConfig, + retry_count: usize, + err: &RateLimitError, +) -> f64 { + let retry_after_cap = config.rate_limit_retry_after_cap_sec.max(0.0); + let backoff_max = config.rate_limit_backoff_max_sec.max(0.0); + let delay = err + .retry_after_sec + .map(|value| value.max(0.0).min(retry_after_cap)) + .unwrap_or_else(|| { + let base = config.rate_limit_backoff_base_sec.max(0.0); + base * 2_f64.powi((retry_count.saturating_sub(1)) as i32) + }); + delay.min(backoff_max) +} + +async fn chat_stream_with_rate_limit_retries( + model: &dyn BaseModel, + messages: &[Message], + tool_defs: &[serde_json::Value], + on_delta: &(dyn Fn(DeltaEvent) + Send + Sync), + cancel: &CancellationToken, + config: &AgentConfig, + emitter: &dyn SolveEmitter, + step: usize, +) -> anyhow::Result { + let max_retries = config.rate_limit_max_retries.max(0) as usize; + let mut retries = 0usize; + + loop { + if cancel.is_cancelled() { + return Err(anyhow!("Cancelled")); + } + + match model + .chat_stream(messages, tool_defs, on_delta, cancel) + .await + { + Ok(turn) => return Ok(turn), + Err(err) => { + if let Some(rate_limit) = err.downcast_ref::() { + if retries >= max_retries { + return Err(err); + } + retries += 1; + let delay_sec = compute_rate_limit_delay_sec(config, retries, rate_limit); + let provider_code = rate_limit + .provider_code + .as_deref() + .map(|code| format!(" ({code})")) + .unwrap_or_default(); + emitter.emit_trace(&format!( + "[d0/s{step}] rate limited{provider_code}. Sleeping {delay_sec:.1}s before retry {retries}/{max_retries}..." + )); + if delay_sec > 0.0 { + tokio::select! { + _ = cancel.cancelled() => return Err(anyhow!("Cancelled")), + _ = tokio::time::sleep(Duration::from_secs_f64(delay_sec)) => {} + } + } + continue; + } + return Err(err); + } + } + } +} + /// Real solve flow with a multi-step agentic loop. /// /// Calls the model with tool definitions. If the model returns tool calls, @@ -240,21 +315,14 @@ pub async fn solve( }; let provider = model.provider_name().to_string(); - emitter.emit_trace(&format!( - "Solving with {}/{}", - provider, - model.model_name() - )); + emitter.emit_trace(&format!("Solving with {}/{}", provider, model.model_name())); // 2. Build tools and messages let tool_defs = build_tool_defs(&provider); let mut tools = WorkspaceTools::new(config); - let system_prompt = build_system_prompt( - config.recursive, - config.acceptance_criteria, - config.demo, - ); + let system_prompt = + build_system_prompt(config.recursive, config.acceptance_criteria, config.demo); let mut messages = vec![ Message::System { content: system_prompt, @@ -288,9 +356,17 @@ pub async fn solve( compact_messages(&mut messages, 100_000); // Call model with streaming - let turn = match model - .chat_stream(&messages, &tool_defs, &|delta| emitter.emit_delta(delta), &cancel) - .await + let turn = match chat_stream_with_rate_limit_retries( + model.as_ref(), + &messages, + &tool_defs, + &|delta| emitter.emit_delta(delta), + &cancel, + config, + emitter, + step, + ) + .await { Ok(t) => t, Err(e) => { @@ -334,7 +410,13 @@ pub async fn solve( emitter.emit_complete(&turn.text); tools.cleanup(); // Wait for in-flight curators before exiting - finish_curators(&mut curator_handles, &mut curator_rx, &mut messages, emitter).await; + finish_curators( + &mut curator_handles, + &mut curator_rx, + &mut messages, + emitter, + ) + .await; return; } @@ -351,7 +433,11 @@ pub async fn solve( let result = tools.execute(&tc.name, &tc.arguments).await; if result.is_error { - emitter.emit_trace(&format!("Tool {} error: {}", tc.name, &result.content[..result.content.len().min(200)])); + emitter.emit_trace(&format!( + "Tool {} error: {}", + tc.name, + &result.content[..result.content.len().min(200)] + )); } messages.push(Message::Tool { @@ -406,7 +492,13 @@ pub async fn solve( // Budget exhausted tools.cleanup(); - finish_curators(&mut curator_handles, &mut curator_rx, &mut messages, emitter).await; + finish_curators( + &mut curator_handles, + &mut curator_rx, + &mut messages, + emitter, + ) + .await; emitter.emit_error(&format!( "Step budget exhausted after {max_steps} steps. \ The model did not produce a final answer within the allowed steps." @@ -460,10 +552,7 @@ mod tests { } fn emit_step(&self, event: StepEvent) { - self.events - .lock() - .unwrap() - .push(RecordedEvent::Step(event)); + self.events.lock().unwrap().push(RecordedEvent::Step(event)); } fn emit_complete(&self, result: &str) { @@ -489,7 +578,11 @@ mod tests { demo_solve("Test objective", &emitter, token).await; let events = emitter.events(); - assert!(events.len() >= 4, "expected at least 4 events, got {}", events.len()); + assert!( + events.len() >= 4, + "expected at least 4 events, got {}", + events.len() + ); // First event: trace assert!(matches!(&events[0], RecordedEvent::Trace(_))); @@ -531,8 +624,13 @@ mod tests { .any(|e| matches!(e, RecordedEvent::Error(m) if m == "Cancelled")); assert!(has_error, "expected a Cancelled error event"); - let has_complete = events.iter().any(|e| matches!(e, RecordedEvent::Complete(_))); - assert!(!has_complete, "should not have a Complete event when cancelled"); + let has_complete = events + .iter() + .any(|e| matches!(e, RecordedEvent::Complete(_))); + assert!( + !has_complete, + "should not have a Complete event when cancelled" + ); } #[tokio::test] @@ -607,7 +705,10 @@ mod tests { let has_error = recorded .iter() .any(|e| matches!(e, RecordedEvent::Error(m) if m == "Cancelled")); - assert!(has_error, "expected Cancelled error after mid-flight cancel"); + assert!( + has_error, + "expected Cancelled error after mid-flight cancel" + ); // Should NOT have a Complete event let has_complete = recorded @@ -661,9 +762,16 @@ mod tests { #[test] fn test_estimate_tokens() { let messages = vec![ - Message::System { content: "System prompt".into() }, // 13 chars - Message::User { content: "Hello".into() }, // 5 chars - Message::Tool { tool_call_id: "t1".into(), content: "x".repeat(4000) }, + Message::System { + content: "System prompt".into(), + }, // 13 chars + Message::User { + content: "Hello".into(), + }, // 5 chars + Message::Tool { + tool_call_id: "t1".into(), + content: "x".repeat(4000), + }, ]; let tokens = estimate_tokens(&messages); // (13 + 5 + 4000) / 4 = 1004 @@ -673,9 +781,16 @@ mod tests { #[test] fn test_compact_messages_no_op_when_under_limit() { let mut messages = vec![ - Message::System { content: "System".into() }, - Message::User { content: "Hello".into() }, - Message::Tool { tool_call_id: "t1".into(), content: "Short result".into() }, + Message::System { + content: "System".into(), + }, + Message::User { + content: "Hello".into(), + }, + Message::Tool { + tool_call_id: "t1".into(), + content: "Short result".into(), + }, ]; compact_messages(&mut messages, 100_000); // Should be unchanged @@ -688,14 +803,24 @@ mod tests { fn test_compact_messages_truncates_old_tool_results() { let big_result = "x".repeat(8000); let mut messages = vec![ - Message::System { content: "System".into() }, - Message::User { content: "Hello".into() }, + Message::System { + content: "System".into(), + }, + Message::User { + content: "Hello".into(), + }, ]; // Add 15 old steps (assistant + tool pairs) to exceed keep_recent for i in 0..15 { - messages.push(Message::Assistant { content: format!("step{i}"), tool_calls: None }); - messages.push(Message::Tool { tool_call_id: format!("t{i}"), content: big_result.clone() }); + messages.push(Message::Assistant { + content: format!("step{i}"), + tool_calls: None, + }); + messages.push(Message::Tool { + tool_call_id: format!("t{i}"), + content: big_result.clone(), + }); } // Total: ~(6 + 5 + 15*(5+8000)) / 4 ≈ 30_000 tokens @@ -704,12 +829,20 @@ mod tests { // Old tool result (index 3, early in the list) should be truncated if let Message::Tool { content, .. } = &messages[3] { - assert!(content.len() < 300, "old tool result should be truncated, got {} chars", content.len()); + assert!( + content.len() < 300, + "old tool result should be truncated, got {} chars", + content.len() + ); assert!(content.contains("truncated")); } // Recent tool result (last one) should be intact - let last_tool = messages.iter().rev().find(|m| matches!(m, Message::Tool { .. })).unwrap(); + let last_tool = messages + .iter() + .rev() + .find(|m| matches!(m, Message::Tool { .. })) + .unwrap(); if let Message::Tool { content, .. } = last_tool { assert_eq!(content.len(), 8000, "recent tool result should be intact"); } diff --git a/openplanter-desktop/crates/op-core/src/events.rs b/openplanter-desktop/crates/op-core/src/events.rs index 70a648a1..156cfce4 100644 --- a/openplanter-desktop/crates/op-core/src/events.rs +++ b/openplanter-desktop/crates/op-core/src/events.rs @@ -120,6 +120,8 @@ pub struct ConfigView { pub provider: String, pub model: String, pub reasoning_effort: Option, + pub zai_plan: String, + pub web_search_provider: String, pub workspace: String, pub session_id: Option, pub recursive: bool, @@ -134,6 +136,8 @@ pub struct PartialConfig { pub provider: Option, pub model: Option, pub reasoning_effort: Option, + pub zai_plan: Option, + pub web_search_provider: Option, } /// Model information for the model list. diff --git a/openplanter-desktop/crates/op-core/src/lib.rs b/openplanter-desktop/crates/op-core/src/lib.rs index 05b9c49a..62efa5cf 100644 --- a/openplanter-desktop/crates/op-core/src/lib.rs +++ b/openplanter-desktop/crates/op-core/src/lib.rs @@ -1,11 +1,11 @@ +pub mod builder; pub mod config; -pub mod prompts; pub mod credentials; -pub mod settings; -pub mod builder; +pub mod engine; pub mod events; pub mod model; -pub mod engine; -pub mod tools; +pub mod prompts; pub mod session; +pub mod settings; +pub mod tools; pub mod wiki; diff --git a/openplanter-desktop/crates/op-core/src/model/anthropic.rs b/openplanter-desktop/crates/op-core/src/model/anthropic.rs index a0705724..e760120a 100644 --- a/openplanter-desktop/crates/op-core/src/model/anthropic.rs +++ b/openplanter-desktop/crates/op-core/src/model/anthropic.rs @@ -2,12 +2,13 @@ // // Uses the Anthropic Messages API with SSE streaming. -use anyhow::{anyhow, Context}; +use anyhow::{Context, anyhow}; use reqwest_eventsource::{Event, RequestBuilderExt}; use tokio_util::sync::CancellationToken; -use crate::events::{DeltaEvent, DeltaKind}; use super::{BaseModel, Message, ModelTurn, ToolCall}; +use crate::config::strip_foundry_model_prefix; +use crate::events::{DeltaEvent, DeltaKind}; pub struct AnthropicModel { client: reqwest::Client, @@ -36,10 +37,14 @@ impl AnthropicModel { } fn is_opus_46(&self) -> bool { - let lower = self.model.to_lowercase(); + let lower = self.request_model_name().to_lowercase(); lower.contains("opus-4-6") || lower.contains("opus-4.6") } + fn request_model_name(&self) -> String { + strip_foundry_model_prefix(&self.model) + } + /// Extract the system prompt from messages (Anthropic uses a top-level `system` field). fn extract_system(messages: &[Message]) -> Option { for msg in messages { @@ -67,7 +72,10 @@ impl AnthropicModel { "content": content, })); } - Message::Assistant { content, tool_calls } => { + Message::Assistant { + content, + tool_calls, + } => { let mut blocks: Vec = Vec::new(); if !content.is_empty() { blocks.push(serde_json::json!({ @@ -77,8 +85,8 @@ impl AnthropicModel { } if let Some(tcs) = tool_calls { for tc in tcs { - let input: serde_json::Value = - serde_json::from_str(&tc.arguments).unwrap_or(serde_json::json!({})); + let input: serde_json::Value = serde_json::from_str(&tc.arguments) + .unwrap_or(serde_json::json!({})); blocks.push(serde_json::json!({ "type": "tool_use", "id": tc.id, @@ -92,7 +100,10 @@ impl AnthropicModel { "content": blocks, })); } - Message::Tool { tool_call_id, content } => { + Message::Tool { + tool_call_id, + content, + } => { let block = serde_json::json!({ "type": "tool_result", "tool_use_id": tool_call_id, @@ -101,8 +112,12 @@ impl AnthropicModel { // Merge into previous user message if it contains tool_result blocks if let Some(last) = result.last_mut() { if last.get("role").and_then(|r| r.as_str()) == Some("user") { - if let Some(arr) = last.get_mut("content").and_then(|c| c.as_array_mut()) { - if arr.iter().any(|b| b.get("type").and_then(|t| t.as_str()) == Some("tool_result")) { + if let Some(arr) = + last.get_mut("content").and_then(|c| c.as_array_mut()) + { + if arr.iter().any(|b| { + b.get("type").and_then(|t| t.as_str()) == Some("tool_result") + }) { arr.push(block); continue; } @@ -133,7 +148,7 @@ impl AnthropicModel { let use_thinking = matches!(effort.as_str(), "low" | "medium" | "high"); let mut payload = serde_json::json!({ - "model": self.model, + "model": self.request_model_name(), "max_tokens": self.max_tokens, "messages": Self::convert_messages(messages), "stream": true, @@ -220,7 +235,8 @@ impl BaseModel for AnthropicModel { tool_name: String, input_json: String, } - let mut blocks: std::collections::HashMap = std::collections::HashMap::new(); + let mut blocks: std::collections::HashMap = + std::collections::HashMap::new(); let mut tool_calls: Vec = Vec::new(); use futures::StreamExt; @@ -271,7 +287,8 @@ impl BaseModel for AnthropicModel { match msg_type { "message_start" => { if let Some(usage) = data.pointer("/message/usage") { - if let Some(it) = usage.get("input_tokens").and_then(|v| v.as_u64()) { + if let Some(it) = usage.get("input_tokens").and_then(|v| v.as_u64()) + { input_tokens = it; } } @@ -279,13 +296,24 @@ impl BaseModel for AnthropicModel { "content_block_start" => { let idx = data.get("index").and_then(|i| i.as_u64()).unwrap_or(0); - let block = data.get("content_block").unwrap_or(&serde_json::Value::Null); - let btype = block.get("type").and_then(|t| t.as_str()).unwrap_or("text"); + let block = data + .get("content_block") + .unwrap_or(&serde_json::Value::Null); + let btype = + block.get("type").and_then(|t| t.as_str()).unwrap_or("text"); let state = match btype { "tool_use" => { - let name = block.get("name").and_then(|n| n.as_str()).unwrap_or("").to_string(); - let id = block.get("id").and_then(|i| i.as_str()).unwrap_or("").to_string(); + let name = block + .get("name") + .and_then(|n| n.as_str()) + .unwrap_or("") + .to_string(); + let id = block + .get("id") + .and_then(|i| i.as_str()) + .unwrap_or("") + .to_string(); if !name.is_empty() { on_delta(DeltaEvent { kind: DeltaKind::ToolCallStart, @@ -321,7 +349,8 @@ impl BaseModel for AnthropicModel { Some(d) => d, None => continue, }; - let delta_type = delta.get("type").and_then(|t| t.as_str()).unwrap_or(""); + let delta_type = + delta.get("type").and_then(|t| t.as_str()).unwrap_or(""); match delta_type { "text_delta" => { @@ -336,7 +365,8 @@ impl BaseModel for AnthropicModel { } } "thinking_delta" => { - if let Some(t) = delta.get("thinking").and_then(|t| t.as_str()) { + if let Some(t) = delta.get("thinking").and_then(|t| t.as_str()) + { if !t.is_empty() { thinking.push_str(t); on_delta(DeltaEvent { @@ -347,7 +377,9 @@ impl BaseModel for AnthropicModel { } } "input_json_delta" => { - if let Some(chunk) = delta.get("partial_json").and_then(|j| j.as_str()) { + if let Some(chunk) = + delta.get("partial_json").and_then(|j| j.as_str()) + { if !chunk.is_empty() { if let Some(block) = blocks.get_mut(&idx) { block.input_json.push_str(chunk); @@ -378,7 +410,9 @@ impl BaseModel for AnthropicModel { "message_delta" => { if let Some(usage) = data.get("usage") { - if let Some(ot) = usage.get("output_tokens").and_then(|v| v.as_u64()) { + if let Some(ot) = + usage.get("output_tokens").and_then(|v| v.as_u64()) + { output_tokens = ot; } } @@ -401,7 +435,11 @@ impl BaseModel for AnthropicModel { Ok(ModelTurn { text, - thinking: if thinking.is_empty() { None } else { Some(thinking) }, + thinking: if thinking.is_empty() { + None + } else { + Some(thinking) + }, tool_calls, input_tokens, output_tokens, @@ -436,6 +474,7 @@ mod tests { fn test_is_opus_46() { assert!(make_model("claude-opus-4-6", None).is_opus_46()); assert!(make_model("claude-opus-4.6-20250610", None).is_opus_46()); + assert!(make_model("anthropic-foundry/claude-opus-4-6", None).is_opus_46()); assert!(!make_model("claude-sonnet-4-5", None).is_opus_46()); } @@ -444,15 +483,24 @@ mod tests { #[test] fn test_extract_system_present() { let msgs = vec![ - Message::System { content: "Be helpful.".to_string() }, - Message::User { content: "Hi".to_string() }, + Message::System { + content: "Be helpful.".to_string(), + }, + Message::User { + content: "Hi".to_string(), + }, ]; - assert_eq!(AnthropicModel::extract_system(&msgs), Some("Be helpful.".to_string())); + assert_eq!( + AnthropicModel::extract_system(&msgs), + Some("Be helpful.".to_string()) + ); } #[test] fn test_extract_system_absent() { - let msgs = vec![Message::User { content: "Hi".to_string() }]; + let msgs = vec![Message::User { + content: "Hi".to_string(), + }]; assert_eq!(AnthropicModel::extract_system(&msgs), None); } @@ -461,8 +509,12 @@ mod tests { #[test] fn test_convert_filters_system() { let msgs = vec![ - Message::System { content: "System prompt".to_string() }, - Message::User { content: "Hello".to_string() }, + Message::System { + content: "System prompt".to_string(), + }, + Message::User { + content: "Hello".to_string(), + }, ]; let converted = AnthropicModel::convert_messages(&msgs); assert_eq!(converted.len(), 1); // System is filtered out @@ -508,18 +560,40 @@ mod tests { Message::Assistant { content: "Using tools.".to_string(), tool_calls: Some(vec![ - ToolCall { id: "t1".into(), name: "read_file".into(), arguments: "{}".into() }, - ToolCall { id: "t2".into(), name: "list_files".into(), arguments: "{}".into() }, + ToolCall { + id: "t1".into(), + name: "read_file".into(), + arguments: "{}".into(), + }, + ToolCall { + id: "t2".into(), + name: "list_files".into(), + arguments: "{}".into(), + }, ]), }, - Message::Tool { tool_call_id: "t1".into(), content: "file1 contents".into() }, - Message::Tool { tool_call_id: "t2".into(), content: "file list".into() }, + Message::Tool { + tool_call_id: "t1".into(), + content: "file1 contents".into(), + }, + Message::Tool { + tool_call_id: "t2".into(), + content: "file list".into(), + }, ]; let converted = AnthropicModel::convert_messages(&msgs); // Should be 2 messages: assistant + one merged user - assert_eq!(converted.len(), 2, "consecutive Tool messages should merge into one user message"); + assert_eq!( + converted.len(), + 2, + "consecutive Tool messages should merge into one user message" + ); let user_content = converted[1]["content"].as_array().unwrap(); - assert_eq!(user_content.len(), 2, "merged user message should have 2 tool_result blocks"); + assert_eq!( + user_content.len(), + 2, + "merged user message should have 2 tool_result blocks" + ); assert_eq!(user_content[0]["tool_use_id"], "t1"); assert_eq!(user_content[1]["tool_use_id"], "t2"); } @@ -530,8 +604,12 @@ mod tests { fn test_payload_no_thinking_has_temperature() { let model = make_model("claude-sonnet-4-5", None); let msgs = vec![ - Message::System { content: "System".to_string() }, - Message::User { content: "Hi".to_string() }, + Message::System { + content: "System".to_string(), + }, + Message::User { + content: "Hi".to_string(), + }, ]; let payload = model.build_payload(&msgs, &[]); assert_eq!(payload["temperature"], 0.0); @@ -543,17 +621,31 @@ mod tests { #[test] fn test_payload_opus_46_adaptive_thinking() { let model = make_model("claude-opus-4-6", Some("high")); - let msgs = vec![Message::User { content: "Hi".to_string() }]; + let msgs = vec![Message::User { + content: "Hi".to_string(), + }]; let payload = model.build_payload(&msgs, &[]); assert!(payload.get("temperature").is_none()); // No temperature with thinking assert_eq!(payload["thinking"]["type"], "adaptive"); assert_eq!(payload["output_config"]["effort"], "high"); } + #[test] + fn test_payload_strips_foundry_prefix() { + let model = make_model("anthropic-foundry/claude-opus-4-6", Some("high")); + let msgs = vec![Message::User { + content: "Hi".to_string(), + }]; + let payload = model.build_payload(&msgs, &[]); + assert_eq!(payload["model"], "claude-opus-4-6"); + } + #[test] fn test_payload_older_model_enabled_thinking() { let model = make_model("claude-sonnet-4-5", Some("medium")); - let msgs = vec![Message::User { content: "Hi".to_string() }]; + let msgs = vec![Message::User { + content: "Hi".to_string(), + }]; let payload = model.build_payload(&msgs, &[]); assert_eq!(payload["thinking"]["type"], "enabled"); assert_eq!(payload["thinking"]["budget_tokens"], 4096); @@ -563,8 +655,12 @@ mod tests { fn test_payload_system_extracted_to_top_level() { let model = make_model("claude-sonnet-4-5", None); let msgs = vec![ - Message::System { content: "You are helpful.".to_string() }, - Message::User { content: "Test".to_string() }, + Message::System { + content: "You are helpful.".to_string(), + }, + Message::User { + content: "Test".to_string(), + }, ]; let payload = model.build_payload(&msgs, &[]); // System should be top-level, not in messages array diff --git a/openplanter-desktop/crates/op-core/src/model/mod.rs b/openplanter-desktop/crates/op-core/src/model/mod.rs index 4f2781ec..2ec516ce 100644 --- a/openplanter-desktop/crates/op-core/src/model/mod.rs +++ b/openplanter-desktop/crates/op-core/src/model/mod.rs @@ -1,6 +1,6 @@ +pub mod anthropic; /// Model abstraction layer — trait + provider implementations. pub mod openai; -pub mod anthropic; pub mod sse; use serde::{Deserialize, Serialize}; @@ -8,6 +8,17 @@ use serde::{Deserialize, Serialize}; use crate::events::DeltaEvent; use tokio_util::sync::CancellationToken; +/// Structured model error for provider rate limiting. +#[derive(Debug, Clone, thiserror::Error)] +#[error("{message}")] +pub struct RateLimitError { + pub message: String, + pub status_code: Option, + pub provider_code: Option, + pub body: String, + pub retry_after_sec: Option, +} + /// A single tool call returned by the model. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ToolCall { @@ -35,16 +46,26 @@ pub enum Message { #[serde(rename = "user")] User { content: String }, #[serde(rename = "assistant")] - Assistant { content: String, tool_calls: Option> }, + Assistant { + content: String, + tool_calls: Option>, + }, #[serde(rename = "tool")] - Tool { tool_call_id: String, content: String }, + Tool { + tool_call_id: String, + content: String, + }, } /// Trait for LLM model implementations. #[async_trait::async_trait] pub trait BaseModel: Send + Sync { /// Send a conversation and return the model's turn. - async fn chat(&self, messages: &[Message], tools: &[serde_json::Value]) -> anyhow::Result; + async fn chat( + &self, + messages: &[Message], + tools: &[serde_json::Value], + ) -> anyhow::Result; /// Send a conversation with streaming deltas and cancellation support. async fn chat_stream( diff --git a/openplanter-desktop/crates/op-core/src/model/openai.rs b/openplanter-desktop/crates/op-core/src/model/openai.rs index 4b1353f6..b3fb5ad4 100644 --- a/openplanter-desktop/crates/op-core/src/model/openai.rs +++ b/openplanter-desktop/crates/op-core/src/model/openai.rs @@ -1,15 +1,32 @@ // OpenAI-compatible model implementation. // -// Handles openai, openrouter, cerebras, and ollama — all use /chat/completions. +// Handles openai, openrouter, cerebras, zai, and ollama via /chat/completions. use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use std::time::Duration; -use anyhow::{anyhow, Context}; +use anyhow::{Context, anyhow}; +use chrono::{DateTime, Utc}; use reqwest_eventsource::{Event, RequestBuilderExt}; +use tokio::time::sleep; use tokio_util::sync::CancellationToken; +use super::{BaseModel, Message, ModelTurn, RateLimitError, ToolCall}; +use crate::config::strip_foundry_model_prefix; use crate::events::{DeltaEvent, DeltaKind}; -use super::{BaseModel, Message, ModelTurn, ToolCall}; + +#[derive(Debug, Clone, Default)] +pub struct ZaiRuntimeConfig { + pub paygo_base_url: String, + pub coding_base_url: String, + pub stream_max_retries: usize, +} + +struct StreamAttemptError { + error: anyhow::Error, + saw_output: bool, +} pub struct OpenAIModel { client: reqwest::Client, @@ -19,6 +36,10 @@ pub struct OpenAIModel { api_key: String, reasoning_effort: Option, extra_headers: HashMap, + thinking_type: Option, + stream_max_retries: usize, + fallback_base_urls: Vec, + active_base_url: Arc>, } impl OpenAIModel { @@ -34,18 +55,53 @@ impl OpenAIModel { client: reqwest::Client::new(), model, provider, - base_url, + base_url: base_url.clone(), api_key, reasoning_effort, extra_headers, + thinking_type: None, + stream_max_retries: 1, + fallback_base_urls: Vec::new(), + active_base_url: Arc::new(RwLock::new(base_url)), } } + pub fn with_zai_runtime(mut self, config: ZaiRuntimeConfig) -> Self { + let effort = self + .reasoning_effort + .as_deref() + .unwrap_or_default() + .trim() + .to_lowercase(); + self.thinking_type = Some(if effort.is_empty() || effort == "none" { + "disabled".to_string() + } else { + "enabled".to_string() + }); + self.stream_max_retries = config.stream_max_retries.max(1); + + let mut fallbacks = Vec::new(); + for candidate in [config.paygo_base_url, config.coding_base_url] { + let trimmed = candidate.trim(); + if trimmed.is_empty() { + continue; + } + if !fallbacks.iter().any(|url| url == trimmed) { + fallbacks.push(trimmed.to_string()); + } + } + self.fallback_base_urls = fallbacks; + self + } + fn is_reasoning_model(&self) -> bool { - let lower = self.model.to_lowercase(); - if lower.starts_with("o1-") || lower == "o1" - || lower.starts_with("o3-") || lower == "o3" - || lower.starts_with("o4-") || lower == "o4" + let lower = self.request_model_name().to_lowercase(); + if lower.starts_with("o1-") + || lower == "o1" + || lower.starts_with("o3-") + || lower == "o3" + || lower.starts_with("o4-") + || lower == "o4" { return true; } @@ -55,6 +111,10 @@ impl OpenAIModel { false } + fn request_model_name(&self) -> String { + strip_foundry_model_prefix(&self.model) + } + fn convert_messages(messages: &[Message]) -> Vec { messages .iter() @@ -67,7 +127,10 @@ impl OpenAIModel { "role": "user", "content": content, }), - Message::Assistant { content, tool_calls } => { + Message::Assistant { + content, + tool_calls, + } => { let mut obj = serde_json::json!({ "role": "assistant", "content": content, @@ -75,20 +138,25 @@ impl OpenAIModel { if let Some(tcs) = tool_calls { let tc_arr: Vec = tcs .iter() - .map(|tc| serde_json::json!({ - "id": tc.id, - "type": "function", - "function": { - "name": tc.name, - "arguments": tc.arguments, - } - })) + .map(|tc| { + serde_json::json!({ + "id": tc.id, + "type": "function", + "function": { + "name": tc.name, + "arguments": tc.arguments, + } + }) + }) .collect(); obj["tool_calls"] = serde_json::Value::Array(tc_arr); } obj } - Message::Tool { tool_call_id, content } => serde_json::json!({ + Message::Tool { + tool_call_id, + content, + } => serde_json::json!({ "role": "tool", "tool_call_id": tool_call_id, "content": content, @@ -104,13 +172,13 @@ impl OpenAIModel { stream: bool, ) -> serde_json::Value { let mut payload = serde_json::json!({ - "model": self.model, + "model": self.request_model_name(), "messages": Self::convert_messages(messages), }); if stream { payload["stream"] = serde_json::json!(true); - payload["stream_options"] = serde_json::json!({"include_usage": true}); + payload["stream_options"] = serde_json::json!({ "include_usage": true }); } if !tools.is_empty() { @@ -129,36 +197,20 @@ impl OpenAIModel { } } - payload - } -} + if let Some(ref thinking_type) = self.thinking_type { + let value = thinking_type.trim().to_lowercase(); + if matches!(value.as_str(), "enabled" | "disabled") { + payload["thinking"] = serde_json::json!({ "type": value }); + } + } -#[async_trait::async_trait] -impl BaseModel for OpenAIModel { - async fn chat( - &self, - messages: &[Message], - tools: &[serde_json::Value], - ) -> anyhow::Result { - // Default: call chat_stream with a no-op callback - let noop = |_: DeltaEvent| {}; - let cancel = CancellationToken::new(); - self.chat_stream(messages, tools, &noop, &cancel).await + payload } - async fn chat_stream( - &self, - messages: &[Message], - tools: &[serde_json::Value], - on_delta: &(dyn Fn(DeltaEvent) + Send + Sync), - cancel: &CancellationToken, - ) -> anyhow::Result { - let url = format!("{}/chat/completions", self.base_url.trim_end_matches('/')); - let payload = self.build_payload(messages, tools, true); - + fn build_request(&self, url: &str, payload: &serde_json::Value) -> reqwest::RequestBuilder { let mut request = self .client - .post(&url) + .post(url) .header("Authorization", format!("Bearer {}", self.api_key)) .header("Content-Type", "application/json"); @@ -166,24 +218,301 @@ impl BaseModel for OpenAIModel { request = request.header(k.as_str(), v.as_str()); } - let mut es = request.json(&payload).eventsource()?; + request.json(payload) + } + + fn current_base_url(&self) -> String { + self.active_base_url + .read() + .map(|value| value.clone()) + .unwrap_or_else(|_| self.base_url.clone()) + } + + fn set_active_base_url(&self, base_url: &str) { + if let Ok(mut guard) = self.active_base_url.write() { + *guard = base_url.to_string(); + } + } + + fn candidate_base_urls(&self) -> Vec { + let mut urls = Vec::new(); + let current = self.current_base_url(); + urls.push(current); + for candidate in &self.fallback_base_urls { + if !urls.iter().any(|url| url == candidate) { + urls.push(candidate.clone()); + } + } + urls + } + + fn should_try_next_zai_base_url(&self, err: &anyhow::Error) -> bool { + if self.provider != "zai" { + return false; + } + let text = err.to_string().to_lowercase(); + text.contains("404") || text.contains("405") || text.contains("not found") + } + + fn should_retry_zai_error(&self, err: &StreamAttemptError) -> bool { + if self.provider != "zai" || err.saw_output { + return false; + } + if err.error.downcast_ref::().is_some() { + return true; + } + let text = err.error.to_string().to_lowercase(); + text.contains("429") + || text.contains("1302") + || text.contains("rate limit") + || text.contains("too many requests") + || text.contains("connection") + || text.contains("timed out") + || text.contains("timeout") + || text.contains("stream ended") + || text.contains("broken pipe") + || text.contains("500") + || text.contains("502") + || text.contains("503") + || text.contains("504") + } + + fn parse_retry_after_value(value: Option<&serde_json::Value>) -> Option { + match value { + Some(serde_json::Value::Number(num)) => num.as_f64().map(|v| v.max(0.0)), + Some(serde_json::Value::String(text)) => Self::parse_retry_after_text(text), + _ => None, + } + } + + fn parse_retry_after_text(text: &str) -> Option { + let trimmed = text.trim(); + if trimmed.is_empty() { + return None; + } + if let Ok(seconds) = trimmed.parse::() { + return Some(seconds.max(0.0)); + } + let parsed = DateTime::parse_from_rfc2822(trimmed).ok()?; + Some( + (parsed.with_timezone(&Utc) - Utc::now()) + .num_milliseconds() + .max(0) as f64 + / 1000.0, + ) + } + + fn parse_retry_after_header(headers: &reqwest::header::HeaderMap) -> Option { + let value = headers.get(reqwest::header::RETRY_AFTER)?; + let text = value.to_str().ok()?; + Self::parse_retry_after_text(text) + } + + fn extract_provider_code(value: Option<&serde_json::Value>) -> Option { + match value { + Some(serde_json::Value::String(text)) => { + let trimmed = text.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) + } + Some(serde_json::Value::Number(num)) => Some(num.to_string()), + Some(other) => { + let rendered = other.to_string(); + let trimmed = rendered.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) + } + None => None, + } + } + + fn extract_openai_style_error( + payload: &serde_json::Value, + ) -> (String, Option, Option) { + if let Some(error) = payload.get("error").and_then(|value| value.as_object()) { + let message = error + .get("message") + .and_then(|value| value.as_str()) + .unwrap_or_default() + .trim() + .to_string(); + let provider_code = Self::extract_provider_code(error.get("code")); + let retry_after = Self::parse_retry_after_value(error.get("retry_after")) + .or_else(|| Self::parse_retry_after_value(payload.get("retry_after"))); + return (message, provider_code, retry_after); + } + ( + String::new(), + None, + Self::parse_retry_after_value(payload.get("retry_after")), + ) + } + + fn is_rate_limit_error( + status_code: Option, + provider_code: Option<&str>, + message: &str, + ) -> bool { + if status_code == Some(429) { + return true; + } + if let Some(code) = provider_code { + let code = code.trim().to_lowercase(); + if matches!( + code.as_str(), + "1302" | "429" | "rate_limit" | "rate_limit_exceeded" | "too_many_requests" + ) { + return true; + } + } + let text = message.to_lowercase(); + text.contains("rate limit") || text.contains("too many requests") + } + + fn classify_stream_payload_error(payload: &serde_json::Value) -> Option { + let is_error_type = payload + .get("type") + .and_then(|value| value.as_str()) + .is_some_and(|value| value == "error"); + let error = payload.get("error")?; + let message = error + .get("message") + .and_then(|value| value.as_str()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| payload.to_string()); + let provider_code = Self::extract_provider_code(error.get("code")); + let retry_after = Self::parse_retry_after_value(error.get("retry_after")); + let prefixed_message = format!("Stream error: {message}"); + + if Self::is_rate_limit_error(None, provider_code.as_deref(), &message) { + return Some(anyhow::Error::new(RateLimitError { + message: prefixed_message, + status_code: None, + provider_code, + body: payload.to_string(), + retry_after_sec: retry_after, + })); + } + + if is_error_type || provider_code.is_some() || payload.get("retry_after").is_some() { + return Some(anyhow!(prefixed_message)); + } + + None + } + + async fn classify_sse_error( + &self, + url: &str, + error: reqwest_eventsource::Error, + ) -> anyhow::Error { + match error { + reqwest_eventsource::Error::InvalidStatusCode(status, response) => { + self.classify_invalid_status(url, status, response).await + } + other => anyhow!("SSE stream error: {other}"), + } + } + + async fn classify_invalid_status( + &self, + url: &str, + status: reqwest::StatusCode, + response: reqwest::Response, + ) -> anyhow::Error { + let response_url = response.url().clone(); + let headers = response.headers().clone(); + let body = response.text().await.unwrap_or_default(); + let parsed = serde_json::from_str::(&body).ok(); + let mut message = String::new(); + let mut provider_code = None; + let mut retry_after = Self::parse_retry_after_header(&headers); + + if let Some(ref payload) = parsed { + let (body_message, body_code, body_retry_after) = + Self::extract_openai_style_error(payload); + message = body_message; + provider_code = body_code; + if retry_after.is_none() { + retry_after = body_retry_after; + } + } + + let detail = if !message.is_empty() { + message.clone() + } else if !body.trim().is_empty() { + body.clone() + } else { + status.to_string() + }; + + if Self::is_rate_limit_error(Some(status.as_u16()), provider_code.as_deref(), &detail) { + return anyhow::Error::new(RateLimitError { + message: format!( + "HTTP {} calling {}: {}", + status.as_u16(), + response_url, + detail + ), + status_code: Some(status.as_u16()), + provider_code, + body, + retry_after_sec: retry_after, + }); + } + + anyhow!( + "HTTP {} calling {}: {}", + status.as_u16(), + if response_url.as_str().is_empty() { + url + } else { + response_url.as_str() + }, + detail + ) + } + + async fn chat_stream_once( + &self, + base_url: &str, + messages: &[Message], + tools: &[serde_json::Value], + on_delta: &(dyn Fn(DeltaEvent) + Send + Sync), + cancel: &CancellationToken, + ) -> Result { + let url = format!("{}/chat/completions", base_url.trim_end_matches('/')); + let payload = self.build_payload(messages, tools, true); + let request = self.build_request(&url, &payload); + let mut es = request.eventsource().map_err(|e| StreamAttemptError { + error: anyhow!("Failed to open SSE stream: {e}"), + saw_output: false, + })?; let mut text = String::new(); - let mut tool_calls_by_index: HashMap = HashMap::new(); // (id, name, args) + let mut thinking = String::new(); + let mut tool_calls_by_index: HashMap = HashMap::new(); let mut input_tokens: u64 = 0; let mut output_tokens: u64 = 0; + let mut saw_output = false; use futures::StreamExt; loop { if cancel.is_cancelled() { es.close(); - return Err(anyhow!("Cancelled")); + return Err(StreamAttemptError { + error: anyhow!("Cancelled"), + saw_output, + }); } let event = tokio::select! { _ = cancel.cancelled() => { es.close(); - return Err(anyhow!("Cancelled")); + return Err(StreamAttemptError { + error: anyhow!("Cancelled"), + saw_output, + }); } ev = es.next() => ev, }; @@ -193,7 +522,8 @@ impl BaseModel for OpenAIModel { Some(Err(reqwest_eventsource::Error::StreamEnded)) => break, Some(Err(e)) => { es.close(); - return Err(anyhow!("SSE stream error: {e}")); + let error = self.classify_sse_error(&url, e).await; + return Err(StreamAttemptError { error, saw_output }); } None => break, }; @@ -206,9 +536,13 @@ impl BaseModel for OpenAIModel { } let chunk: serde_json::Value = serde_json::from_str(&msg.data) - .with_context(|| format!("Failed to parse SSE chunk: {}", &msg.data))?; + .with_context(|| format!("Failed to parse SSE chunk: {}", &msg.data)) + .map_err(|error| StreamAttemptError { error, saw_output })?; + + if let Some(error) = Self::classify_stream_payload_error(&chunk) { + return Err(StreamAttemptError { error, saw_output }); + } - // Extract usage from any chunk that has it if let Some(usage) = chunk.get("usage") { if let Some(pt) = usage.get("prompt_tokens").and_then(|v| v.as_u64()) { input_tokens = pt; @@ -222,7 +556,6 @@ impl BaseModel for OpenAIModel { Some(c) => c, None => continue, }; - if choices.is_empty() { continue; } @@ -232,9 +565,9 @@ impl BaseModel for OpenAIModel { None => continue, }; - // Text content delta if let Some(content) = delta.get("content").and_then(|c| c.as_str()) { if !content.is_empty() { + saw_output = true; text.push_str(content); on_delta(DeltaEvent { kind: DeltaKind::Text, @@ -243,13 +576,26 @@ impl BaseModel for OpenAIModel { } } - // Tool call deltas + for field in ["reasoning_content", "reasoning", "thinking"] { + if let Some(value) = delta.get(field).and_then(|c| c.as_str()) { + if !value.is_empty() { + saw_output = true; + thinking.push_str(value); + on_delta(DeltaEvent { + kind: DeltaKind::Thinking, + text: value.to_string(), + }); + } + } + } + if let Some(tc_deltas) = delta.get("tool_calls").and_then(|t| t.as_array()) { for tc_delta in tc_deltas { - let idx = tc_delta.get("index").and_then(|i| i.as_u64()).unwrap_or(0) as usize; - let entry = tool_calls_by_index.entry(idx).or_insert_with(|| { - (String::new(), String::new(), String::new()) - }); + let idx = tc_delta.get("index").and_then(|i| i.as_u64()).unwrap_or(0) + as usize; + let entry = tool_calls_by_index + .entry(idx) + .or_insert_with(|| (String::new(), String::new(), String::new())); if let Some(id) = tc_delta.get("id").and_then(|i| i.as_str()) { if !id.is_empty() { @@ -260,6 +606,7 @@ impl BaseModel for OpenAIModel { if let Some(func) = tc_delta.get("function") { if let Some(name) = func.get("name").and_then(|n| n.as_str()) { if !name.is_empty() { + saw_output = true; entry.1 = name.to_string(); on_delta(DeltaEvent { kind: DeltaKind::ToolCallStart, @@ -269,6 +616,7 @@ impl BaseModel for OpenAIModel { } if let Some(args) = func.get("arguments").and_then(|a| a.as_str()) { if !args.is_empty() { + saw_output = true; entry.2.push_str(args); on_delta(DeltaEvent { kind: DeltaKind::ToolCallArgs, @@ -283,23 +631,96 @@ impl BaseModel for OpenAIModel { } } - // Build tool calls from accumulated data let mut tool_calls: Vec = Vec::new(); let mut indices: Vec = tool_calls_by_index.keys().copied().collect(); indices.sort(); for idx in indices { let (id, name, arguments) = tool_calls_by_index.remove(&idx).unwrap(); - tool_calls.push(ToolCall { id, name, arguments }); + tool_calls.push(ToolCall { + id, + name, + arguments, + }); } Ok(ModelTurn { text, - thinking: None, + thinking: if thinking.is_empty() { + None + } else { + Some(thinking) + }, tool_calls, input_tokens, output_tokens, }) } +} + +#[async_trait::async_trait] +impl BaseModel for OpenAIModel { + async fn chat( + &self, + messages: &[Message], + tools: &[serde_json::Value], + ) -> anyhow::Result { + let noop = |_: DeltaEvent| {}; + let cancel = CancellationToken::new(); + self.chat_stream(messages, tools, &noop, &cancel).await + } + + async fn chat_stream( + &self, + messages: &[Message], + tools: &[serde_json::Value], + on_delta: &(dyn Fn(DeltaEvent) + Send + Sync), + cancel: &CancellationToken, + ) -> anyhow::Result { + let max_attempts = if self.provider == "zai" { + self.stream_max_retries.max(1) + } else { + 1 + }; + let mut last_error: Option = None; + + for attempt in 0..max_attempts { + for base_url in self.candidate_base_urls() { + match self + .chat_stream_once(&base_url, messages, tools, on_delta, cancel) + .await + { + Ok(turn) => { + self.set_active_base_url(&base_url); + return Ok(turn); + } + Err(err) => { + let should_try_next = self.should_try_next_zai_base_url(&err.error); + let should_retry = self.should_retry_zai_error(&err); + last_error = Some(err.error); + + if should_try_next { + continue; + } + + if should_retry && attempt + 1 < max_attempts { + break; + } + + return Err(last_error + .take() + .unwrap_or_else(|| anyhow!("OpenAI-compatible request failed"))); + } + } + } + + if attempt + 1 < max_attempts { + let backoff_ms = (250_u64 << attempt.min(3)).min(2_000); + sleep(Duration::from_millis(backoff_ms)).await; + } + } + + Err(last_error.unwrap_or_else(|| anyhow!("OpenAI-compatible request failed"))) + } fn model_name(&self) -> &str { &self.model @@ -325,8 +746,6 @@ mod tests { ) } - // ── is_reasoning_model ── - #[test] fn test_reasoning_model_o1() { assert!(make_model("o1", None).is_reasoning_model()); @@ -343,6 +762,7 @@ mod tests { fn test_reasoning_model_gpt5() { assert!(make_model("gpt-5.2", None).is_reasoning_model()); assert!(make_model("gpt-5", None).is_reasoning_model()); + assert!(make_model("azure-foundry/gpt-5.3-codex", None).is_reasoning_model()); } #[test] @@ -351,8 +771,6 @@ mod tests { assert!(!make_model("claude-opus-4-6", None).is_reasoning_model()); } - // ── convert_messages ── - #[test] fn test_convert_system_message() { let msgs = vec![Message::System { @@ -405,8 +823,6 @@ mod tests { assert_eq!(converted[0]["content"], "file contents"); } - // ── build_payload ── - #[test] fn test_payload_non_reasoning_has_temperature() { let model = make_model("gpt-4o", None); @@ -430,6 +846,16 @@ mod tests { assert_eq!(payload["reasoning_effort"], "high"); } + #[test] + fn test_payload_strips_foundry_prefix() { + let model = make_model("azure-foundry/gpt-5.3-codex", Some("high")); + let msgs = vec![Message::User { + content: "Hi".to_string(), + }]; + let payload = model.build_payload(&msgs, &[], true); + assert_eq!(payload["model"], "gpt-5.3-codex"); + } + #[test] fn test_payload_with_tools() { let model = make_model("gpt-4o", None); @@ -453,7 +879,100 @@ mod tests { assert!(payload.get("tool_choice").is_none()); } - // ── model_name / provider_name ── + #[test] + fn test_payload_zai_includes_thinking() { + let model = OpenAIModel::new( + "glm-5".to_string(), + "zai".to_string(), + "https://api.z.ai/api/paas/v4".to_string(), + "zai-key".to_string(), + Some("high".to_string()), + HashMap::new(), + ) + .with_zai_runtime(ZaiRuntimeConfig { + paygo_base_url: "https://api.z.ai/api/paas/v4".to_string(), + coding_base_url: "https://api.z.ai/api/coding/paas/v4".to_string(), + stream_max_retries: 4, + }); + let msgs = vec![Message::User { + content: "Hi".to_string(), + }]; + let payload = model.build_payload(&msgs, &[], true); + assert_eq!(payload["thinking"]["type"], "enabled"); + } + + #[test] + fn test_zai_runtime_switches_to_disabled_when_no_effort() { + let model = OpenAIModel::new( + "glm-5".to_string(), + "zai".to_string(), + "https://api.z.ai/api/paas/v4".to_string(), + "zai-key".to_string(), + None, + HashMap::new(), + ) + .with_zai_runtime(ZaiRuntimeConfig { + paygo_base_url: "https://api.z.ai/api/paas/v4".to_string(), + coding_base_url: "https://api.z.ai/api/coding/paas/v4".to_string(), + stream_max_retries: 4, + }); + let msgs = vec![Message::User { + content: "Hi".to_string(), + }]; + let payload = model.build_payload(&msgs, &[], true); + assert_eq!(payload["thinking"]["type"], "disabled"); + } + + #[test] + fn test_zai_candidate_base_urls_prefers_active() { + let model = OpenAIModel::new( + "glm-5".to_string(), + "zai".to_string(), + "https://api.z.ai/api/paas/v4".to_string(), + "zai-key".to_string(), + Some("medium".to_string()), + HashMap::new(), + ) + .with_zai_runtime(ZaiRuntimeConfig { + paygo_base_url: "https://api.z.ai/api/paas/v4".to_string(), + coding_base_url: "https://api.z.ai/api/coding/paas/v4".to_string(), + stream_max_retries: 4, + }); + model.set_active_base_url("https://api.z.ai/api/coding/paas/v4"); + assert_eq!( + model.candidate_base_urls(), + vec![ + "https://api.z.ai/api/coding/paas/v4".to_string(), + "https://api.z.ai/api/paas/v4".to_string(), + ] + ); + } + + #[test] + fn test_retry_after_parses_seconds_and_http_dates() { + assert_eq!(OpenAIModel::parse_retry_after_text("3"), Some(3.0)); + assert!(OpenAIModel::parse_retry_after_text("Wed, 21 Oct 2015 07:28:00 GMT").is_some()); + assert_eq!(OpenAIModel::parse_retry_after_text(""), None); + } + + #[test] + fn test_classify_stream_payload_rate_limit_error() { + let payload = serde_json::json!({ + "type": "error", + "error": { + "message": "Too many requests", + "code": "1302", + "retry_after": 4 + } + }); + let error = OpenAIModel::classify_stream_payload_error(&payload) + .expect("payload should classify as an error"); + let rate_limit = error + .downcast_ref::() + .expect("expected a structured rate-limit error"); + assert_eq!(rate_limit.provider_code.as_deref(), Some("1302")); + assert_eq!(rate_limit.retry_after_sec, Some(4.0)); + } #[test] fn test_model_name_and_provider() { diff --git a/openplanter-desktop/crates/op-core/src/session/mod.rs b/openplanter-desktop/crates/op-core/src/session/mod.rs index 83085b45..3ef1e89c 100644 --- a/openplanter-desktop/crates/op-core/src/session/mod.rs +++ b/openplanter-desktop/crates/op-core/src/session/mod.rs @@ -1,6 +1,6 @@ +pub mod credentials; /// Session store and runtime. /// /// Full implementation in Phase 5. pub mod replay; pub mod settings; -pub mod credentials; diff --git a/openplanter-desktop/crates/op-core/src/session/replay.rs b/openplanter-desktop/crates/op-core/src/session/replay.rs index 367c27cc..d347874a 100644 --- a/openplanter-desktop/crates/op-core/src/session/replay.rs +++ b/openplanter-desktop/crates/op-core/src/session/replay.rs @@ -179,13 +179,11 @@ mod tests { step_tokens_out: Some(2100), step_elapsed: Some(5000), step_model_preview: Some("The analysis shows...".into()), - step_tool_calls: Some(vec![ - StepToolCallEntry { - name: "read_file".into(), - key_arg: "/src/main.ts".into(), - elapsed: 1200, - }, - ]), + step_tool_calls: Some(vec![StepToolCallEntry { + name: "read_file".into(), + key_arg: "/src/main.ts".into(), + elapsed: 1200, + }]), }; logger.append(entry).await.unwrap(); @@ -218,7 +216,8 @@ mod tests { step_elapsed: None, step_model_preview: None, step_tool_calls: None, - }).unwrap(), + }) + .unwrap(), serde_json::to_string(&ReplayEntry { seq: 2, timestamp: "2026-01-01T00:01:00Z".into(), @@ -232,7 +231,8 @@ mod tests { step_elapsed: None, step_model_preview: None, step_tool_calls: None, - }).unwrap(), + }) + .unwrap(), ); fs::write(&path, content).await.unwrap(); @@ -285,7 +285,9 @@ mod tests { }; logger.append(entry).await.unwrap(); - let content = fs::read_to_string(tmp.path().join("replay.jsonl")).await.unwrap(); + let content = fs::read_to_string(tmp.path().join("replay.jsonl")) + .await + .unwrap(); assert!(!content.contains("tool_name")); assert!(!content.contains("step_number")); assert!(!content.contains("step_tool_calls")); diff --git a/openplanter-desktop/crates/op-core/src/settings.rs b/openplanter-desktop/crates/op-core/src/settings.rs index 69fcd320..de0688f6 100644 --- a/openplanter-desktop/crates/op-core/src/settings.rs +++ b/openplanter-desktop/crates/op-core/src/settings.rs @@ -4,6 +4,8 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; +use crate::config::{normalize_web_search_provider, normalize_zai_plan}; + const VALID_REASONING_EFFORTS: &[&str] = &["low", "medium", "high"]; /// Normalize and validate a reasoning effort value. @@ -38,7 +40,10 @@ pub struct PersistentSettings { pub default_model_anthropic: Option, pub default_model_openrouter: Option, pub default_model_cerebras: Option, + pub default_model_zai: Option, pub default_model_ollama: Option, + pub zai_plan: Option, + pub web_search_provider: Option, } impl PersistentSettings { @@ -49,6 +54,7 @@ impl PersistentSettings { "anthropic" => self.default_model_anthropic.as_deref(), "openrouter" => self.default_model_openrouter.as_deref(), "cerebras" => self.default_model_cerebras.as_deref(), + "zai" => self.default_model_zai.as_deref(), "ollama" => self.default_model_ollama.as_deref(), _ => None, }; @@ -67,8 +73,16 @@ impl PersistentSettings { .filter(|s| !s.is_empty()) .map(String::from); - let effort = - normalize_reasoning_effort(self.default_reasoning_effort.as_deref())?; + let effort = normalize_reasoning_effort(self.default_reasoning_effort.as_deref())?; + + let web_search_provider = self + .web_search_provider + .as_deref() + .map(|value| normalize_web_search_provider(Some(value))); + let zai_plan = self + .zai_plan + .as_deref() + .map(|value| normalize_zai_plan(Some(value))); fn trim_opt(v: &Option) -> Option { v.as_deref() @@ -84,7 +98,10 @@ impl PersistentSettings { default_model_anthropic: trim_opt(&self.default_model_anthropic), default_model_openrouter: trim_opt(&self.default_model_openrouter), default_model_cerebras: trim_opt(&self.default_model_cerebras), + default_model_zai: trim_opt(&self.default_model_zai), default_model_ollama: trim_opt(&self.default_model_ollama), + zai_plan, + web_search_provider, }) } @@ -104,7 +121,10 @@ impl PersistentSettings { add!(default_model_anthropic, "default_model_anthropic"); add!(default_model_openrouter, "default_model_openrouter"); add!(default_model_cerebras, "default_model_cerebras"); + add!(default_model_zai, "default_model_zai"); add!(default_model_ollama, "default_model_ollama"); + add!(zai_plan, "zai_plan"); + add!(web_search_provider, "web_search_provider"); payload } @@ -129,7 +149,10 @@ impl PersistentSettings { default_model_anthropic: get_str(obj, "default_model_anthropic"), default_model_openrouter: get_str(obj, "default_model_openrouter"), default_model_cerebras: get_str(obj, "default_model_cerebras"), + default_model_zai: get_str(obj, "default_model_zai"), default_model_ollama: get_str(obj, "default_model_ollama"), + zai_plan: get_str(obj, "zai_plan"), + web_search_provider: get_str(obj, "web_search_provider"), }; settings.normalized() } @@ -165,9 +188,9 @@ impl SettingsStore { } pub fn save(&self, settings: &PersistentSettings) -> std::io::Result<()> { - let normalized = settings.normalized().map_err(|e| { - std::io::Error::new(std::io::ErrorKind::InvalidInput, e) - })?; + let normalized = settings + .normalized() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; let json = serde_json::to_string_pretty(&normalized.to_json()) .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; fs::write(&self.settings_path, json) @@ -213,6 +236,7 @@ mod tests { let settings = PersistentSettings { default_model: Some("global-model".into()), default_model_openai: Some("gpt-5.2".into()), + default_model_zai: Some("glm-5".into()), ..Default::default() }; assert_eq!( @@ -223,6 +247,7 @@ mod tests { settings.default_model_for_provider("anthropic"), Some("global-model") ); + assert_eq!(settings.default_model_for_provider("zai"), Some("glm-5")); assert_eq!( settings.default_model_for_provider("unknown"), Some("global-model") @@ -236,12 +261,18 @@ mod tests { let settings = PersistentSettings { default_model: Some("gpt-5.2".into()), default_reasoning_effort: Some("high".into()), + default_model_zai: Some("glm-5".into()), + zai_plan: Some("coding".into()), + web_search_provider: Some("firecrawl".into()), ..Default::default() }; store.save(&settings).unwrap(); let loaded = store.load(); assert_eq!(loaded.default_model, Some("gpt-5.2".into())); assert_eq!(loaded.default_reasoning_effort, Some("high".into())); + assert_eq!(loaded.default_model_zai, Some("glm-5".into())); + assert_eq!(loaded.zai_plan, Some("coding".into())); + assert_eq!(loaded.web_search_provider, Some("firecrawl".into())); } #[test] @@ -270,6 +301,9 @@ mod tests { default_model: Some("gpt-5.2".into()), default_reasoning_effort: Some("high".into()), default_model_openai: Some("gpt-5.2".into()), + default_model_zai: Some("glm-5".into()), + zai_plan: Some("coding".into()), + web_search_provider: Some("firecrawl".into()), ..Default::default() }; let json_val = serde_json::to_value(settings.to_json()).unwrap(); @@ -277,5 +311,28 @@ mod tests { assert_eq!(loaded.default_model, Some("gpt-5.2".into())); assert_eq!(loaded.default_reasoning_effort, Some("high".into())); assert_eq!(loaded.default_model_openai, Some("gpt-5.2".into())); + assert_eq!(loaded.default_model_zai, Some("glm-5".into())); + assert_eq!(loaded.zai_plan, Some("coding".into())); + assert_eq!(loaded.web_search_provider, Some("firecrawl".into())); + } + + #[test] + fn test_web_search_provider_normalized() { + let settings = PersistentSettings { + web_search_provider: Some("unexpected".into()), + ..Default::default() + }; + let normalized = settings.normalized().unwrap(); + assert_eq!(normalized.web_search_provider, Some("exa".into())); + } + + #[test] + fn test_zai_plan_normalized() { + let settings = PersistentSettings { + zai_plan: Some("unexpected".into()), + ..Default::default() + }; + let normalized = settings.normalized().unwrap(); + assert_eq!(normalized.zai_plan, Some("paygo".into())); } } diff --git a/openplanter-desktop/crates/op-core/src/tools/defs.rs b/openplanter-desktop/crates/op-core/src/tools/defs.rs index 9f630fcb..e0fe40cf 100644 --- a/openplanter-desktop/crates/op-core/src/tools/defs.rs +++ b/openplanter-desktop/crates/op-core/src/tools/defs.rs @@ -2,8 +2,7 @@ /// /// Single source of truth for tool schemas. Converter helpers produce the /// provider-specific shapes expected by OpenAI and Anthropic APIs. - -use serde_json::{json, Value}; +use serde_json::{Value, json}; struct ToolDef { name: &'static str, @@ -177,7 +176,7 @@ fn mvp_tool_defs() -> Vec { // ── Web ── ToolDef { name: "web_search", - description: "Search the web using the Exa API. Returns URLs, titles, and optional page text.", + description: "Search the web using the configured Exa or Firecrawl backend. Returns URLs, titles, snippets, and optional page text.", parameters: json!({ "type": "object", "properties": { @@ -200,7 +199,7 @@ fn mvp_tool_defs() -> Vec { }, ToolDef { name: "fetch_url", - description: "Fetch and return the text content of one or more URLs.", + description: "Fetch and return the text content of one or more URLs using the configured Exa or Firecrawl backend.", parameters: json!({ "type": "object", "properties": { @@ -297,7 +296,11 @@ fn mvp_tool_defs() -> Vec { /// For OpenAI strict mode: make all properties required, wrapping optional ones /// with `anyOf [original, null]`. Recurse into nested objects and array items. fn strict_fixup(schema: &mut Value) { - let Some(schema_type) = schema.get("type").and_then(|t| t.as_str()).map(String::from) else { + let Some(schema_type) = schema + .get("type") + .and_then(|t| t.as_str()) + .map(String::from) + else { return; }; @@ -524,9 +527,15 @@ mod tests { fn test_strict_fixup_wraps_optional_with_anyof() { // list_files has only optional "glob" parameter let tools = to_openai_tools(); - let list_files = tools.iter().find(|t| t["function"]["name"] == "list_files").unwrap(); + let list_files = tools + .iter() + .find(|t| t["function"]["name"] == "list_files") + .unwrap(); let glob_prop = &list_files["function"]["parameters"]["properties"]["glob"]; - assert!(glob_prop.get("anyOf").is_some(), "Optional 'glob' should be wrapped with anyOf"); + assert!( + glob_prop.get("anyOf").is_some(), + "Optional 'glob' should be wrapped with anyOf" + ); } #[test] @@ -534,7 +543,8 @@ mod tests { let tools = build_curator_tool_defs("openai"); assert_eq!(tools.len(), 8, "curator should have exactly 8 tools"); - let names: Vec = tools.iter() + let names: Vec = tools + .iter() .map(|t| t["function"]["name"].as_str().unwrap().to_string()) .collect(); diff --git a/openplanter-desktop/crates/op-core/src/tools/filesystem.rs b/openplanter-desktop/crates/op-core/src/tools/filesystem.rs index bee02d2f..2c67d62b 100644 --- a/openplanter-desktop/crates/op-core/src/tools/filesystem.rs +++ b/openplanter-desktop/crates/op-core/src/tools/filesystem.rs @@ -1,5 +1,4 @@ /// Filesystem tools: read, write, edit, list, search. - use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::process::Command; @@ -20,10 +19,7 @@ pub(crate) fn clip(text: &str, max_chars: usize) -> String { } let end = text.floor_char_boundary(max_chars); let omitted = text.len() - end; - format!( - "{}\n\n...[truncated {omitted} chars]...", - &text[..end] - ) + format!("{}\n\n...[truncated {omitted} chars]...", &text[..end]) } pub(crate) fn resolve_path(root: &Path, raw_path: &str) -> Result { @@ -374,12 +370,7 @@ pub fn search_files( let rel = entry.path().strip_prefix(root).unwrap_or(entry.path()); for (idx, line) in text.lines().enumerate() { if line.to_lowercase().contains(&lower_query) { - matches.push(format!( - "{}:{}:{}", - rel.to_string_lossy(), - idx + 1, - line - )); + matches.push(format!("{}:{}:{}", rel.to_string_lossy(), idx + 1, line)); if matches.len() >= max_hits { let mut result = matches.join("\n"); result.push_str("\n...[match limit reached]..."); @@ -467,13 +458,7 @@ mod tests { let dir = TempDir::new().unwrap(); std::fs::write(dir.path().join("test.txt"), "hello world").unwrap(); let mut files_read = HashSet::new(); - let result = edit_file( - dir.path(), - "test.txt", - "hello", - "goodbye", - &mut files_read, - ); + let result = edit_file(dir.path(), "test.txt", "hello", "goodbye", &mut files_read); assert!(!result.is_error); assert_eq!( std::fs::read_to_string(dir.path().join("test.txt")).unwrap(), diff --git a/openplanter-desktop/crates/op-core/src/tools/mod.rs b/openplanter-desktop/crates/op-core/src/tools/mod.rs index 6781f4e2..a5e4589b 100644 --- a/openplanter-desktop/crates/op-core/src/tools/mod.rs +++ b/openplanter-desktop/crates/op-core/src/tools/mod.rs @@ -2,17 +2,16 @@ /// /// The `WorkspaceTools` struct is the central dispatcher that owns tool state /// (files-read set, background jobs) and routes tool calls to the appropriate module. - pub mod defs; pub mod filesystem; +pub mod patching; pub mod shell; pub mod web; -pub mod patching; use std::collections::HashSet; use std::path::PathBuf; -use crate::config::AgentConfig; +use crate::config::{AgentConfig, normalize_web_search_provider}; /// Result of executing a tool call. #[derive(Debug, Clone)] @@ -47,8 +46,11 @@ pub struct WorkspaceTools { max_files_listed: usize, max_search_hits: usize, max_observation_chars: usize, + web_search_provider: String, exa_api_key: Option, exa_base_url: String, + firecrawl_api_key: Option, + firecrawl_base_url: String, files_read: HashSet, bg_jobs: shell::BgJobs, } @@ -64,8 +66,11 @@ impl WorkspaceTools { max_files_listed: config.max_files_listed as usize, max_search_hits: config.max_search_hits as usize, max_observation_chars: config.max_observation_chars as usize, + web_search_provider: normalize_web_search_provider(Some(&config.web_search_provider)), exa_api_key: config.exa_api_key.clone(), exa_base_url: config.exa_base_url.clone(), + firecrawl_api_key: config.firecrawl_api_key.clone(), + firecrawl_base_url: config.firecrawl_base_url.clone(), files_read: HashSet::new(), bg_jobs: shell::BgJobs::new(), } @@ -74,14 +79,17 @@ impl WorkspaceTools { /// Execute a tool by name with JSON arguments string. /// Returns the tool result, clipped to max_observation_chars. pub async fn execute(&mut self, name: &str, args_json: &str) -> ToolResult { - let args: serde_json::Value = - serde_json::from_str(args_json).unwrap_or(serde_json::Value::Object(Default::default())); + let args: serde_json::Value = serde_json::from_str(args_json) + .unwrap_or(serde_json::Value::Object(Default::default())); let result = match name { // Filesystem "read_file" => { let path = args.get("path").and_then(|v| v.as_str()).unwrap_or(""); - let hashline = args.get("hashline").and_then(|v| v.as_bool()).unwrap_or(true); + let hashline = args + .get("hashline") + .and_then(|v| v.as_bool()) + .unwrap_or(true); filesystem::read_file( &self.root, path, @@ -99,13 +107,7 @@ impl WorkspaceTools { let path = args.get("path").and_then(|v| v.as_str()).unwrap_or(""); let old_text = args.get("old_text").and_then(|v| v.as_str()).unwrap_or(""); let new_text = args.get("new_text").and_then(|v| v.as_str()).unwrap_or(""); - filesystem::edit_file( - &self.root, - path, - old_text, - new_text, - &mut self.files_read, - ) + filesystem::edit_file(&self.root, path, old_text, new_text, &mut self.files_read) } "list_files" => { let glob = args.get("glob").and_then(|v| v.as_str()); @@ -145,12 +147,7 @@ impl WorkspaceTools { } "run_shell_bg" => { let command = args.get("command").and_then(|v| v.as_str()).unwrap_or(""); - shell::run_shell_bg( - &self.root, - &self.shell_path, - command, - &mut self.bg_jobs, - ) + shell::run_shell_bg(&self.root, &self.shell_path, command, &mut self.bg_jobs) } "check_shell_bg" => { let job_id = args.get("job_id").and_then(|v| v.as_u64()).unwrap_or(0) as u32; @@ -164,11 +161,20 @@ impl WorkspaceTools { // Web "web_search" => { let query = args.get("query").and_then(|v| v.as_str()).unwrap_or(""); - let num_results = args.get("num_results").and_then(|v| v.as_i64()).unwrap_or(10); - let include_text = args.get("include_text").and_then(|v| v.as_bool()).unwrap_or(false); + let num_results = args + .get("num_results") + .and_then(|v| v.as_i64()) + .unwrap_or(10); + let include_text = args + .get("include_text") + .and_then(|v| v.as_bool()) + .unwrap_or(false); web::web_search( + &self.web_search_provider, self.exa_api_key.as_deref(), &self.exa_base_url, + self.firecrawl_api_key.as_deref(), + &self.firecrawl_base_url, query, num_results, include_text, @@ -188,8 +194,11 @@ impl WorkspaceTools { }) .unwrap_or_default(); web::fetch_url( + &self.web_search_provider, self.exa_api_key.as_deref(), &self.exa_base_url, + self.firecrawl_api_key.as_deref(), + &self.firecrawl_base_url, &urls, self.max_file_chars, self.command_timeout_sec, diff --git a/openplanter-desktop/crates/op-core/src/tools/patching.rs b/openplanter-desktop/crates/op-core/src/tools/patching.rs index 8a136b09..2db9d1d6 100644 --- a/openplanter-desktop/crates/op-core/src/tools/patching.rs +++ b/openplanter-desktop/crates/op-core/src/tools/patching.rs @@ -1,5 +1,4 @@ /// Codex-style patch application and hashline editing. - use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -24,8 +23,13 @@ fn resolve_path(root: &Path, raw_path: &str) -> Result { // ── Codex-style patch format ── enum PatchOp { - Add { path: String, content: String }, - Delete { path: String }, + Add { + path: String, + content: String, + }, + Delete { + path: String, + }, Update { path: String, move_to: Option, @@ -66,10 +70,7 @@ fn parse_agent_patch(text: &str) -> Result, String> { let line = body[i].trim(); if line.starts_with("*** Add File:") { - let path = line - .trim_start_matches("*** Add File:") - .trim() - .to_string(); + let path = line.trim_start_matches("*** Add File:").trim().to_string(); i += 1; let mut content_lines: Vec = Vec::new(); while i < body.len() && !body[i].trim().starts_with("***") { @@ -174,11 +175,7 @@ fn parse_chunks(raw_lines: &[&str]) -> Vec { } } -fn find_subsequence( - haystack: &[String], - needle: &[String], - start_idx: usize, -) -> Option { +fn find_subsequence(haystack: &[String], needle: &[String], start_idx: usize) -> Option { if needle.is_empty() { return Some(start_idx.min(haystack.len())); } @@ -202,8 +199,7 @@ fn find_subsequence( } // Pass 2: whitespace-normalized match - let normalize = - |s: &str| -> String { s.split_whitespace().collect::>().join(" ") }; + let normalize = |s: &str| -> String { s.split_whitespace().collect::>().join(" ") }; let norm_needle: Vec = needle.iter().map(|s| normalize(s)).collect(); for i in 0..=max_start { @@ -219,11 +215,7 @@ fn find_subsequence( None } -pub fn apply_patch( - root: &Path, - patch_text: &str, - files_read: &mut HashSet, -) -> ToolResult { +pub fn apply_patch(root: &Path, patch_text: &str, files_read: &mut HashSet) -> ToolResult { if patch_text.trim().is_empty() { return ToolResult::error("apply_patch requires non-empty patch text".into()); } @@ -248,9 +240,7 @@ pub fn apply_patch( let _ = std::fs::create_dir_all(parent); } if let Err(e) = std::fs::write(&resolved, &content) { - return ToolResult::error(format!( - "Patch failed: could not write {path}: {e}" - )); + return ToolResult::error(format!("Patch failed: could not write {path}: {e}")); } files_read.insert(resolved); added.push(path); @@ -261,9 +251,7 @@ pub fn apply_patch( Err(e) => return ToolResult::error(format!("Patch failed: {e}")), }; if !resolved.exists() { - return ToolResult::error(format!( - "Patch failed: file not found: {path}" - )); + return ToolResult::error(format!("Patch failed: file not found: {path}")); } if let Err(e) = std::fs::remove_file(&resolved) { return ToolResult::error(format!( @@ -286,14 +274,13 @@ pub fn apply_patch( Err(e) => { return ToolResult::error(format!( "Patch failed: could not read {path}: {e}" - )) + )); } }; files_read.insert(resolved.clone()); let had_trailing_newline = content.ends_with('\n'); - let mut lines: Vec = - content.lines().map(|l| l.to_string()).collect(); + let mut lines: Vec = content.lines().map(|l| l.to_string()).collect(); let mut cursor = 0usize; for chunk in &chunks { @@ -333,9 +320,7 @@ pub fn apply_patch( let _ = std::fs::create_dir_all(parent); } if let Err(e) = std::fs::write(&target, &result) { - return ToolResult::error(format!( - "Patch failed: could not write {path}: {e}" - )); + return ToolResult::error(format!("Patch failed: could not write {path}: {e}")); } files_read.insert(target); updated.push(path); @@ -414,10 +399,8 @@ pub fn hashline_edit( new_lines: vec![new_line], }); } else if let Some(range) = edit.get("replace_lines") { - let start_anchor = - range.get("start").and_then(|v| v.as_str()).unwrap_or(""); - let end_anchor = - range.get("end").and_then(|v| v.as_str()).unwrap_or(""); + let start_anchor = range.get("start").and_then(|v| v.as_str()).unwrap_or(""); + let end_anchor = range.get("end").and_then(|v| v.as_str()).unwrap_or(""); let (start, err) = validate_anchor(start_anchor, &line_hashes, &lines); if let Some(e) = err { return ToolResult::error(e); @@ -427,12 +410,9 @@ pub fn hashline_edit( return ToolResult::error(e); } if end < start { - return ToolResult::error(format!( - "End line {end} is before start line {start}" - )); + return ToolResult::error(format!("End line {end} is before start line {start}")); } - let raw_content = - edit.get("content").and_then(|v| v.as_str()).unwrap_or(""); + let raw_content = edit.get("content").and_then(|v| v.as_str()).unwrap_or(""); let new_lines: Vec = raw_content .lines() .map(|l| HASHLINE_PREFIX_RE.replace(l, "").to_string()) @@ -443,15 +423,12 @@ pub fn hashline_edit( end, new_lines, }); - } else if let Some(anchor) = - edit.get("insert_after").and_then(|v| v.as_str()) - { + } else if let Some(anchor) = edit.get("insert_after").and_then(|v| v.as_str()) { let (lineno, err) = validate_anchor(anchor, &line_hashes, &lines); if let Some(e) = err { return ToolResult::error(e); } - let raw_content = - edit.get("content").and_then(|v| v.as_str()).unwrap_or(""); + let raw_content = edit.get("content").and_then(|v| v.as_str()).unwrap_or(""); let new_lines: Vec = raw_content .lines() .map(|l| HASHLINE_PREFIX_RE.replace(l, "").to_string()) @@ -483,13 +460,9 @@ pub fn hashline_edit( } } "replace" => { - let old_slice: Vec = - lines[edit.start - 1..edit.end].to_vec(); + let old_slice: Vec = lines[edit.start - 1..edit.end].to_vec(); if old_slice != edit.new_lines { - lines.splice( - edit.start - 1..edit.end, - edit.new_lines.iter().cloned(), - ); + lines.splice(edit.start - 1..edit.end, edit.new_lines.iter().cloned()); changed += 1; } } @@ -527,9 +500,7 @@ fn validate_anchor( if parts.len() != 2 || parts[1].len() != 2 { return ( 0, - Some(format!( - "Invalid anchor format: {anchor:?} (expected N:HH)" - )), + Some(format!("Invalid anchor format: {anchor:?} (expected N:HH)")), ); } let lineno: usize = match parts[0].parse() { @@ -537,10 +508,8 @@ fn validate_anchor( Err(_) => { return ( 0, - Some(format!( - "Invalid anchor format: {anchor:?} (expected N:HH)" - )), - ) + Some(format!("Invalid anchor format: {anchor:?} (expected N:HH)")), + ); } }; let expected_hash = parts[1]; @@ -553,10 +522,7 @@ fn validate_anchor( )), ); } - let actual_hash = line_hashes - .get(&lineno) - .map(|s| s.as_str()) - .unwrap_or(""); + let actual_hash = line_hashes.get(&lineno).map(|s| s.as_str()).unwrap_or(""); if actual_hash != expected_hash { let ctx_start = lineno.saturating_sub(2).max(1); let ctx_end = (lineno + 2).min(lines.len()); @@ -565,10 +531,7 @@ fn validate_anchor( format!( " {}:{}|{}", i, - line_hashes - .get(&i) - .map(|s| s.as_str()) - .unwrap_or("??"), + line_hashes.get(&i).map(|s| s.as_str()).unwrap_or("??"), lines[i - 1] ) }) @@ -603,8 +566,7 @@ mod tests { let result = apply_patch(dir.path(), patch, &mut files_read); assert!(!result.is_error, "error: {}", result.content); assert!(result.content.contains("Added")); - let content = - std::fs::read_to_string(dir.path().join("new_file.txt")).unwrap(); + let content = std::fs::read_to_string(dir.path().join("new_file.txt")).unwrap(); assert_eq!(content, "hello\nworld\n"); } @@ -625,8 +587,7 @@ mod tests { #[test] fn test_apply_patch_update_file() { let dir = TempDir::new().unwrap(); - std::fs::write(dir.path().join("test.txt"), "line1\nline2\nline3\n") - .unwrap(); + std::fs::write(dir.path().join("test.txt"), "line1\nline2\nline3\n").unwrap(); let mut files_read = HashSet::new(); let patch = "\ *** Begin Patch @@ -639,8 +600,7 @@ mod tests { *** End Patch"; let result = apply_patch(dir.path(), patch, &mut files_read); assert!(!result.is_error, "error: {}", result.content); - let content = - std::fs::read_to_string(dir.path().join("test.txt")).unwrap(); + let content = std::fs::read_to_string(dir.path().join("test.txt")).unwrap(); assert!(content.contains("line2_modified")); assert!(!content.contains("\nline2\n")); } @@ -656,11 +616,9 @@ mod tests { "set_line": format!("2:{hash}"), "content": "BBB" })]; - let result = - hashline_edit(dir.path(), "test.txt", &edits, &mut files_read); + let result = hashline_edit(dir.path(), "test.txt", &edits, &mut files_read); assert!(!result.is_error, "error: {}", result.content); - let content = - std::fs::read_to_string(dir.path().join("test.txt")).unwrap(); + let content = std::fs::read_to_string(dir.path().join("test.txt")).unwrap(); assert!(content.contains("BBB")); assert!(!content.contains("\nbbb\n")); } @@ -676,11 +634,9 @@ mod tests { "insert_after": format!("2:{hash}"), "content": "inserted_line" })]; - let result = - hashline_edit(dir.path(), "test.txt", &edits, &mut files_read); + let result = hashline_edit(dir.path(), "test.txt", &edits, &mut files_read); assert!(!result.is_error, "error: {}", result.content); - let content = - std::fs::read_to_string(dir.path().join("test.txt")).unwrap(); + let content = std::fs::read_to_string(dir.path().join("test.txt")).unwrap(); let lines: Vec<&str> = content.lines().collect(); assert_eq!(lines[2], "inserted_line"); } @@ -693,16 +649,14 @@ mod tests { #[test] fn test_find_subsequence_exact() { - let haystack: Vec = - vec!["a".into(), "b".into(), "c".into()]; + let haystack: Vec = vec!["a".into(), "b".into(), "c".into()]; let needle: Vec = vec!["b".into(), "c".into()]; assert_eq!(find_subsequence(&haystack, &needle, 0), Some(1)); } #[test] fn test_find_subsequence_whitespace() { - let haystack: Vec = - vec!["a".into(), " b ".into(), "c".into()]; + let haystack: Vec = vec!["a".into(), " b ".into(), "c".into()]; let needle: Vec = vec!["b".into(), "c".into()]; assert_eq!(find_subsequence(&haystack, &needle, 0), Some(1)); } diff --git a/openplanter-desktop/crates/op-core/src/tools/shell.rs b/openplanter-desktop/crates/op-core/src/tools/shell.rs index f1023b37..5f032f75 100644 --- a/openplanter-desktop/crates/op-core/src/tools/shell.rs +++ b/openplanter-desktop/crates/op-core/src/tools/shell.rs @@ -1,5 +1,4 @@ /// Shell execution tools: run_shell, run_shell_bg, check_shell_bg, kill_shell_bg. - use std::collections::HashMap; use std::path::Path; use std::process::{Child, Command, Stdio}; @@ -22,10 +21,7 @@ fn clip(text: &str, max_chars: usize) -> String { } let end = text.floor_char_boundary(max_chars); let omitted = text.len() - end; - format!( - "{}\n\n...[truncated {omitted} chars]...", - &text[..end] - ) + format!("{}\n\n...[truncated {omitted} chars]...", &text[..end]) } fn check_shell_policy(command: &str) -> Option { @@ -134,18 +130,11 @@ pub fn run_shell( let stderr = String::from_utf8_lossy(&output.stderr); let code = output.status.code().unwrap_or(-1); - let merged = format!( - "$ {command}\n[exit_code={code}]\n[stdout]\n{stdout}\n[stderr]\n{stderr}" - ); + let merged = format!("$ {command}\n[exit_code={code}]\n[stdout]\n{stdout}\n[stderr]\n{stderr}"); ToolResult::ok(clip(&merged, max_output_chars)) } -pub fn run_shell_bg( - root: &Path, - shell: &str, - command: &str, - bg_jobs: &mut BgJobs, -) -> ToolResult { +pub fn run_shell_bg(root: &Path, shell: &str, command: &str, bg_jobs: &mut BgJobs) -> ToolResult { if let Some(err) = check_shell_policy(command) { return ToolResult::error(err); } @@ -195,11 +184,7 @@ pub fn run_shell_bg( )) } -pub fn check_shell_bg( - job_id: u32, - bg_jobs: &mut BgJobs, - max_output_chars: usize, -) -> ToolResult { +pub fn check_shell_bg(job_id: u32, bg_jobs: &mut BgJobs, max_output_chars: usize) -> ToolResult { let job = match bg_jobs.jobs.get_mut(&job_id) { Some(j) => j, None => return ToolResult::error(format!("No background job with id {job_id}")), @@ -220,9 +205,7 @@ pub fn check_shell_bg( } Ok(None) => { let pid = job.child.id(); - ToolResult::ok(format!( - "[job {job_id} still running, pid={pid}]\n{output}" - )) + ToolResult::ok(format!("[job {job_id} still running, pid={pid}]\n{output}")) } Err(e) => ToolResult::error(format!("Error checking job {job_id}: {e}")), } @@ -258,13 +241,7 @@ mod tests { #[test] fn test_run_shell_heredoc_blocked() { let dir = TempDir::new().unwrap(); - let result = run_shell( - dir.path(), - "/bin/sh", - "cat << EOF\nhello\nEOF", - 10, - 16000, - ); + let result = run_shell(dir.path(), "/bin/sh", "cat << EOF\nhello\nEOF", 10, 16000); assert!(result.is_error); assert!(result.content.contains("BLOCKED")); } diff --git a/openplanter-desktop/crates/op-core/src/tools/web.rs b/openplanter-desktop/crates/op-core/src/tools/web.rs index c9629e89..fb67a633 100644 --- a/openplanter-desktop/crates/op-core/src/tools/web.rs +++ b/openplanter-desktop/crates/op-core/src/tools/web.rs @@ -1,7 +1,10 @@ -/// Web tools: Exa search, fetch_url. +/// Web tools: Exa / Firecrawl search and fetch_url. +use std::time::Duration; use serde_json::json; +use crate::config::normalize_web_search_provider; + use super::ToolResult; fn clip(text: &str, max_chars: usize) -> String { @@ -10,15 +13,84 @@ fn clip(text: &str, max_chars: usize) -> String { } let end = text.floor_char_boundary(max_chars); let omitted = text.len() - end; - format!( - "{}\n\n...[truncated {omitted} chars]...", - &text[..end] - ) + format!("{}\n\n...[truncated {omitted} chars]...", &text[..end]) +} + +async fn exa_request( + api_key: Option<&str>, + exa_base_url: &str, + endpoint: &str, + payload: &serde_json::Value, + timeout_sec: u64, +) -> Result { + let api_key = match api_key { + Some(value) if !value.trim().is_empty() => value, + _ => return Err("EXA_API_KEY not configured".into()), + }; + + let url = format!("{}{}", exa_base_url.trim_end_matches('/'), endpoint); + let client = reqwest::Client::new(); + let response = client + .post(&url) + .header("x-api-key", api_key) + .header("Content-Type", "application/json") + .header("User-Agent", "exa-py 1.0.18") + .timeout(Duration::from_secs(timeout_sec)) + .json(payload) + .send() + .await + .map_err(|e| format!("Exa API request failed: {e}"))?; + + let response = response + .error_for_status() + .map_err(|e| format!("Exa API request failed: {e}"))?; + + response + .json::() + .await + .map_err(|e| format!("Exa API returned non-JSON payload: {e}")) +} + +async fn firecrawl_request( + api_key: Option<&str>, + firecrawl_base_url: &str, + endpoint: &str, + payload: &serde_json::Value, + timeout_sec: u64, +) -> Result { + let api_key = match api_key { + Some(value) if !value.trim().is_empty() => value, + _ => return Err("FIRECRAWL_API_KEY not configured".into()), + }; + + let url = format!("{}{}", firecrawl_base_url.trim_end_matches('/'), endpoint); + let client = reqwest::Client::new(); + let response = client + .post(&url) + .header("Authorization", format!("Bearer {api_key}")) + .header("Content-Type", "application/json") + .timeout(Duration::from_secs(timeout_sec)) + .json(payload) + .send() + .await + .map_err(|e| format!("Firecrawl API request failed: {e}"))?; + + let response = response + .error_for_status() + .map_err(|e| format!("Firecrawl API request failed: {e}"))?; + + response + .json::() + .await + .map_err(|e| format!("Firecrawl API returned non-JSON payload: {e}")) } pub async fn web_search( + provider: &str, exa_api_key: Option<&str>, exa_base_url: &str, + firecrawl_api_key: Option<&str>, + firecrawl_base_url: &str, query: &str, num_results: i64, include_text: bool, @@ -30,66 +102,132 @@ pub async fn web_search( return ToolResult::error("web_search requires non-empty query".into()); } - let api_key = match exa_api_key { - Some(k) if !k.trim().is_empty() => k, - _ => return ToolResult::error("EXA_API_KEY not configured".into()), - }; + let provider = normalize_web_search_provider(Some(provider)); + let clamped = num_results.clamp(1, 20); - let clamped = num_results.max(1).min(20); - let mut payload = json!({ - "query": query, - "numResults": clamped, - }); - if include_text { - payload["contents"] = json!({"text": {"maxCharacters": 4000}}); - } + let output = if provider == "firecrawl" { + let mut payload = json!({ + "query": query, + "limit": clamped, + }); + if include_text { + payload["scrapeOptions"] = json!({ "formats": ["markdown"] }); + } - let url = format!("{}/search", exa_base_url.trim_end_matches('/')); - let client = reqwest::Client::new(); - let response = client - .post(&url) - .header("x-api-key", api_key) - .header("Content-Type", "application/json") - .header("User-Agent", "exa-py 1.0.18") - .timeout(std::time::Duration::from_secs(timeout_sec)) - .json(&payload) - .send() - .await; + match firecrawl_request( + firecrawl_api_key, + firecrawl_base_url, + "/search", + &payload, + timeout_sec, + ) + .await + { + Ok(body) => { + let mut rows: Vec = Vec::new(); + if let Some(items) = body.get("data").and_then(|value| value.as_array()) { + rows.extend(items.iter().cloned()); + } else if let Some(items) = body + .get("data") + .and_then(|value| value.get("web")) + .and_then(|value| value.as_array()) + { + rows.extend(items.iter().cloned()); + } - let resp = match response { - Ok(r) => r, - Err(e) => return ToolResult::error(format!("Web search failed: {e}")), - }; + let mut results: Vec = Vec::new(); + for row in rows { + let metadata = row.get("metadata").and_then(|value| value.as_object()); + let title = row + .get("title") + .and_then(|value| value.as_str()) + .filter(|value| !value.is_empty()) + .or_else(|| { + metadata + .and_then(|meta| meta.get("title")) + .and_then(|value| value.as_str()) + }) + .unwrap_or(""); - let body: serde_json::Value = match resp.json().await { - Ok(b) => b, - Err(e) => return ToolResult::error(format!("Web search response parse error: {e}")), - }; + let mut item = json!({ + "url": row.get("url").and_then(|value| value.as_str()).unwrap_or(""), + "title": title, + "snippet": row + .get("description") + .and_then(|value| value.as_str()) + .or_else(|| row.get("snippet").and_then(|value| value.as_str())) + .unwrap_or(""), + }); - let mut out_results: Vec = Vec::new(); - if let Some(results) = body.get("results").and_then(|r| r.as_array()) { - for row in results { - let mut item = json!({ - "url": row.get("url").and_then(|u| u.as_str()).unwrap_or(""), - "title": row.get("title").and_then(|t| t.as_str()).unwrap_or(""), - "snippet": row.get("highlight").and_then(|h| h.as_str()) - .or_else(|| row.get("snippet").and_then(|s| s.as_str())) - .unwrap_or(""), - }); - if include_text { - if let Some(text) = row.get("text").and_then(|t| t.as_str()) { - item["text"] = json!(clip(text, 4000)); + if include_text { + if let Some(text) = row + .get("markdown") + .and_then(|value| value.as_str()) + .or_else(|| row.get("text").and_then(|value| value.as_str())) + { + if !text.is_empty() { + item["text"] = json!(clip(text, 4_000)); + } + } + } + + results.push(item); } + + json!({ + "query": query, + "provider": provider, + "results": results, + "total": results.len(), + }) } - out_results.push(item); + Err(error) => return ToolResult::error(format!("Web search failed: {error}")), } - } + } else { + let mut payload = json!({ + "query": query, + "numResults": clamped, + }); + if include_text { + payload["contents"] = json!({ "text": { "maxCharacters": 4_000 } }); + } + + match exa_request(exa_api_key, exa_base_url, "/search", &payload, timeout_sec).await { + Ok(body) => { + let mut results: Vec = Vec::new(); + if let Some(rows) = body.get("results").and_then(|value| value.as_array()) { + for row in rows { + let mut item = json!({ + "url": row.get("url").and_then(|value| value.as_str()).unwrap_or(""), + "title": row.get("title").and_then(|value| value.as_str()).unwrap_or(""), + "snippet": row + .get("highlight") + .and_then(|value| value.as_str()) + .or_else(|| row.get("snippet").and_then(|value| value.as_str())) + .unwrap_or(""), + }); + if include_text { + if let Some(text) = row.get("text").and_then(|value| value.as_str()) { + if !text.is_empty() { + item["text"] = json!(clip(text, 4_000)); + } + } + } + results.push(item); + } + } + + json!({ + "query": query, + "provider": provider, + "results": results, + "total": results.len(), + }) + } + Err(error) => return ToolResult::error(format!("Web search failed: {error}")), + } + }; - let output = json!({ - "query": query, - "results": out_results, - "total": out_results.len(), - }); ToolResult::ok(clip( &serde_json::to_string_pretty(&output).unwrap_or_default(), max_file_chars, @@ -97,79 +235,319 @@ pub async fn web_search( } pub async fn fetch_url( + provider: &str, exa_api_key: Option<&str>, exa_base_url: &str, + firecrawl_api_key: Option<&str>, + firecrawl_base_url: &str, urls: &[String], max_file_chars: usize, timeout_sec: u64, ) -> ToolResult { - if urls.is_empty() { - return ToolResult::error("fetch_url requires at least one valid URL".into()); - } - - let api_key = match exa_api_key { - Some(k) if !k.trim().is_empty() => k, - _ => return ToolResult::error("EXA_API_KEY not configured".into()), - }; - - let normalized: Vec<&str> = urls + let normalized: Vec = urls .iter() - .map(|u| u.trim()) - .filter(|u| !u.is_empty()) + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) .take(10) + .map(String::from) .collect(); if normalized.is_empty() { return ToolResult::error("fetch_url requires at least one valid URL".into()); } - let payload = json!({ - "ids": normalized, - "text": { "maxCharacters": 8000 }, - }); + let provider = normalize_web_search_provider(Some(provider)); - let url = format!("{}/contents", exa_base_url.trim_end_matches('/')); - let client = reqwest::Client::new(); - let response = client - .post(&url) - .header("x-api-key", api_key) - .header("Content-Type", "application/json") - .header("User-Agent", "exa-py 1.0.18") - .timeout(std::time::Duration::from_secs(timeout_sec)) - .json(&payload) - .send() - .await; + let output = if provider == "firecrawl" { + let mut pages: Vec = Vec::new(); + for url in &normalized { + let payload = json!({ + "url": url, + "formats": ["markdown"], + }); + let body = match firecrawl_request( + firecrawl_api_key, + firecrawl_base_url, + "/scrape", + &payload, + timeout_sec, + ) + .await + { + Ok(body) => body, + Err(error) => return ToolResult::error(format!("Fetch URL failed: {error}")), + }; - let resp = match response { - Ok(r) => r, - Err(e) => return ToolResult::error(format!("Fetch URL failed: {e}")), - }; + if let Some(data) = body.get("data").and_then(|value| value.as_object()) { + let title = data + .get("metadata") + .and_then(|value| value.as_object()) + .and_then(|meta| meta.get("title")) + .and_then(|value| value.as_str()) + .unwrap_or(""); + let text = data + .get("markdown") + .and_then(|value| value.as_str()) + .or_else(|| data.get("text").and_then(|value| value.as_str())) + .or_else(|| data.get("html").and_then(|value| value.as_str())) + .unwrap_or(""); - let body: serde_json::Value = match resp.json().await { - Ok(b) => b, - Err(e) => return ToolResult::error(format!("Fetch URL response parse error: {e}")), - }; + pages.push(json!({ + "url": data.get("url").and_then(|value| value.as_str()).unwrap_or(url), + "title": title, + "text": clip(text, 8_000), + })); + } + } + + json!({ + "provider": provider, + "pages": pages, + "total": pages.len(), + }) + } else { + let payload = json!({ + "ids": normalized, + "text": { "maxCharacters": 8_000 }, + }); + + match exa_request( + exa_api_key, + exa_base_url, + "/contents", + &payload, + timeout_sec, + ) + .await + { + Ok(body) => { + let mut pages: Vec = Vec::new(); + if let Some(rows) = body.get("results").and_then(|value| value.as_array()) { + for row in rows { + pages.push(json!({ + "url": row.get("url").and_then(|value| value.as_str()).unwrap_or(""), + "title": row.get("title").and_then(|value| value.as_str()).unwrap_or(""), + "text": clip( + row.get("text").and_then(|value| value.as_str()).unwrap_or(""), + 8_000, + ), + })); + } + } - let mut pages: Vec = Vec::new(); - if let Some(results) = body.get("results").and_then(|r| r.as_array()) { - for row in results { - pages.push(json!({ - "url": row.get("url").and_then(|u| u.as_str()).unwrap_or(""), - "title": row.get("title").and_then(|t| t.as_str()).unwrap_or(""), - "text": clip( - row.get("text").and_then(|t| t.as_str()).unwrap_or(""), - 8000, - ), - })); + json!({ + "provider": provider, + "pages": pages, + "total": pages.len(), + }) + } + Err(error) => return ToolResult::error(format!("Fetch URL failed: {error}")), } - } + }; - let output = json!({ - "pages": pages, - "total": pages.len(), - }); ToolResult::ok(clip( &serde_json::to_string_pretty(&output).unwrap_or_default(), max_file_chars, )) } + +#[cfg(test)] +mod tests { + use axum::body::Body; + use axum::http::StatusCode; + use axum::response::Response; + use axum::routing::post; + use axum::{Json, Router}; + use serde_json::{Value, json}; + + use super::*; + + async fn start_json_server( + path: &'static str, + response_payload: Value, + ) -> std::net::SocketAddr { + let app = Router::new().route( + path, + post(move || { + let response_payload = response_payload.clone(); + async move { Json(response_payload) } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + addr + } + + async fn start_status_server(path: &'static str, status: StatusCode) -> std::net::SocketAddr { + let app = Router::new().route( + path, + post(move || async move { + Response::builder() + .status(status) + .body(Body::from("{\"error\":\"boom\"}")) + .unwrap() + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + addr + } + + #[tokio::test] + async fn test_web_search_exa_output_shape() { + let addr = start_json_server( + "/search", + json!({ + "results": [ + { + "url": "https://example.com", + "title": "Example", + "highlight": "Snippet", + "text": "Long page body" + } + ] + }), + ) + .await; + + let result = web_search( + "exa", + Some("exa-key"), + &format!("http://{addr}"), + None, + "https://api.firecrawl.dev/v1", + "example query", + 5, + true, + 20_000, + 5, + ) + .await; + + assert!(!result.is_error); + let parsed: Value = serde_json::from_str(&result.content).unwrap(); + assert_eq!(parsed["provider"], "exa"); + assert_eq!(parsed["query"], "example query"); + assert_eq!(parsed["results"][0]["url"], "https://example.com"); + assert_eq!(parsed["results"][0]["text"], "Long page body"); + } + + #[tokio::test] + async fn test_web_search_firecrawl_output_shape() { + let addr = start_json_server( + "/search", + json!({ + "data": [ + { + "url": "https://example.com/firecrawl", + "description": "Firecrawl snippet", + "markdown": "# Hello", + "metadata": { "title": "Firecrawl Title" } + } + ] + }), + ) + .await; + + let result = web_search( + "firecrawl", + None, + "https://api.exa.ai", + Some("fc-key"), + &format!("http://{addr}"), + "example query", + 5, + true, + 20_000, + 5, + ) + .await; + + assert!(!result.is_error); + let parsed: Value = serde_json::from_str(&result.content).unwrap(); + assert_eq!(parsed["provider"], "firecrawl"); + assert_eq!(parsed["results"][0]["title"], "Firecrawl Title"); + assert_eq!(parsed["results"][0]["text"], "# Hello"); + } + + #[tokio::test] + async fn test_fetch_url_firecrawl_output_shape() { + let addr = start_json_server( + "/scrape", + json!({ + "data": { + "url": "https://example.com/article", + "markdown": "Article body", + "metadata": { "title": "Article Title" } + } + }), + ) + .await; + + let result = fetch_url( + "firecrawl", + None, + "https://api.exa.ai", + Some("fc-key"), + &format!("http://{addr}"), + &[String::from("https://example.com/article")], + 20_000, + 5, + ) + .await; + + assert!(!result.is_error); + let parsed: Value = serde_json::from_str(&result.content).unwrap(); + assert_eq!(parsed["provider"], "firecrawl"); + assert_eq!(parsed["pages"][0]["title"], "Article Title"); + assert_eq!(parsed["pages"][0]["text"], "Article body"); + } + + #[tokio::test] + async fn test_missing_firecrawl_key_errors() { + let result = web_search( + "firecrawl", + None, + "https://api.exa.ai", + None, + "https://api.firecrawl.dev/v1", + "example query", + 5, + false, + 20_000, + 5, + ) + .await; + + assert!(result.is_error); + assert!(result.content.contains("FIRECRAWL_API_KEY")); + } + + #[tokio::test] + async fn test_exa_http_error_bubbles_up() { + let addr = start_status_server("/search", StatusCode::BAD_GATEWAY).await; + + let result = web_search( + "exa", + Some("exa-key"), + &format!("http://{addr}"), + None, + "https://api.firecrawl.dev/v1", + "example query", + 5, + false, + 20_000, + 5, + ) + .await; + + assert!(result.is_error); + assert!(result.content.contains("Web search failed")); + } +} diff --git a/openplanter-desktop/crates/op-core/src/wiki/matching.rs b/openplanter-desktop/crates/op-core/src/wiki/matching.rs index 4955b0ac..48620b1d 100644 --- a/openplanter-desktop/crates/op-core/src/wiki/matching.rs +++ b/openplanter-desktop/crates/op-core/src/wiki/matching.rs @@ -18,15 +18,13 @@ impl NameRegistry { /// Register a canonical name for an entity. pub fn register(&mut self, name: &str, entity_id: &str) { - self.entries - .push((name.to_string(), entity_id.to_string())); + self.entries.push((name.to_string(), entity_id.to_string())); } /// Register multiple aliases for the same entity. pub fn register_aliases(&mut self, aliases: &[String], entity_id: &str) { for alias in aliases { - self.entries - .push((alias.clone(), entity_id.to_string())); + self.entries.push((alias.clone(), entity_id.to_string())); } } @@ -123,10 +121,7 @@ mod tests { fn test_aliases() { let mut reg = NameRegistry::new(); reg.register("Acme Corp", "acme-corp"); - reg.register_aliases( - &["AC".to_string(), "Acme".to_string()], - "acme-corp", - ); + reg.register_aliases(&["AC".to_string(), "Acme".to_string()], "acme-corp"); assert_eq!(reg.len(), 3); let result = reg.find_best("Acme"); diff --git a/openplanter-desktop/crates/op-core/src/wiki/mod.rs b/openplanter-desktop/crates/op-core/src/wiki/mod.rs index 149037b4..02051a6d 100644 --- a/openplanter-desktop/crates/op-core/src/wiki/mod.rs +++ b/openplanter-desktop/crates/op-core/src/wiki/mod.rs @@ -1,6 +1,6 @@ +pub mod matching; /// Wiki knowledge graph model (petgraph). /// /// Full implementation in Phase 5. pub mod parser; -pub mod matching; pub mod watcher; diff --git a/openplanter-desktop/crates/op-core/src/wiki/parser.rs b/openplanter-desktop/crates/op-core/src/wiki/parser.rs index 638222e3..f24746b5 100644 --- a/openplanter-desktop/crates/op-core/src/wiki/parser.rs +++ b/openplanter-desktop/crates/op-core/src/wiki/parser.rs @@ -37,9 +37,7 @@ pub fn parse_index(content: &str) -> Vec { if trimmed.starts_with("### ") { current_category = trimmed[4..].trim().to_lowercase(); // Normalize common category names - current_category = current_category - .replace(' ', "-") - .replace('_', "-"); + current_category = current_category.replace(' ', "-").replace('_', "-"); continue; } diff --git a/openplanter-desktop/crates/op-core/src/wiki/watcher.rs b/openplanter-desktop/crates/op-core/src/wiki/watcher.rs index a9d4ca86..19abf89d 100644 --- a/openplanter-desktop/crates/op-core/src/wiki/watcher.rs +++ b/openplanter-desktop/crates/op-core/src/wiki/watcher.rs @@ -37,34 +37,35 @@ impl WikiWatcher { ) -> std::io::Result<(Self, mpsc::UnboundedReceiver)> { let (tx, rx) = mpsc::unbounded_channel(); - let mut watcher = notify::recommended_watcher(move |result: Result| { - let event = match result { - Ok(e) => e, - Err(err) => { - eprintln!("[wiki-watcher] error: {err}"); - return; - } - }; - - let kind = match event.kind { - EventKind::Create(_) => WikiChangeKind::Created, - EventKind::Modify(_) => WikiChangeKind::Modified, - EventKind::Remove(_) => WikiChangeKind::Deleted, - _ => return, - }; - - for path in event.paths { - // Only watch .md files - if path.extension().and_then(|e| e.to_str()) != Some("md") { - continue; + let mut watcher = + notify::recommended_watcher(move |result: Result| { + let event = match result { + Ok(e) => e, + Err(err) => { + eprintln!("[wiki-watcher] error: {err}"); + return; + } + }; + + let kind = match event.kind { + EventKind::Create(_) => WikiChangeKind::Created, + EventKind::Modify(_) => WikiChangeKind::Modified, + EventKind::Remove(_) => WikiChangeKind::Deleted, + _ => return, + }; + + for path in event.paths { + // Only watch .md files + if path.extension().and_then(|e| e.to_str()) != Some("md") { + continue; + } + let _ = tx.send(WikiChangeEvent { + path, + kind: kind.clone(), + }); } - let _ = tx.send(WikiChangeEvent { - path, - kind: kind.clone(), - }); - } - }) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + }) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; watcher .watch(&wiki_dir, RecursiveMode::Recursive) @@ -83,7 +84,7 @@ impl WikiWatcher { mod tests { use super::*; use tempfile::tempdir; - use tokio::time::{sleep, Duration}; + use tokio::time::{Duration, sleep}; #[tokio::test] async fn test_watcher_detects_create() { diff --git a/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs b/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs index 5e792de0..c2ce34c6 100644 --- a/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs +++ b/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs @@ -6,17 +6,17 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::sync::{Arc, Mutex}; +use axum::Router; use axum::body::Body; use axum::http::StatusCode; use axum::response::Response; use axum::routing::post; -use axum::Router; use tokio_util::sync::CancellationToken; use op_core::events::{DeltaEvent, DeltaKind}; -use op_core::model::openai::OpenAIModel; use op_core::model::anthropic::AnthropicModel; -use op_core::model::{BaseModel, Message}; +use op_core::model::openai::OpenAIModel; +use op_core::model::{BaseModel, Message, RateLimitError}; // ─── Helpers ─── @@ -81,10 +81,62 @@ async fn start_error_server(status: u16, body: &'static str) -> SocketAddr { addr } +#[derive(Clone)] +struct MockHttpResponse { + status: u16, + content_type: &'static str, + body: &'static str, + headers: Vec<(&'static str, &'static str)>, +} + +async fn start_stateful_http_server(responses: Vec) -> SocketAddr { + let counter = Arc::new(Mutex::new(0usize)); + let responses = Arc::new(responses); + + let app = Router::new().route( + "/{*path}", + post(move || { + let counter = counter.clone(); + let responses = responses.clone(); + async move { + let mut idx = counter.lock().unwrap(); + let response = if *idx < responses.len() { + responses[*idx].clone() + } else { + responses + .last() + .expect("expected at least one HTTP response") + .clone() + }; + *idx += 1; + + let mut builder = Response::builder() + .status(StatusCode::from_u16(response.status).unwrap()) + .header("content-type", response.content_type); + for (name, value) in &response.headers { + builder = builder.header(*name, *value); + } + builder.body(Body::from(response.body)).unwrap() + } + }), + ); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + addr +} + fn simple_messages() -> Vec { vec![ - Message::System { content: "You are helpful.".to_string() }, - Message::User { content: "Say hello".to_string() }, + Message::System { + content: "You are helpful.".to_string(), + }, + Message::User { + content: "Say hello".to_string(), + }, ] } @@ -350,7 +402,10 @@ async fn test_openai_chat_non_streaming() { ); // chat() should internally call chat_stream with no-op callback - let turn = model.chat(&simple_messages(), &[]).await.expect("chat should succeed"); + let turn = model + .chat(&simple_messages(), &[]) + .await + .expect("chat should succeed"); assert_eq!(turn.text, "Hello world"); assert_eq!(turn.input_tokens, 10); } @@ -365,7 +420,10 @@ async fn test_anthropic_chat_non_streaming() { None, ); - let turn = model.chat(&simple_messages(), &[]).await.expect("chat should succeed"); + let turn = model + .chat(&simple_messages(), &[]) + .await + .expect("chat should succeed"); assert_eq!(turn.text, "Hello from Claude"); assert_eq!(turn.input_tokens, 25); } @@ -377,7 +435,8 @@ async fn test_openai_http_error() { let addr = start_error_server( 401, r#"{"error":{"message":"Invalid API key","type":"invalid_request_error"}}"#, - ).await; + ) + .await; let model = OpenAIModel::new( "gpt-4o".to_string(), "openai".to_string(), @@ -395,12 +454,50 @@ async fn test_openai_http_error() { assert!(result.is_err(), "should fail with HTTP error"); } +#[tokio::test] +async fn test_openai_rate_limit_error_includes_retry_after() { + let addr = start_stateful_http_server(vec![MockHttpResponse { + status: 429, + content_type: "application/json", + body: r#"{"error":{"message":"Too many requests","code":"1302"}}"#, + headers: vec![("retry-after", "3")], + }]) + .await; + let model = OpenAIModel::new( + "glm-5".to_string(), + "zai".to_string(), + format!("http://{addr}"), + "zai-key".to_string(), + Some("high".to_string()), + HashMap::new(), + ) + .with_zai_runtime(op_core::model::openai::ZaiRuntimeConfig { + paygo_base_url: format!("http://{addr}"), + coding_base_url: format!("http://{addr}"), + stream_max_retries: 1, + }); + + let cancel = CancellationToken::new(); + let error = model + .chat_stream(&simple_messages(), &[], &|_| {}, &cancel) + .await + .expect_err("should fail with a structured rate-limit error"); + + let rate_limit = error + .downcast_ref::() + .expect("expected a structured rate-limit error"); + assert_eq!(rate_limit.status_code, Some(429)); + assert_eq!(rate_limit.provider_code.as_deref(), Some("1302")); + assert_eq!(rate_limit.retry_after_sec, Some(3.0)); +} + #[tokio::test] async fn test_anthropic_http_error() { let addr = start_error_server( 401, r#"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}"#, - ).await; + ) + .await; let model = AnthropicModel::new( "claude-sonnet-4-5".to_string(), format!("http://{addr}"), @@ -421,12 +518,13 @@ async fn test_anthropic_http_error() { #[tokio::test] async fn test_solve_with_mock_anthropic() { use op_core::config::AgentConfig; - use op_core::engine::{solve, SolveEmitter}; + use op_core::engine::{SolveEmitter, solve}; use op_core::events::StepEvent; let addr = start_mock_sse_server(ANTHROPIC_SSE_SIMPLE).await; #[derive(Debug, Clone)] + #[allow(dead_code)] enum Ev { Trace(String), Delta(DeltaEvent), @@ -440,7 +538,10 @@ async fn test_solve_with_mock_anthropic() { } impl SolveEmitter for TestEmitter { fn emit_trace(&self, message: &str) { - self.events.lock().unwrap().push(Ev::Trace(message.to_string())); + self.events + .lock() + .unwrap() + .push(Ev::Trace(message.to_string())); } fn emit_delta(&self, event: DeltaEvent) { self.events.lock().unwrap().push(Ev::Delta(event)); @@ -449,15 +550,23 @@ async fn test_solve_with_mock_anthropic() { self.events.lock().unwrap().push(Ev::Step(event)); } fn emit_complete(&self, result: &str) { - self.events.lock().unwrap().push(Ev::Complete(result.to_string())); + self.events + .lock() + .unwrap() + .push(Ev::Complete(result.to_string())); } fn emit_error(&self, message: &str) { - self.events.lock().unwrap().push(Ev::Error(message.to_string())); + self.events + .lock() + .unwrap() + .push(Ev::Error(message.to_string())); } } let events = Arc::new(Mutex::new(Vec::new())); - let emitter = TestEmitter { events: events.clone() }; + let emitter = TestEmitter { + events: events.clone(), + }; let cfg = AgentConfig { provider: "anthropic".into(), @@ -475,7 +584,9 @@ async fn test_solve_with_mock_anthropic() { // Should have a trace assert!( - recorded.iter().any(|e| matches!(e, Ev::Trace(m) if m.contains("anthropic"))), + recorded + .iter() + .any(|e| matches!(e, Ev::Trace(m) if m.contains("anthropic"))), "should have a trace mentioning anthropic" ); @@ -491,13 +602,17 @@ async fn test_solve_with_mock_anthropic() { // Should have a step assert!( - recorded.iter().any(|e| matches!(e, Ev::Step(s) if s.is_final && s.tokens.input_tokens == 25)), + recorded + .iter() + .any(|e| matches!(e, Ev::Step(s) if s.is_final && s.tokens.input_tokens == 25)), "should have a final step with correct token count" ); // Should have complete with the full text assert!( - recorded.iter().any(|e| matches!(e, Ev::Complete(t) if t == "Hello from Claude")), + recorded + .iter() + .any(|e| matches!(e, Ev::Complete(t) if t == "Hello from Claude")), "should complete with full text" ); @@ -511,7 +626,7 @@ async fn test_solve_with_mock_anthropic() { #[tokio::test] async fn test_solve_with_mock_openai() { use op_core::config::AgentConfig; - use op_core::engine::{solve, SolveEmitter}; + use op_core::engine::{SolveEmitter, solve}; use op_core::events::StepEvent; let addr = start_mock_sse_server(OPENAI_SSE_SIMPLE).await; @@ -531,7 +646,10 @@ async fn test_solve_with_mock_openai() { } impl SolveEmitter for TestEmitter2 { fn emit_trace(&self, message: &str) { - self.events.lock().unwrap().push(Ev2::Trace(message.to_string())); + self.events + .lock() + .unwrap() + .push(Ev2::Trace(message.to_string())); } fn emit_delta(&self, event: DeltaEvent) { self.events.lock().unwrap().push(Ev2::Delta(event)); @@ -540,15 +658,23 @@ async fn test_solve_with_mock_openai() { self.events.lock().unwrap().push(Ev2::Step(event)); } fn emit_complete(&self, result: &str) { - self.events.lock().unwrap().push(Ev2::Complete(result.to_string())); + self.events + .lock() + .unwrap() + .push(Ev2::Complete(result.to_string())); } fn emit_error(&self, message: &str) { - self.events.lock().unwrap().push(Ev2::Error(message.to_string())); + self.events + .lock() + .unwrap() + .push(Ev2::Error(message.to_string())); } } let events = Arc::new(Mutex::new(Vec::new())); - let emitter = TestEmitter2 { events: events.clone() }; + let emitter = TestEmitter2 { + events: events.clone(), + }; let cfg = AgentConfig { provider: "openai".into(), @@ -567,9 +693,17 @@ async fn test_solve_with_mock_openai() { // Should have a trace mentioning openai assert!( - recorded.iter().any(|e| matches!(e, Ev2::Trace(m) if m.contains("openai"))), + recorded + .iter() + .any(|e| matches!(e, Ev2::Trace(m) if m.contains("openai"))), "should have a trace mentioning openai, got: {:?}", - recorded.iter().filter_map(|e| match e { Ev2::Trace(m) => Some(m.clone()), _ => None }).collect::>() + recorded + .iter() + .filter_map(|e| match e { + Ev2::Trace(m) => Some(m.clone()), + _ => None, + }) + .collect::>() ); // Should have text deltas that spell "Hello world" @@ -584,13 +718,17 @@ async fn test_solve_with_mock_openai() { // Should have a step with correct tokens assert!( - recorded.iter().any(|e| matches!(e, Ev2::Step(s) if s.is_final && s.tokens.input_tokens == 10)), + recorded + .iter() + .any(|e| matches!(e, Ev2::Step(s) if s.is_final && s.tokens.input_tokens == 10)), "should have a final step with 10 input tokens" ); // Should complete with the full text assert!( - recorded.iter().any(|e| matches!(e, Ev2::Complete(t) if t == "Hello world")), + recorded + .iter() + .any(|e| matches!(e, Ev2::Complete(t) if t == "Hello world")), "should complete with 'Hello world'" ); @@ -604,13 +742,10 @@ async fn test_solve_with_mock_openai() { #[tokio::test] async fn test_solve_http_error_emits_error() { use op_core::config::AgentConfig; - use op_core::engine::{solve, SolveEmitter}; + use op_core::engine::{SolveEmitter, solve}; use op_core::events::StepEvent; - let addr = start_error_server( - 401, - r#"{"error":{"message":"Invalid API key"}}"#, - ).await; + let addr = start_error_server(401, r#"{"error":{"message":"Invalid API key"}}"#).await; struct ErrorEmitter { errors: Arc>>, @@ -626,7 +761,9 @@ async fn test_solve_http_error_emits_error() { } let errors = Arc::new(Mutex::new(Vec::new())); - let emitter = ErrorEmitter { errors: errors.clone() }; + let emitter = ErrorEmitter { + errors: errors.clone(), + }; let cfg = AgentConfig { provider: "openai".into(), @@ -642,16 +779,117 @@ async fn test_solve_http_error_emits_error() { solve("Test", &cfg, &emitter, cancel).await; let recorded = errors.lock().unwrap().clone(); + assert!(!recorded.is_empty(), "should emit an error for HTTP 401"); +} + +#[tokio::test] +async fn test_solve_rate_limit_retry_eventually_completes() { + use op_core::config::AgentConfig; + use op_core::engine::{SolveEmitter, solve}; + use op_core::events::StepEvent; + + #[derive(Debug, Clone)] + #[allow(dead_code)] + enum Ev { + Trace(String), + Complete(String), + Error(String), + } + + struct RetryEmitter { + events: Arc>>, + } + + impl SolveEmitter for RetryEmitter { + fn emit_trace(&self, message: &str) { + self.events + .lock() + .unwrap() + .push(Ev::Trace(message.to_string())); + } + + fn emit_delta(&self, _: DeltaEvent) {} + + fn emit_step(&self, _: StepEvent) {} + + fn emit_complete(&self, result: &str) { + self.events + .lock() + .unwrap() + .push(Ev::Complete(result.to_string())); + } + + fn emit_error(&self, message: &str) { + self.events + .lock() + .unwrap() + .push(Ev::Error(message.to_string())); + } + } + + let addr = start_stateful_http_server(vec![ + MockHttpResponse { + status: 429, + content_type: "application/json", + body: r#"{"error":{"message":"Too many requests","code":"1302"}}"#, + headers: vec![("retry-after", "0")], + }, + MockHttpResponse { + status: 200, + content_type: "text/event-stream", + body: OPENAI_SSE_SIMPLE, + headers: vec![("cache-control", "no-cache")], + }, + ]) + .await; + + let events = Arc::new(Mutex::new(Vec::new())); + let emitter = RetryEmitter { + events: events.clone(), + }; + + let cfg = AgentConfig { + provider: "zai".into(), + model: "glm-5".into(), + zai_api_key: Some("zai-key".into()), + zai_base_url: format!("http://{addr}"), + zai_paygo_base_url: format!("http://{addr}"), + zai_coding_base_url: format!("http://{addr}"), + rate_limit_max_retries: 1, + rate_limit_backoff_base_sec: 0.0, + rate_limit_backoff_max_sec: 0.0, + rate_limit_retry_after_cap_sec: 0.0, + zai_stream_max_retries: 1, + demo: false, + ..Default::default() + }; + + let cancel = CancellationToken::new(); + solve("Test", &cfg, &emitter, cancel).await; + + let recorded = events.lock().unwrap().clone(); assert!( - !recorded.is_empty(), - "should emit an error for HTTP 401" + recorded.iter().any(|event| { + matches!(event, Ev::Trace(message) if message.contains("rate limited (1302)")) + }), + "expected a retry trace after the 429, got: {recorded:?}" + ); + assert!( + recorded + .iter() + .any(|event| matches!(event, Ev::Complete(text) if text == "Hello world")), + "expected the solve to complete after retry, got: {recorded:?}" + ); + assert!( + !recorded.iter().any(|event| matches!(event, Ev::Error(_))), + "did not expect an error after retry success, got: {recorded:?}" ); } #[tokio::test] async fn test_solve_cancel_emits_cancelled() { use op_core::config::AgentConfig; - use op_core::engine::{solve, SolveEmitter}; + use op_core::engine::{SolveEmitter, solve}; use op_core::events::StepEvent; // Use a server that returns data but we cancel before processing @@ -671,7 +909,9 @@ async fn test_solve_cancel_emits_cancelled() { } let events = Arc::new(Mutex::new(Vec::new())); - let emitter = CancelEmitter { events: events.clone() }; + let emitter = CancelEmitter { + events: events.clone(), + }; let cfg = AgentConfig { provider: "anthropic".into(), @@ -697,7 +937,7 @@ async fn test_solve_cancel_emits_cancelled() { #[tokio::test] async fn test_solve_demo_mode_bypasses_llm() { use op_core::config::AgentConfig; - use op_core::engine::{solve, SolveEmitter}; + use op_core::engine::{SolveEmitter, solve}; use op_core::events::StepEvent; struct TestEmitter { @@ -716,7 +956,9 @@ async fn test_solve_demo_mode_bypasses_llm() { } let events = Arc::new(Mutex::new(Vec::new())); - let emitter = TestEmitter { events: events.clone() }; + let emitter = TestEmitter { + events: events.clone(), + }; let cfg = AgentConfig { demo: true, @@ -736,7 +978,7 @@ async fn test_solve_demo_mode_bypasses_llm() { #[tokio::test] async fn test_solve_missing_key_emits_error() { use op_core::config::AgentConfig; - use op_core::engine::{solve, SolveEmitter}; + use op_core::engine::{SolveEmitter, solve}; use op_core::events::StepEvent; struct TestEmitter { @@ -753,11 +995,17 @@ async fn test_solve_missing_key_emits_error() { } let errors = Arc::new(Mutex::new(Vec::new())); - let emitter = TestEmitter { errors: errors.clone() }; + let emitter = TestEmitter { + errors: errors.clone(), + }; let cfg = AgentConfig { provider: "openai".into(), model: "gpt-4o".into(), + base_url: "https://api.openai.com/v1".into(), + openai_base_url: "https://api.openai.com/v1".into(), + api_key: None, + openai_api_key: None, demo: false, // No API key set ..Default::default() @@ -840,14 +1088,12 @@ async fn start_stateful_mock_server(responses: Vec<&'static str>) -> SocketAddr #[tokio::test] async fn test_solve_multi_step_agentic_loop() { use op_core::config::AgentConfig; - use op_core::engine::{solve, SolveEmitter}; + use op_core::engine::{SolveEmitter, solve}; use op_core::events::StepEvent; // Mock server: first call → tool call, second call → final answer - let addr = start_stateful_mock_server(vec![ - ANTHROPIC_SSE_TOOL_LIST, - ANTHROPIC_SSE_FINAL_ANSWER, - ]).await; + let addr = + start_stateful_mock_server(vec![ANTHROPIC_SSE_TOOL_LIST, ANTHROPIC_SSE_FINAL_ANSWER]).await; #[derive(Debug, Clone)] #[allow(dead_code)] @@ -864,7 +1110,10 @@ async fn test_solve_multi_step_agentic_loop() { } impl SolveEmitter for TestEmitter3 { fn emit_trace(&self, message: &str) { - self.events.lock().unwrap().push(Ev3::Trace(message.to_string())); + self.events + .lock() + .unwrap() + .push(Ev3::Trace(message.to_string())); } fn emit_delta(&self, event: DeltaEvent) { self.events.lock().unwrap().push(Ev3::Delta(event)); @@ -873,15 +1122,23 @@ async fn test_solve_multi_step_agentic_loop() { self.events.lock().unwrap().push(Ev3::Step(event)); } fn emit_complete(&self, result: &str) { - self.events.lock().unwrap().push(Ev3::Complete(result.to_string())); + self.events + .lock() + .unwrap() + .push(Ev3::Complete(result.to_string())); } fn emit_error(&self, message: &str) { - self.events.lock().unwrap().push(Ev3::Error(message.to_string())); + self.events + .lock() + .unwrap() + .push(Ev3::Error(message.to_string())); } } let events = Arc::new(Mutex::new(Vec::new())); - let emitter = TestEmitter3 { events: events.clone() }; + let emitter = TestEmitter3 { + events: events.clone(), + }; // Use a temp dir as workspace so list_files has something to work with let tmp = tempfile::TempDir::new().unwrap(); @@ -930,14 +1187,16 @@ async fn test_solve_multi_step_agentic_loop() { ); // Last step should be final - assert!( - steps.last().unwrap().is_final, - "last step should be final" - ); + assert!(steps.last().unwrap().is_final, "last step should be final"); // Should have tool execution trace - let has_tool_trace = recorded.iter().any(|e| matches!(e, Ev3::Trace(m) if m.contains("list_files"))); - assert!(has_tool_trace, "should have a trace mentioning list_files tool execution"); + let has_tool_trace = recorded + .iter() + .any(|e| matches!(e, Ev3::Trace(m) if m.contains("list_files"))); + assert!( + has_tool_trace, + "should have a trace mentioning list_files tool execution" + ); // Should have text deltas from both steps let text_content: String = recorded @@ -958,7 +1217,9 @@ async fn test_solve_multi_step_agentic_loop() { // Should complete with the final answer text assert!( - recorded.iter().any(|e| matches!(e, Ev3::Complete(t) if t.contains("Here is the answer"))), + recorded + .iter() + .any(|e| matches!(e, Ev3::Complete(t) if t.contains("Here is the answer"))), "should complete with the final answer" ); diff --git a/openplanter-desktop/crates/op-tauri/Cargo.toml b/openplanter-desktop/crates/op-tauri/Cargo.toml index 8b7200bb..a70210ef 100644 --- a/openplanter-desktop/crates/op-tauri/Cargo.toml +++ b/openplanter-desktop/crates/op-tauri/Cargo.toml @@ -11,7 +11,6 @@ tauri-plugin-shell = "2" tokio = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -uuid = { workspace = true } chrono = { workspace = true } tokio-util = { workspace = true } regex = { workspace = true } diff --git a/openplanter-desktop/crates/op-tauri/src/bridge.rs b/openplanter-desktop/crates/op-tauri/src/bridge.rs index 8904bd1d..bc8aa326 100644 --- a/openplanter-desktop/crates/op-tauri/src/bridge.rs +++ b/openplanter-desktop/crates/op-tauri/src/bridge.rs @@ -10,7 +10,9 @@ use std::sync::{Arc, Mutex}; use tauri::{AppHandle, Emitter}; use op_core::engine::SolveEmitter; -use op_core::events::{CompleteEvent, CuratorUpdateEvent, DeltaEvent, DeltaKind, ErrorEvent, StepEvent, TraceEvent}; +use op_core::events::{ + CompleteEvent, CuratorUpdateEvent, DeltaEvent, DeltaKind, ErrorEvent, StepEvent, TraceEvent, +}; use op_core::session::replay::{ReplayEntry, ReplayLogger, StepToolCallEntry}; pub struct TauriEmitter { @@ -35,12 +37,18 @@ impl SolveEmitter for TauriEmitter { } fn emit_delta(&self, event: DeltaEvent) { - eprintln!("[bridge] delta: kind={:?} text={:?}", event.kind, event.text); + eprintln!( + "[bridge] delta: kind={:?} text={:?}", + event.kind, event.text + ); let _ = self.handle.emit("agent:delta", event); } fn emit_step(&self, event: StepEvent) { - eprintln!("[bridge] step: depth={} step={} is_final={}", event.depth, event.step, event.is_final); + eprintln!( + "[bridge] step: depth={} step={} is_final={}", + event.depth, event.step, event.is_final + ); let _ = self.handle.emit("agent:step", event); } @@ -172,7 +180,11 @@ impl SolveEmitter for LoggingEmitter { let model_preview = { let buf = self.streaming_buf.lock().unwrap(); let trimmed = buf.trim().to_string(); - if trimmed.is_empty() { None } else { Some(trimmed) } + if trimmed.is_empty() { + None + } else { + Some(trimmed) + } }; let step_tools: Vec = { @@ -327,7 +339,12 @@ mod tests { assert_eq!(step.step_number, Some(1)); assert!(step.step_tokens_in.is_some()); assert!(step.step_model_preview.is_some()); - assert!(step.step_model_preview.as_ref().unwrap().contains("Test persistence")); + assert!( + step.step_model_preview + .as_ref() + .unwrap() + .contains("Test persistence") + ); let assistant = entries.iter().find(|e| e.role == "assistant"); assert!(assistant.is_some(), "expected an assistant entry"); @@ -355,20 +372,23 @@ mod tests { // 1. Log user message let mut replay = ReplayLogger::new(tmp.path()); - replay.append(ReplayEntry { - seq: 0, - timestamp: String::new(), - role: "user".into(), - content: "Roundtrip test".into(), - tool_name: None, - is_rendered: None, - step_number: None, - step_tokens_in: None, - step_tokens_out: None, - step_elapsed: None, - step_model_preview: None, - step_tool_calls: None, - }).await.unwrap(); + replay + .append(ReplayEntry { + seq: 0, + timestamp: String::new(), + role: "user".into(), + content: "Roundtrip test".into(), + tool_name: None, + is_rendered: None, + step_number: None, + step_tokens_in: None, + step_tokens_out: None, + step_elapsed: None, + step_model_preview: None, + step_tool_calls: None, + }) + .await + .unwrap(); // 2. Run demo_solve through LoggingEmitter let emitter = LoggingEmitter::new(NullEmitter, replay); @@ -377,7 +397,11 @@ mod tests { // 3. Read back full conversation let entries = ReplayLogger::read_all(tmp.path()).await.unwrap(); - assert!(entries.len() >= 3, "expected user + step-summary + assistant, got {}", entries.len()); + assert!( + entries.len() >= 3, + "expected user + step-summary + assistant, got {}", + entries.len() + ); assert_eq!(entries[0].role, "user"); assert_eq!(entries[0].content, "Roundtrip test"); diff --git a/openplanter-desktop/crates/op-tauri/src/commands/agent.rs b/openplanter-desktop/crates/op-tauri/src/commands/agent.rs index 2144ab50..fc40649b 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/agent.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/agent.rs @@ -67,12 +67,7 @@ pub async fn solve( if let Err(e) = result { let msg = format!("Internal error: {e}"); eprintln!("[bridge] panic: {msg}"); - let _ = error_handle.emit( - "agent:error", - op_core::events::ErrorEvent { - message: msg, - }, - ); + let _ = error_handle.emit("agent:error", op_core::events::ErrorEvent { message: msg }); } }); @@ -81,9 +76,7 @@ pub async fn solve( /// Cancel a running solve. #[tauri::command] -pub async fn cancel( - state: State<'_, AppState>, -) -> Result<(), String> { +pub async fn cancel(state: State<'_, AppState>) -> Result<(), String> { let token = state.cancel_token.lock().await; token.cancel(); Ok(()) diff --git a/openplanter-desktop/crates/op-tauri/src/commands/config.rs b/openplanter-desktop/crates/op-tauri/src/commands/config.rs index 2015140c..7bfecbc6 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/config.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/config.rs @@ -1,28 +1,65 @@ -use std::collections::HashMap; -use tauri::State; use crate::state::AppState; +use op_core::config::{normalize_web_search_provider, normalize_zai_plan, resolve_zai_base_url}; +use op_core::credentials::credentials_from_env; use op_core::events::{ConfigView, ModelInfo, PartialConfig}; use op_core::settings::{PersistentSettings, SettingsStore}; -use op_core::credentials::credentials_from_env; +use std::collections::HashMap; +use tauri::State; -/// Get the current configuration. -#[tauri::command] -pub async fn get_config( - state: State<'_, AppState>, -) -> Result { - let cfg = state.config.lock().await; - let session_id = state.session_id.lock().await; - Ok(ConfigView { +fn make_config_view(cfg: &op_core::config::AgentConfig, session_id: Option) -> ConfigView { + ConfigView { provider: cfg.provider.clone(), model: cfg.model.clone(), reasoning_effort: cfg.reasoning_effort.clone(), + zai_plan: cfg.zai_plan.clone(), + web_search_provider: cfg.web_search_provider.clone(), workspace: cfg.workspace.display().to_string(), - session_id: session_id.clone(), + session_id, recursive: cfg.recursive, max_depth: cfg.max_depth, max_steps_per_call: cfg.max_steps_per_call, demo: cfg.demo, - }) + } +} + +fn merge_settings( + existing: PersistentSettings, + incoming: PersistentSettings, +) -> PersistentSettings { + PersistentSettings { + default_model: incoming.default_model.or(existing.default_model), + default_reasoning_effort: incoming + .default_reasoning_effort + .or(existing.default_reasoning_effort), + default_model_openai: incoming + .default_model_openai + .or(existing.default_model_openai), + default_model_anthropic: incoming + .default_model_anthropic + .or(existing.default_model_anthropic), + default_model_openrouter: incoming + .default_model_openrouter + .or(existing.default_model_openrouter), + default_model_cerebras: incoming + .default_model_cerebras + .or(existing.default_model_cerebras), + default_model_zai: incoming.default_model_zai.or(existing.default_model_zai), + default_model_ollama: incoming + .default_model_ollama + .or(existing.default_model_ollama), + zai_plan: incoming.zai_plan.or(existing.zai_plan), + web_search_provider: incoming + .web_search_provider + .or(existing.web_search_provider), + } +} + +/// Get the current configuration. +#[tauri::command] +pub async fn get_config(state: State<'_, AppState>) -> Result { + let cfg = state.config.lock().await; + let session_id = state.session_id.lock().await; + Ok(make_config_view(&cfg, session_id.clone())) } /// Update configuration fields. @@ -45,35 +82,35 @@ pub async fn update_config( Some(effort) }; } + if let Some(plan) = partial.zai_plan { + cfg.zai_plan = normalize_zai_plan(Some(&plan)); + cfg.zai_base_url = resolve_zai_base_url( + &cfg.zai_plan, + &cfg.zai_paygo_base_url, + &cfg.zai_coding_base_url, + ); + } + if let Some(provider) = partial.web_search_provider { + cfg.web_search_provider = normalize_web_search_provider(Some(&provider)); + } let session_id = state.session_id.lock().await; - Ok(ConfigView { - provider: cfg.provider.clone(), - model: cfg.model.clone(), - reasoning_effort: cfg.reasoning_effort.clone(), - workspace: cfg.workspace.display().to_string(), - session_id: session_id.clone(), - recursive: cfg.recursive, - max_depth: cfg.max_depth, - max_steps_per_call: cfg.max_steps_per_call, - demo: cfg.demo, - }) + Ok(make_config_view(&cfg, session_id.clone())) } /// Known models per provider for listing. fn known_models_for_provider(provider: &str) -> Vec { let models: Vec<(&str, &str)> = match provider { "openai" => vec![ - ("gpt-5.2", "GPT-5.2"), - ("gpt-4o", "GPT-4o"), - ("gpt-4o-mini", "GPT-4o Mini"), - ("o1", "o1"), - ("o3", "o3"), - ("o4-mini", "o4-mini"), + ("azure-foundry/gpt-5.3-codex", "GPT-5.3 Codex (Foundry)"), + ("azure-foundry/Kimi-K2.5", "Kimi K2.5 (Foundry)"), ], "anthropic" => vec![ - ("claude-opus-4-6", "Claude Opus 4.6"), - ("claude-sonnet-4-5", "Claude Sonnet 4.5"), - ("claude-haiku-4-5", "Claude Haiku 4.5"), + ("anthropic-foundry/claude-opus-4-6", "Claude Opus 4.6 (Foundry)"), + ( + "anthropic-foundry/claude-sonnet-4-6", + "Claude Sonnet 4.6 (Foundry)", + ), + ("anthropic-foundry/claude-haiku-4-5", "Claude Haiku 4.5 (Foundry)"), ], "openrouter" => vec![ ("anthropic/claude-sonnet-4-5", "Claude Sonnet 4.5 (OR)"), @@ -84,6 +121,11 @@ fn known_models_for_provider(provider: &str) -> Vec { ("qwen-3-235b-a22b-instruct-2507", "Qwen-3 235B"), ("llama-4-scout-17b-16e-instruct", "Llama-4 Scout"), ], + "zai" => vec![ + ("glm-5", "GLM-5"), + ("glm-4.6", "GLM-4.6"), + ("zai-glm-4.6", "Z.AI GLM 4.6"), + ], "ollama" => vec![ ("llama3.2", "Llama 3.2"), ("mistral", "Mistral"), @@ -113,7 +155,14 @@ pub async fn list_models( ) -> Result, String> { if provider == "all" { let mut all = Vec::new(); - for p in &["openai", "anthropic", "openrouter", "cerebras", "ollama"] { + for p in &[ + "openai", + "anthropic", + "openrouter", + "cerebras", + "zai", + "ollama", + ] { all.extend(known_models_for_provider(p)); } Ok(all) @@ -130,7 +179,8 @@ pub async fn save_settings( ) -> Result<(), String> { let cfg = state.config.lock().await; let store = SettingsStore::new(&cfg.workspace, &cfg.session_root_dir); - store.save(&settings).map_err(|e| e.to_string()) + let merged = merge_settings(store.load(), settings); + store.save(&merged).map_err(|e| e.to_string()) } /// Build credential status from config: which providers/services have API keys configured. @@ -140,8 +190,10 @@ pub fn build_credential_status(cfg: &op_core::config::AgentConfig) -> HashMap) -> PathBuf { @@ -54,11 +54,7 @@ pub fn create_session(dir: &Path) -> Result { fs::create_dir_all(dir)?; let now = chrono::Utc::now(); - let new_id = format!( - "{}-{:08x}", - now.format("%Y%m%d-%H%M%S"), - rand_hex() - ); + let new_id = format!("{}-{:08x}", now.format("%Y%m%d-%H%M%S"), rand_hex()); let session_dir = dir.join(&new_id); fs::create_dir_all(&session_dir)?; @@ -120,10 +116,7 @@ pub async fn open_session( /// Delete a session by removing its directory. #[tauri::command] -pub async fn delete_session( - id: String, - state: State<'_, AppState>, -) -> Result<(), String> { +pub async fn delete_session(id: String, state: State<'_, AppState>) -> Result<(), String> { let dir = sessions_dir(&state).await; let session_dir = dir.join(&id); @@ -135,7 +128,9 @@ pub async fn delete_session( } // Ensure it's actually a session directory (has metadata.json) if !session_dir.join("metadata.json").exists() { - return Err(format!("Session '{id}' has no metadata — refusing to delete")); + return Err(format!( + "Session '{id}' has no metadata — refusing to delete" + )); } fs::remove_dir_all(&session_dir).map_err(|e| format!("Failed to delete session: {e}"))?; @@ -156,7 +151,9 @@ pub async fn get_session_history( state: State<'_, AppState>, ) -> Result, String> { let dir = sessions_dir(&state).await.join(&session_id); - ReplayLogger::read_all(&dir).await.map_err(|e| e.to_string()) + ReplayLogger::read_all(&dir) + .await + .map_err(|e| e.to_string()) } /// Update session metadata: increment turn_count, set last_objective. @@ -172,13 +169,11 @@ pub async fn update_session_metadata( let mut info: SessionInfo = serde_json::from_str(&content) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; info.turn_count += 1; - info.last_objective = Some( - if objective.len() > 100 { - format!("{}...", &objective[..97]) - } else { - objective.to_string() - }, - ); + info.last_objective = Some(if objective.len() > 100 { + format!("{}...", &objective[..97]) + } else { + objective.to_string() + }); let json = serde_json::to_string_pretty(&info) .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; tokio::fs::write(&meta_path, json).await @@ -317,8 +312,14 @@ mod tests { let info = create_session(&dir).unwrap(); let session_dir = dir.join(&info.id); assert!(session_dir.exists(), "session dir should exist"); - assert!(session_dir.join("artifacts").exists(), "artifacts/ should exist"); - assert!(session_dir.join("metadata.json").exists(), "metadata.json should exist"); + assert!( + session_dir.join("artifacts").exists(), + "artifacts/ should exist" + ); + assert!( + session_dir.join("metadata.json").exists(), + "metadata.json should exist" + ); } #[test] diff --git a/openplanter-desktop/crates/op-tauri/src/commands/wiki.rs b/openplanter-desktop/crates/op-tauri/src/commands/wiki.rs index de84697e..53df0a6e 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/wiki.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/wiki.rs @@ -1,16 +1,15 @@ +use crate::state::AppState; +use op_core::events::{GraphData, GraphEdge, GraphNode, NodeType}; +use regex::Regex; use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; use std::sync::LazyLock; -use regex::Regex; use tauri::State; -use crate::state::AppState; -use op_core::events::{GraphData, GraphEdge, GraphNode, NodeType}; static LINK_RE: LazyLock = LazyLock::new(|| Regex::new(r"\[([^\]]+)\]\(([^)]+\.md)\)").unwrap()); -static CATEGORY_RE: LazyLock = - LazyLock::new(|| Regex::new(r"^#{2,3}\s+(.+)").unwrap()); +static CATEGORY_RE: LazyLock = LazyLock::new(|| Regex::new(r"^#{2,3}\s+(.+)").unwrap()); /// Walk up from `start` to find a directory containing `wiki/index.md`. /// Checks both `.openplanter/wiki/` (preferred) and `wiki/` at each level. @@ -117,27 +116,47 @@ pub fn parse_index_nodes(content: &str) -> Vec { /// Extract distinctive search terms from a node's label for text-based matching. fn search_terms_for_node(node: &GraphNode) -> Vec { let stopwords: HashSet<&str> = [ - "a", "an", "the", "of", "and", "or", "in", "to", "for", "by", - "on", "at", "is", "it", "its", "us", "gov", "list", - ].into_iter().collect(); + "a", "an", "the", "of", "and", "or", "in", "to", "for", "by", "on", "at", "is", "it", + "its", "us", "gov", "list", + ] + .into_iter() + .collect(); let generic: HashSet<&str> = [ - "federal", "state", "united", "states", "government", "bureau", - "department", "database", "national", "public", - ].into_iter().collect(); + "federal", + "state", + "united", + "states", + "government", + "bureau", + "department", + "database", + "national", + "public", + ] + .into_iter() + .collect(); let mut terms = Vec::new(); // Full label (lowercased) terms.push(node.label.to_lowercase()); - for word in node.label.split(|c: char| c.is_whitespace() || c == '/' || c == '(' || c == ')') { - let clean: String = word.chars() + for word in node + .label + .split(|c: char| c.is_whitespace() || c == '/' || c == '(' || c == ')') + { + let clean: String = word + .chars() .filter(|c| c.is_alphanumeric() || *c == '.' || *c == '-') .collect(); - if clean.is_empty() { continue; } + if clean.is_empty() { + continue; + } let lower = clean.to_lowercase(); - if stopwords.contains(lower.as_str()) { continue; } + if stopwords.contains(lower.as_str()) { + continue; + } // Acronyms: all uppercase, >= 2 chars (OCPF, FEC, EDGAR, FDIC, etc.) let alpha_chars: String = clean.chars().filter(|c| c.is_alphabetic()).collect(); @@ -165,15 +184,16 @@ pub fn find_cross_references(nodes: &[GraphNode], wiki_dir: &Path) -> Vec = HashSet::new(); // Pre-compute search terms for all nodes - let node_terms: Vec> = nodes.iter() - .map(|n| search_terms_for_node(n)) - .collect(); + let node_terms: Vec> = nodes.iter().map(|n| search_terms_for_node(n)).collect(); // Read all file contents upfront - let file_contents: HashMap = nodes.iter() + let file_contents: HashMap = nodes + .iter() .filter_map(|node| { let file_path = wiki_dir.join(&node.path); - fs::read_to_string(&file_path).ok().map(|c| (node.id.clone(), c)) + fs::read_to_string(&file_path) + .ok() + .map(|c| (node.id.clone(), c)) }) .collect(); @@ -207,11 +227,17 @@ pub fn find_cross_references(nodes: &[GraphNode], wiki_dir: &Path) -> Vec = nodes.iter().map(|n| n.id.clone()).collect(); // Find section IDs that are the source of at least one structural child edge - let parent_section_ids: HashSet<&str> = edges.iter() + let parent_section_ids: HashSet<&str> = edges + .iter() .filter(|e| { let label = e.label.as_deref().unwrap_or(""); (label == "has-section" || label == "contains") && node_ids.contains(&e.target) @@ -499,7 +526,8 @@ pub fn parse_source_file( .collect(); // IDs to remove: childless sections + empty-content facts - let remove_ids: HashSet = nodes.iter() + let remove_ids: HashSet = nodes + .iter() .filter(|n| { match n.node_type.as_ref() { Some(NodeType::Section) => !parent_section_ids.contains(n.id.as_str()), @@ -554,9 +582,10 @@ pub fn extract_cross_ref_edges( continue; } // Check if this fact is under a cross-reference section - let in_cross_ref = node.parent_id.as_ref().map_or(false, |pid| { - pid.contains("cross-reference") - }); + let in_cross_ref = node + .parent_id + .as_ref() + .map_or(false, |pid| pid.contains("cross-reference")); if !in_cross_ref { continue; } @@ -604,15 +633,21 @@ pub fn find_shared_field_edges(all_nodes: &[GraphNode]) -> Vec { continue; } // Check if this fact is under a data-schema section - let in_data_schema = node.parent_id.as_ref().map_or(false, |pid| { - pid.contains("data-schema") - }); + let in_data_schema = node + .parent_id + .as_ref() + .map_or(false, |pid| pid.contains("data-schema")); if !in_data_schema { continue; } // Normalize field name: lowercase, strip backticks - let normalized = node.label.to_lowercase().replace('`', "").trim().to_string(); + let normalized = node + .label + .to_lowercase() + .replace('`', "") + .trim() + .to_string(); if !normalized.is_empty() { field_map.entry(normalized).or_default().push(node); } @@ -651,13 +686,16 @@ pub fn find_shared_field_edges(all_nodes: &[GraphNode]) -> Vec { /// Get the wiki knowledge graph data by parsing wiki/index.md and all source files. #[tauri::command] -pub async fn get_graph_data( - state: State<'_, AppState>, -) -> Result { +pub async fn get_graph_data(state: State<'_, AppState>) -> Result { let cfg = state.config.lock().await; let wiki_dir = match find_wiki_dir(&cfg.workspace) { Some(d) => d, - None => return Ok(GraphData { nodes: vec![], edges: vec![] }), + None => { + return Ok(GraphData { + nodes: vec![], + edges: vec![], + }); + } }; let index_path = wiki_dir.join("index.md"); @@ -690,15 +728,15 @@ pub async fn get_graph_data( let shared_field_edges = find_shared_field_edges(&all_nodes); all_edges.extend(shared_field_edges); - Ok(GraphData { nodes: all_nodes, edges: all_edges }) + Ok(GraphData { + nodes: all_nodes, + edges: all_edges, + }) } /// Read a wiki markdown file's contents, given a relative path like "wiki/fec.md". #[tauri::command] -pub async fn read_wiki_file( - path: String, - state: State<'_, AppState>, -) -> Result { +pub async fn read_wiki_file(path: String, state: State<'_, AppState>) -> Result { // Validate: must end in .md if !path.ends_with(".md") { return Err("Path must end in .md".into()); @@ -713,14 +751,16 @@ pub async fn read_wiki_file( } let cfg = state.config.lock().await; - let wiki_dir = find_wiki_dir(&cfg.workspace) - .ok_or_else(|| "Wiki directory not found".to_string())?; + let wiki_dir = + find_wiki_dir(&cfg.workspace).ok_or_else(|| "Wiki directory not found".to_string())?; let project_root = wiki_dir.parent().unwrap_or(&cfg.workspace); let resolved = project_root.join(&path); // Canonicalize and verify it's under the wiki dir - let canonical = resolved.canonicalize().map_err(|e| format!("File not found: {e}"))?; + let canonical = resolved + .canonicalize() + .map_err(|e| format!("File not found: {e}"))?; let canon_wiki = wiki_dir.canonicalize().map_err(|e| e.to_string())?; if !canonical.starts_with(&canon_wiki) { return Err("Path is outside wiki directory".into()); @@ -862,7 +902,9 @@ mod tests { label: "A".to_string(), category: "test".to_string(), path: "wiki/a.md".to_string(), - node_type: None, parent_id: None, content: None, + node_type: None, + parent_id: None, + content: None, }]; let edges = find_cross_references(&nodes, tmp.path()); assert!(edges.is_empty()); @@ -883,14 +925,18 @@ mod tests { label: "A".to_string(), category: "test".to_string(), path: "wiki/a.md".to_string(), - node_type: None, parent_id: None, content: None, + node_type: None, + parent_id: None, + content: None, }, GraphNode { id: "b".to_string(), label: "B".to_string(), category: "test".to_string(), path: "wiki/b.md".to_string(), - node_type: None, parent_id: None, content: None, + node_type: None, + parent_id: None, + content: None, }, ]; let edges = find_cross_references(&nodes, tmp.path()); @@ -996,7 +1042,10 @@ mod tests { // project_root should be .openplanter/ so joining with wiki/fec.md works let project_root = found.parent().unwrap(); let file_path = project_root.join(&nodes[0].path); - assert!(file_path.exists(), "should resolve to .openplanter/wiki/fec.md"); + assert!( + file_path.exists(), + "should resolve to .openplanter/wiki/fec.md" + ); } #[test] @@ -1046,7 +1095,11 @@ mod tests { let wiki_dir = tmp.path().join("wiki"); fs::create_dir_all(&wiki_dir).unwrap(); // File A mentions EDGAR (from B's label "SEC EDGAR") but doesn't link to it - fs::write(wiki_dir.join("a.md"), "Cross-reference with EDGAR filings for details.").unwrap(); + fs::write( + wiki_dir.join("a.md"), + "Cross-reference with EDGAR filings for details.", + ) + .unwrap(); fs::write(wiki_dir.join("b.md"), "# SEC EDGAR\nContent.").unwrap(); let nodes = vec![ @@ -1055,14 +1108,18 @@ mod tests { label: "FEC Data".to_string(), category: "campaign-finance".to_string(), path: "wiki/a.md".to_string(), - node_type: None, parent_id: None, content: None, + node_type: None, + parent_id: None, + content: None, }, GraphNode { id: "b".to_string(), label: "SEC EDGAR".to_string(), category: "corporate".to_string(), path: "wiki/b.md".to_string(), - node_type: None, parent_id: None, content: None, + node_type: None, + parent_id: None, + content: None, }, ]; let edges = find_cross_references(&nodes, tmp.path()); @@ -1080,17 +1137,20 @@ mod tests { // File A mentions its own label — should not create edge fs::write(wiki_dir.join("a.md"), "# EDGAR\nThis is SEC EDGAR data.").unwrap(); - let nodes = vec![ - GraphNode { - id: "a".to_string(), - label: "SEC EDGAR".to_string(), - category: "corporate".to_string(), - path: "wiki/a.md".to_string(), - node_type: None, parent_id: None, content: None, - }, - ]; + let nodes = vec![GraphNode { + id: "a".to_string(), + label: "SEC EDGAR".to_string(), + category: "corporate".to_string(), + path: "wiki/a.md".to_string(), + node_type: None, + parent_id: None, + content: None, + }]; let edges = find_cross_references(&nodes, tmp.path()); - assert!(edges.is_empty(), "should not create self-referencing edge from text mention"); + assert!( + edges.is_empty(), + "should not create self-referencing edge from text mention" + ); } #[test] @@ -1107,14 +1167,18 @@ mod tests { label: "EPA Data".to_string(), category: "regulatory".to_string(), path: "wiki/a.md".to_string(), - node_type: None, parent_id: None, content: None, + node_type: None, + parent_id: None, + content: None, }, GraphNode { id: "b".to_string(), label: "OSHA Inspections".to_string(), category: "regulatory".to_string(), path: "wiki/b.md".to_string(), - node_type: None, parent_id: None, content: None, + node_type: None, + parent_id: None, + content: None, }, ]; let edges = find_cross_references(&nodes, tmp.path()); @@ -1136,14 +1200,18 @@ mod tests { label: "A Data".to_string(), category: "test".to_string(), path: "wiki/a.md".to_string(), - node_type: None, parent_id: None, content: None, + node_type: None, + parent_id: None, + content: None, }, GraphNode { id: "b".to_string(), label: "SEC EDGAR".to_string(), category: "corporate".to_string(), path: "wiki/b.md".to_string(), - node_type: None, parent_id: None, content: None, + node_type: None, + parent_id: None, + content: None, }, ]; let edges = find_cross_references(&nodes, tmp.path()); @@ -1163,7 +1231,9 @@ mod tests { label: "A".to_string(), category: "test".to_string(), path: "wiki/a.md".to_string(), - node_type: None, parent_id: None, content: None, + node_type: None, + parent_id: None, + content: None, }]; let edges = find_cross_references(&nodes, tmp.path()); assert!(edges.is_empty(), "self-references should be excluded"); @@ -1174,7 +1244,10 @@ mod tests { #[test] fn test_slugify_basic() { assert_eq!(slugify("Data Schema"), "data-schema"); - assert_eq!(slugify("Cross-Reference Potential"), "cross-reference-potential"); + assert_eq!( + slugify("Cross-Reference Potential"), + "cross-reference-potential" + ); assert_eq!(slugify("Legal & Licensing"), "legal-licensing"); assert_eq!(slugify(" multiple spaces "), "multiple-spaces"); } @@ -1258,13 +1331,19 @@ mod tests { let (nodes, edges) = parse_source_file(&source, content); // Data Schema + 2 subsections + 2 facts = 5 assert_eq!(nodes.len(), 5); - let sections: Vec<_> = nodes.iter().filter(|n| n.node_type == Some(NodeType::Section)).collect(); + let sections: Vec<_> = nodes + .iter() + .filter(|n| n.node_type == Some(NodeType::Section)) + .collect(); assert_eq!(sections.len(), 3); // Subsections are children of the h2 assert_eq!(sections[1].parent_id.as_deref(), Some("fec::data-schema")); assert_eq!(sections[2].parent_id.as_deref(), Some("fec::data-schema")); // has-section edges - let has_section: Vec<_> = edges.iter().filter(|e| e.label.as_deref() == Some("has-section")).collect(); + let has_section: Vec<_> = edges + .iter() + .filter(|e| e.label.as_deref() == Some("has-section")) + .collect(); assert_eq!(has_section.len(), 3); } @@ -1275,16 +1354,26 @@ mod tests { let (nodes, edges) = parse_source_file(&source, content); // 1 section + 2 facts assert_eq!(nodes.len(), 3); - let facts: Vec<_> = nodes.iter().filter(|n| n.node_type == Some(NodeType::Fact)).collect(); + let facts: Vec<_> = nodes + .iter() + .filter(|n| n.node_type == Some(NodeType::Fact)) + .collect(); assert_eq!(facts.len(), 2); assert_eq!(facts[0].label, "Jurisdiction"); assert_eq!(facts[1].label, "Time range"); // Facts should have content assert!(facts[0].content.as_ref().unwrap().contains("Federal")); // Facts parented to section - assert!(facts.iter().all(|f| f.parent_id.as_deref() == Some("fec::coverage"))); + assert!( + facts + .iter() + .all(|f| f.parent_id.as_deref() == Some("fec::coverage")) + ); // Contains edges - let contains: Vec<_> = edges.iter().filter(|e| e.label.as_deref() == Some("contains")).collect(); + let contains: Vec<_> = edges + .iter() + .filter(|e| e.label.as_deref() == Some("contains")) + .collect(); assert_eq!(contains.len(), 2); } @@ -1293,13 +1382,22 @@ mod tests { let source = make_source("fec"); let content = "## Coverage\n\n- **Time range**:\n - Records: 1979-present\n - Contributions: 1979-present\n- **Jurisdiction**: Federal"; let (nodes, _) = parse_source_file(&source, content); - let facts: Vec<_> = nodes.iter().filter(|n| n.node_type == Some(NodeType::Fact)).collect(); + let facts: Vec<_> = nodes + .iter() + .filter(|n| n.node_type == Some(NodeType::Fact)) + .collect(); assert_eq!(facts.len(), 2); // Time range should have accumulated sub-bullets let time_range = facts.iter().find(|f| f.label == "Time range").unwrap(); let content = time_range.content.as_ref().unwrap(); - assert!(content.contains("Records: 1979-present"), "should contain sub-bullet"); - assert!(content.contains("Contributions: 1979-present"), "should contain second sub-bullet"); + assert!( + content.contains("Records: 1979-present"), + "should contain sub-bullet" + ); + assert!( + content.contains("Contributions: 1979-present"), + "should contain second sub-bullet" + ); } #[test] @@ -1308,7 +1406,10 @@ mod tests { // Bold bullet with NO sub-bullets and NO value after colon → should be pruned let content = "## Coverage\n\n- **Empty**:\n- **Jurisdiction**: Federal"; let (nodes, _) = parse_source_file(&source, content); - let facts: Vec<_> = nodes.iter().filter(|n| n.node_type == Some(NodeType::Fact)).collect(); + let facts: Vec<_> = nodes + .iter() + .filter(|n| n.node_type == Some(NodeType::Fact)) + .collect(); // "Empty" should be pruned, only "Jurisdiction" remains assert_eq!(facts.len(), 1); assert_eq!(facts[0].label, "Jurisdiction"); @@ -1320,7 +1421,10 @@ mod tests { let content = "## Data Schema\n\n| Field | Description |\n|-------|-------------|\n| `candidate_id` | Unique ID |\n| `name` | Full name |"; let (nodes, edges) = parse_source_file(&source, content); // 1 section + 2 fact rows (header + separator skipped) - let facts: Vec<_> = nodes.iter().filter(|n| n.node_type == Some(NodeType::Fact)).collect(); + let facts: Vec<_> = nodes + .iter() + .filter(|n| n.node_type == Some(NodeType::Fact)) + .collect(); assert_eq!(facts.len(), 2); assert_eq!(facts[0].label, "candidate_id"); // backticks stripped assert_eq!(facts[1].label, "name"); @@ -1331,7 +1435,10 @@ mod tests { let source = make_source("fec"); let content = "## Schema\n\n| Header1 | Header2 |\n| --- | --- |\n| value1 | desc1 |"; let (nodes, _edges) = parse_source_file(&source, content); - let facts: Vec<_> = nodes.iter().filter(|n| n.node_type == Some(NodeType::Fact)).collect(); + let facts: Vec<_> = nodes + .iter() + .filter(|n| n.node_type == Some(NodeType::Fact)) + .collect(); assert_eq!(facts.len(), 1); assert_eq!(facts[0].label, "value1"); } @@ -1339,11 +1446,17 @@ mod tests { #[test] fn test_parse_fact_parents_correct() { let source = make_source("fec"); - let content = "## Data Schema\n\n### Candidate Records\n\n| Field | Desc |\n|---|---|\n| cid | ID |"; + let content = + "## Data Schema\n\n### Candidate Records\n\n| Field | Desc |\n|---|---|\n| cid | ID |"; let (nodes, _) = parse_source_file(&source, content); let fact = nodes.iter().find(|n| n.label == "cid").unwrap(); // Fact should be parented to the h3 section, not the h2 - assert!(fact.parent_id.as_ref().unwrap().contains("candidate-records")); + assert!( + fact.parent_id + .as_ref() + .unwrap() + .contains("candidate-records") + ); } #[test] @@ -1352,7 +1465,10 @@ mod tests { // Two sections with same name, each with a fact so they survive pruning let content = "## Summary\n\n- **A**: 1\n\n## Summary\n\n- **B**: 2"; let (nodes, _) = parse_source_file(&source, content); - let sections: Vec<_> = nodes.iter().filter(|n| n.node_type == Some(NodeType::Section)).collect(); + let sections: Vec<_> = nodes + .iter() + .filter(|n| n.node_type == Some(NodeType::Section)) + .collect(); assert_eq!(sections.len(), 2); assert_eq!(sections[0].id, "fec::summary"); assert_eq!(sections[1].id, "fec::summary-2"); // deduplicated @@ -1393,15 +1509,27 @@ Overview paragraph. Links here."; let (nodes, edges) = parse_source_file(&source, content); - let sections: Vec<_> = nodes.iter().filter(|n| n.node_type == Some(NodeType::Section)).collect(); - let facts: Vec<_> = nodes.iter().filter(|n| n.node_type == Some(NodeType::Fact)).collect(); + let sections: Vec<_> = nodes + .iter() + .filter(|n| n.node_type == Some(NodeType::Section)) + .collect(); + let facts: Vec<_> = nodes + .iter() + .filter(|n| n.node_type == Some(NodeType::Fact)) + .collect(); // Summary and References pruned (no children), Coverage + Data Schema + Records remain = 3 assert_eq!(sections.len(), 3); // 2 bullets + 2 table rows = 4 facts assert_eq!(facts.len(), 4); // Structural edges: 2 has-section (Coverage→source, Data Schema→source) + 1 has-section (Records→Data Schema) + 4 contains - let has_section_count = edges.iter().filter(|e| e.label.as_deref() == Some("has-section")).count(); - let contains_count = edges.iter().filter(|e| e.label.as_deref() == Some("contains")).count(); + let has_section_count = edges + .iter() + .filter(|e| e.label.as_deref() == Some("has-section")) + .count(); + let contains_count = edges + .iter() + .filter(|e| e.label.as_deref() == Some("contains")) + .count(); assert_eq!(has_section_count, 3); assert_eq!(contains_count, 4); } @@ -1481,7 +1609,10 @@ Links here."; let all_nodes = vec![source_a.clone(), source_b.clone(), fact]; let source_nodes = vec![source_a, source_b]; let edges = extract_cross_ref_edges(&all_nodes, &source_nodes); - assert!(edges.is_empty(), "should only match facts under cross-reference sections"); + assert!( + edges.is_empty(), + "should only match facts under cross-reference sections" + ); } // ── find_shared_field_edges ── @@ -1532,7 +1663,10 @@ Links here."; content: None, }; let edges = find_shared_field_edges(&vec![fact_a, fact_b]); - assert!(edges.is_empty(), "should not create edge between same-source facts"); + assert!( + edges.is_empty(), + "should not create edge between same-source facts" + ); } #[test] @@ -1601,7 +1735,9 @@ Links here."; content: None, }; let edges = find_shared_field_edges(&vec![fact_a, fact_b]); - assert!(edges.is_empty(), "should only match facts under data-schema sections"); + assert!( + edges.is_empty(), + "should only match facts under data-schema sections" + ); } - } diff --git a/openplanter-desktop/crates/op-tauri/src/main.rs b/openplanter-desktop/crates/op-tauri/src/main.rs index e5b80c36..20088713 100644 --- a/openplanter-desktop/crates/op-tauri/src/main.rs +++ b/openplanter-desktop/crates/op-tauri/src/main.rs @@ -1,9 +1,9 @@ // Prevents additional console window on Windows in release. #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] -mod state; mod bridge; mod commands; +mod state; use state::AppState; @@ -27,8 +27,6 @@ fn main() { commands::wiki::get_graph_data, commands::wiki::read_wiki_file, ]) - .run(tauri::generate_context!( - "tauri.conf.json" - )) + .run(tauri::generate_context!("tauri.conf.json")) .expect("error while running tauri application"); } diff --git a/openplanter-desktop/crates/op-tauri/src/state.rs b/openplanter-desktop/crates/op-tauri/src/state.rs index f4b831bb..3109c5c0 100644 --- a/openplanter-desktop/crates/op-tauri/src/state.rs +++ b/openplanter-desktop/crates/op-tauri/src/state.rs @@ -1,8 +1,14 @@ +use op_core::config::{ + AgentConfig, normalize_web_search_provider, normalize_zai_plan, resolve_zai_base_url, +}; +use op_core::credentials::{ + CredentialBundle, credentials_from_env, discover_env_candidates, parse_env_file, +}; +use op_core::settings::{PersistentSettings, SettingsStore}; +use std::env; use std::sync::Arc; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; -use op_core::config::AgentConfig; -use op_core::credentials::{credentials_from_env, discover_env_candidates, parse_env_file, CredentialBundle}; /// Merge credentials into an AgentConfig. /// Priority: existing config value > env_creds > file_creds. @@ -14,7 +20,9 @@ pub fn merge_credentials_into_config( macro_rules! merge { ($field:ident) => { if cfg.$field.is_none() { - cfg.$field = env_creds.$field.clone() + cfg.$field = env_creds + .$field + .clone() .or_else(|| file_creds.$field.clone()); } }; @@ -23,10 +31,62 @@ pub fn merge_credentials_into_config( merge!(anthropic_api_key); merge!(openrouter_api_key); merge!(cerebras_api_key); + merge!(zai_api_key); merge!(exa_api_key); + merge!(firecrawl_api_key); merge!(voyage_api_key); } +fn has_env_value(keys: &[&str]) -> bool { + keys.iter().any(|key| { + env::var(key) + .ok() + .map(|value| !value.trim().is_empty()) + .unwrap_or(false) + }) +} + +fn apply_settings_to_config(cfg: &mut AgentConfig, settings: &PersistentSettings) { + if !has_env_value(&["OPENPLANTER_REASONING_EFFORT"]) { + if let Some(reasoning_effort) = settings.default_reasoning_effort.clone() { + cfg.reasoning_effort = Some(reasoning_effort); + } + } + + if !has_env_value(&["OPENPLANTER_ZAI_PLAN"]) { + if let Some(plan) = settings.zai_plan.as_deref() { + cfg.zai_plan = normalize_zai_plan(Some(plan)); + } + } + + if !has_env_value(&["OPENPLANTER_ZAI_BASE_URL"]) { + cfg.zai_base_url = resolve_zai_base_url( + &cfg.zai_plan, + &cfg.zai_paygo_base_url, + &cfg.zai_coding_base_url, + ); + } + + if !has_env_value(&["OPENPLANTER_WEB_SEARCH_PROVIDER"]) { + if let Some(provider) = settings.web_search_provider.as_deref() { + cfg.web_search_provider = normalize_web_search_provider(Some(provider)); + } + } + + if !has_env_value(&["OPENPLANTER_MODEL"]) { + let saved_model = if cfg.provider == "auto" { + settings.default_model.as_deref() + } else { + settings + .default_model_for_provider(cfg.provider.as_str()) + .or(settings.default_model.as_deref()) + }; + if let Some(model) = saved_model { + cfg.model = model.to_string(); + } + } +} + /// Application state shared across Tauri commands. pub struct AppState { pub config: Arc>, @@ -52,6 +112,9 @@ impl AppState { merge_credentials_into_config(&mut cfg, &env_creds, &empty); } + let settings = SettingsStore::new(&cfg.workspace, &cfg.session_root_dir).load(); + apply_settings_to_config(&mut cfg, &settings); + Self { config: Arc::new(Mutex::new(cfg)), session_id: Arc::new(Mutex::new(None)), @@ -63,6 +126,7 @@ impl AppState { #[cfg(test)] mod tests { use super::*; + use std::env; fn empty_cfg() -> AgentConfig { let mut cfg = AgentConfig::from_env("/nonexistent"); @@ -70,7 +134,9 @@ mod tests { cfg.anthropic_api_key = None; cfg.openrouter_api_key = None; cfg.cerebras_api_key = None; + cfg.zai_api_key = None; cfg.exa_api_key = None; + cfg.firecrawl_api_key = None; cfg.voyage_api_key = None; cfg } @@ -126,4 +192,59 @@ mod tests { merge_credentials_into_config(&mut cfg, &env_creds, &file_creds); assert_eq!(cfg.cerebras_api_key, Some("file-cer".to_string())); } + + #[test] + fn test_merge_includes_zai_and_firecrawl() { + let mut cfg = empty_cfg(); + let env_creds = CredentialBundle { + zai_api_key: Some("zai-env".to_string()), + firecrawl_api_key: Some("fc-env".to_string()), + ..Default::default() + }; + merge_credentials_into_config(&mut cfg, &env_creds, &CredentialBundle::default()); + assert_eq!(cfg.zai_api_key, Some("zai-env".to_string())); + assert_eq!(cfg.firecrawl_api_key, Some("fc-env".to_string())); + } + + #[test] + fn test_apply_settings_to_config_sets_model_and_web_search() { + let keys = [ + "OPENPLANTER_MODEL", + "OPENPLANTER_REASONING_EFFORT", + "OPENPLANTER_ZAI_PLAN", + "OPENPLANTER_ZAI_BASE_URL", + "OPENPLANTER_WEB_SEARCH_PROVIDER", + ]; + let saved: Vec<_> = keys.iter().map(|key| (*key, env::var(key).ok())).collect(); + unsafe { + for key in &keys { + env::remove_var(key); + } + } + + let mut cfg = empty_cfg(); + cfg.provider = "zai".to_string(); + let settings = PersistentSettings { + default_model_zai: Some("glm-5".to_string()), + default_reasoning_effort: Some("medium".to_string()), + zai_plan: Some("coding".to_string()), + web_search_provider: Some("firecrawl".to_string()), + ..Default::default() + }; + apply_settings_to_config(&mut cfg, &settings); + assert_eq!(cfg.model, "glm-5"); + assert_eq!(cfg.reasoning_effort, Some("medium".to_string())); + assert_eq!(cfg.zai_plan, "coding"); + assert_eq!(cfg.zai_base_url, op_core::config::ZAI_CODING_BASE_URL); + assert_eq!(cfg.web_search_provider, "firecrawl"); + + for (key, value) in saved { + unsafe { + match value { + Some(value) => env::set_var(key, value), + None => env::remove_var(key), + } + } + } + } } diff --git a/openplanter-desktop/crates/op-tauri/tauri.conf.json b/openplanter-desktop/crates/op-tauri/tauri.conf.json index d5d88bda..3d46600b 100644 --- a/openplanter-desktop/crates/op-tauri/tauri.conf.json +++ b/openplanter-desktop/crates/op-tauri/tauri.conf.json @@ -5,7 +5,7 @@ "build": { "frontendDist": "../../frontend/dist", "devUrl": "http://localhost:5173", - "beforeDevCommand": "", + "beforeDevCommand": "npm run dev", "beforeBuildCommand": "npm run build" }, "app": { diff --git a/openplanter-desktop/frontend/package-lock.json b/openplanter-desktop/frontend/package-lock.json index 841a0662..387190b3 100644 --- a/openplanter-desktop/frontend/package-lock.json +++ b/openplanter-desktop/frontend/package-lock.json @@ -1092,6 +1092,7 @@ "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.33.1.tgz", "integrity": "sha512-iJc4TwyANnOGR1OmWhsS9ayRS3s+XQ185FmuHObThD+5AeJCakAAbWv8KimMTt08xCCLNgneQwFp+JRJOr9qGQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10" } @@ -1397,6 +1398,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -1642,6 +1644,7 @@ "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", diff --git a/openplanter-desktop/frontend/src/api/invoke.test.ts b/openplanter-desktop/frontend/src/api/invoke.test.ts index 7a8318d2..00d2da0a 100644 --- a/openplanter-desktop/frontend/src/api/invoke.test.ts +++ b/openplanter-desktop/frontend/src/api/invoke.test.ts @@ -46,54 +46,74 @@ describe("invoke wrappers", () => { it("getConfig returns config", async () => { __setHandler("get_config", () => ({ provider: "anthropic", - model: "claude-opus-4-6", + model: "anthropic-foundry/claude-opus-4-6", + zai_plan: "paygo", workspace: ".", session_id: null, recursive: true, max_depth: 4, max_steps_per_call: 100, reasoning_effort: "high", + web_search_provider: "exa", demo: false, })); const config = await getConfig(); expect(config.provider).toBe("anthropic"); - expect(config.model).toBe("claude-opus-4-6"); + expect(config.model).toBe("anthropic-foundry/claude-opus-4-6"); + expect(config.zai_plan).toBe("paygo"); + expect(config.web_search_provider).toBe("exa"); }); it("updateConfig sends partial and returns config", async () => { __setHandler("update_config", ({ partial }: any) => { - expect(partial.model).toBe("gpt-5.2"); + expect(partial.model).toBe("azure-foundry/gpt-5.3-codex"); return { provider: "openai", - model: "gpt-5.2", + model: "azure-foundry/gpt-5.3-codex", + zai_plan: "coding", workspace: ".", session_id: null, recursive: true, max_depth: 4, max_steps_per_call: 100, reasoning_effort: null, + web_search_provider: "firecrawl", demo: false, }; }); - const config = await updateConfig({ model: "gpt-5.2" }); - expect(config.model).toBe("gpt-5.2"); + const config = await updateConfig({ model: "azure-foundry/gpt-5.3-codex" }); + expect(config.model).toBe("azure-foundry/gpt-5.3-codex"); + expect(config.zai_plan).toBe("coding"); + expect(config.web_search_provider).toBe("firecrawl"); }); it("listModels sends provider filter", async () => { __setHandler("list_models", ({ provider }: any) => { expect(provider).toBe("openai"); - return [{ id: "gpt-5.2", name: "GPT-5.2", provider: "openai" }]; + return [ + { + id: "azure-foundry/gpt-5.3-codex", + name: "GPT-5.3 Codex (Foundry)", + provider: "openai", + }, + ]; }); const models = await listModels("openai"); expect(models).toHaveLength(1); - expect(models[0].id).toBe("gpt-5.2"); + expect(models[0].id).toBe("azure-foundry/gpt-5.3-codex"); }); it("saveSettings sends settings object", async () => { __setHandler("save_settings", ({ settings }: any) => { - expect(settings.model).toBe("claude-opus-4-6"); + expect(settings.default_model_zai).toBe("glm-5"); + expect(settings.zai_plan).toBe("coding"); + expect(settings.web_search_provider).toBe("firecrawl"); + }); + await saveSettings({ + default_model_zai: "glm-5", + zai_plan: "coding", + web_search_provider: "firecrawl", }); - await saveSettings({ model: "claude-opus-4-6" } as any); }); it("getCredentialsStatus returns provider map", async () => { @@ -102,12 +122,16 @@ describe("invoke wrappers", () => { anthropic: true, openrouter: false, cerebras: false, + zai: true, ollama: true, exa: false, + firecrawl: true, })); const status = await getCredentialsStatus(); expect(status.openai).toBe(true); expect(status.openrouter).toBe(false); + expect(status.zai).toBe(true); + expect(status.firecrawl).toBe(true); }); it("listSessions sends limit", async () => { diff --git a/openplanter-desktop/frontend/src/api/types.ts b/openplanter-desktop/frontend/src/api/types.ts index a47e1fb4..9bc29eb7 100644 --- a/openplanter-desktop/frontend/src/api/types.ts +++ b/openplanter-desktop/frontend/src/api/types.ts @@ -65,6 +65,8 @@ export interface ConfigView { provider: string; model: string; reasoning_effort: string | null; + zai_plan: string; + web_search_provider: string; workspace: string; session_id: string | null; recursive: boolean; @@ -77,6 +79,8 @@ export interface PartialConfig { provider?: string; model?: string; reasoning_effort?: string; + zai_plan?: string; + web_search_provider?: string; } export interface ModelInfo { @@ -99,7 +103,10 @@ export interface PersistentSettings { default_model_anthropic?: string | null; default_model_openrouter?: string | null; default_model_cerebras?: string | null; + default_model_zai?: string | null; default_model_ollama?: string | null; + zai_plan?: string | null; + web_search_provider?: string | null; } export interface SlashResult { diff --git a/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts b/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts index cd1e5a1d..ef51eed2 100644 --- a/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts +++ b/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts @@ -23,6 +23,8 @@ describe("completionRegistry", () => { expect(values).toContain("/exit"); expect(values).toContain("/status"); expect(values).toContain("/model"); + expect(values).toContain("/zai-plan"); + expect(values).toContain("/web-search"); expect(values).toContain("/reasoning"); }); @@ -60,6 +62,7 @@ describe("completionRegistry", () => { expect(providerValues).toContain("openai"); expect(providerValues).toContain("anthropic"); expect(providerValues).toContain("ollama"); + expect(providerValues).toContain("zai"); }); it("model alias children have --save flag", () => { @@ -78,6 +81,26 @@ describe("completionRegistry", () => { expect(childValues).toEqual(["low", "medium", "high", "off"]); }); + it("/web-search has exa and firecrawl children", () => { + const webSearchCmd = COMMAND_COMPLETIONS.find((c) => c.value === "/web-search"); + expect(webSearchCmd).toBeDefined(); + expect(webSearchCmd!.children).toBeDefined(); + + const childValues = webSearchCmd!.children!.map((c) => c.value); + expect(childValues).toEqual(["exa", "firecrawl"]); + expect(webSearchCmd!.children![0].children?.[0].value).toBe("--save"); + }); + + it("/zai-plan has paygo and coding children", () => { + const zaiPlanCmd = COMMAND_COMPLETIONS.find((c) => c.value === "/zai-plan"); + expect(zaiPlanCmd).toBeDefined(); + expect(zaiPlanCmd!.children).toBeDefined(); + + const childValues = zaiPlanCmd!.children!.map((c) => c.value); + expect(childValues).toEqual(["paygo", "coding"]); + expect(zaiPlanCmd!.children![0].children?.[0].value).toBe("--save"); + }); + it("reasoning level children have --save flag", () => { const reasoningCmd = COMMAND_COMPLETIONS.find((c) => c.value === "/reasoning")!; for (const level of reasoningCmd.children!) { diff --git a/openplanter-desktop/frontend/src/commands/completionRegistry.ts b/openplanter-desktop/frontend/src/commands/completionRegistry.ts index 6318f4c1..2bb2b166 100644 --- a/openplanter-desktop/frontend/src/commands/completionRegistry.ts +++ b/openplanter-desktop/frontend/src/commands/completionRegistry.ts @@ -13,6 +13,7 @@ const PROVIDER_FILTERS: CompletionItem[] = [ { value: "anthropic", description: "Anthropic models" }, { value: "ollama", description: "Local Ollama models" }, { value: "cerebras", description: "Cerebras models" }, + { value: "zai", description: "Z.AI models" }, { value: "openrouter", description: "OpenRouter models" }, ]; @@ -35,6 +36,16 @@ const REASONING_LEVELS: CompletionItem[] = [ { value: "off", description: "Disable reasoning", children: SAVE_FLAG }, ]; +const WEB_SEARCH_PROVIDERS: CompletionItem[] = [ + { value: "exa", description: "Use Exa for web search", children: SAVE_FLAG }, + { value: "firecrawl", description: "Use Firecrawl for web search", children: SAVE_FLAG }, +]; + +const ZAI_PLANS: CompletionItem[] = [ + { value: "paygo", description: "Use the Z.AI PAYGO endpoint", children: SAVE_FLAG }, + { value: "coding", description: "Use the Z.AI Coding Plan endpoint", children: SAVE_FLAG }, +]; + export const COMMAND_COMPLETIONS: CompletionItem[] = [ { value: "/help", description: "Show available commands" }, { value: "/new", description: "Start a new session" }, @@ -50,6 +61,16 @@ export const COMMAND_COMPLETIONS: CompletionItem[] = [ ...MODEL_ALIAS_ITEMS, ], }, + { + value: "/zai-plan", + description: "Show or switch the Z.AI endpoint family", + children: ZAI_PLANS, + }, + { + value: "/web-search", + description: "Show or switch the web search provider", + children: WEB_SEARCH_PROVIDERS, + }, { value: "/reasoning", description: "Set reasoning effort", diff --git a/openplanter-desktop/frontend/src/commands/model.test.ts b/openplanter-desktop/frontend/src/commands/model.test.ts index 1e8bc2bd..f98dfb86 100644 --- a/openplanter-desktop/frontend/src/commands/model.test.ts +++ b/openplanter-desktop/frontend/src/commands/model.test.ts @@ -12,10 +12,12 @@ import { appState } from "../state/store"; describe("inferProvider", () => { it("claude returns anthropic", () => { expect(inferProvider("claude-opus-4-6")).toBe("anthropic"); + expect(inferProvider("anthropic-foundry/claude-opus-4-6")).toBe("anthropic"); }); it("gpt returns openai", () => { expect(inferProvider("gpt-5.2")).toBe("openai"); + expect(inferProvider("azure-foundry/gpt-5.3-codex")).toBe("openai"); }); it("o1 returns openai", () => { @@ -34,6 +36,11 @@ describe("inferProvider", () => { expect(inferProvider("qwen-3-235b-a22b-instruct-2507")).toBe("cerebras"); }); + it("glm returns zai", () => { + expect(inferProvider("glm-5")).toBe("zai"); + expect(inferProvider("zai-glm-4.6")).toBe("zai"); + }); + it("qwen without 3 returns ollama", () => { expect(inferProvider("qwen2")).toBe("ollama"); }); @@ -52,11 +59,15 @@ describe("MODEL_ALIASES", () => { }); it("opus alias", () => { - expect(MODEL_ALIASES["opus"]).toBe("claude-opus-4-6"); + expect(MODEL_ALIASES["opus"]).toBe("anthropic-foundry/claude-opus-4-6"); }); it("gpt5 alias", () => { - expect(MODEL_ALIASES["gpt5"]).toBe("gpt-5.2"); + expect(MODEL_ALIASES["gpt5"]).toBe("azure-foundry/gpt-5.3-codex"); + }); + + it("zai alias", () => { + expect(MODEL_ALIASES["zai"]).toBe("glm-5"); }); }); @@ -68,6 +79,7 @@ describe("handleModelCommand", () => { ...originalState, provider: "anthropic", model: "claude-opus-4-6", + webSearchProvider: "exa", }); }); @@ -95,4 +107,34 @@ describe("handleModelCommand", () => { expect(result.action).toBe("handled"); expect(result.lines.some((l) => l.includes("gpt-5.2"))).toBe(true); }); + + it("save persists provider-specific model default", async () => { + __setHandler("update_config", ({ partial }: { partial: Record }) => { + expect(partial.model).toBe("glm-5"); + expect(partial.provider).toBe("zai"); + return { + provider: "zai", + model: "glm-5", + zai_plan: "coding", + workspace: ".", + session_id: null, + recursive: true, + max_depth: 4, + max_steps_per_call: 100, + reasoning_effort: "high", + web_search_provider: "exa", + demo: false, + }; + }); + __setHandler("save_settings", ({ settings }: { settings: Record }) => { + expect(settings.default_model).toBe("glm-5"); + expect(settings.default_model_zai).toBe("glm-5"); + }); + + const result = await handleModelCommand("zai --save"); + expect(result.lines).toContain("(Settings saved)"); + expect(appState.get().provider).toBe("zai"); + expect(appState.get().model).toBe("glm-5"); + expect(appState.get().zaiPlan).toBe("coding"); + }); }); diff --git a/openplanter-desktop/frontend/src/commands/model.ts b/openplanter-desktop/frontend/src/commands/model.ts index 0fa9e9c2..45f7016d 100644 --- a/openplanter-desktop/frontend/src/commands/model.ts +++ b/openplanter-desktop/frontend/src/commands/model.ts @@ -1,22 +1,30 @@ /** /model slash command handler. */ -import { updateConfig, listModels } from "../api/invoke"; +import { listModels, saveSettings, updateConfig } from "../api/invoke"; +import type { PersistentSettings } from "../api/types"; import { appState } from "../state/store"; /** Aliases mapping short names to full model identifiers. */ export const MODEL_ALIASES: Record = { - opus: "claude-opus-4-6", - sonnet: "claude-sonnet-4-5", - haiku: "claude-haiku-4-5", - "sonnet-4": "claude-sonnet-4-5", - "haiku-4": "claude-haiku-4-5", - "opus-4": "claude-opus-4-6", - gpt5: "gpt-5.2", - "gpt-5": "gpt-5.2", + opus: "anthropic-foundry/claude-opus-4-6", + sonnet: "anthropic-foundry/claude-sonnet-4-6", + haiku: "anthropic-foundry/claude-haiku-4-5", + "sonnet-4": "anthropic-foundry/claude-sonnet-4-6", + "haiku-4": "anthropic-foundry/claude-haiku-4-5", + "opus-4": "anthropic-foundry/claude-opus-4-6", + gpt5: "azure-foundry/gpt-5.3-codex", + "gpt-5": "azure-foundry/gpt-5.3-codex", + "gpt-5.3": "azure-foundry/gpt-5.3-codex", + kimi: "azure-foundry/Kimi-K2.5", gpt4o: "gpt-4o", "gpt-4o": "gpt-4o", - "o1": "o1", - "o3": "o3", + o1: "o1", + o3: "o3", "o4-mini": "o4-mini", + glm: "glm-5", + glm5: "glm-5", + "glm-5": "glm-5", + zai: "glm-5", + "zai-glm": "zai-glm-4.6", llama: "llama3.2", mistral: "mistral", gemma: "gemma", @@ -28,14 +36,40 @@ export const MODEL_ALIASES: Record = { /** Infer provider from a model name, matching builder.rs patterns. */ export function inferProvider(model: string): string | null { + if (/^anthropic-foundry\//i.test(model)) return "anthropic"; + if (/^azure-foundry\//i.test(model)) return "openai"; if (model.includes("/")) return "openrouter"; if (/^claude/i.test(model)) return "anthropic"; - if (/^(llama.*cerebras|qwen-3|gpt-oss|zai-glm)/i.test(model)) return "cerebras"; + if (/^(llama.*cerebras|qwen-3|gpt-oss)/i.test(model)) return "cerebras"; + if (/^(glm|zai-glm)/i.test(model)) return "zai"; if (/^(gpt|o[1-4]-|o[1-4]$|chatgpt|dall-e|tts-|whisper)/i.test(model)) return "openai"; - if (/^(llama|mistral|gemma|phi|codellama|deepseek|vicuna|tinyllama|neural-chat|dolphin|wizardlm|orca|nous-hermes|command-r|qwen)/i.test(model)) return "ollama"; + if (/^(llama|mistral|gemma|phi|codellama|deepseek|vicuna|tinyllama|neural-chat|dolphin|wizardlm|orca|nous-hermes|command-r|qwen(?!-3))/i.test(model)) return "ollama"; return null; } +function buildProviderDefaultModelSettings( + provider: string, + model: string, +): PersistentSettings { + const base: PersistentSettings = { default_model: model }; + switch (provider) { + case "openai": + return { ...base, default_model_openai: model }; + case "anthropic": + return { ...base, default_model_anthropic: model }; + case "openrouter": + return { ...base, default_model_openrouter: model }; + case "cerebras": + return { ...base, default_model_cerebras: model }; + case "zai": + return { ...base, default_model_zai: model }; + case "ollama": + return { ...base, default_model_ollama: model }; + default: + return base; + } +} + export interface CommandResult { action: "handled" | "clear" | "quit"; lines: string[]; @@ -43,10 +77,9 @@ export interface CommandResult { /** Handle /model [args]. */ export async function handleModelCommand(args: string): Promise { - const parts = args.trim().split(/\s+/); + const parts = args.trim().split(/\s+/).filter(Boolean); const subcommand = parts[0] || ""; - // /model (no args) — show current info if (!subcommand) { const s = appState.get(); const aliasEntries = Object.entries(MODEL_ALIASES) @@ -57,6 +90,7 @@ export async function handleModelCommand(args: string): Promise { lines: [ `Provider: ${s.provider}`, `Model: ${s.model}`, + `Z.AI plan: ${s.zaiPlan || "paygo"}`, "", "Aliases:", aliasEntries, @@ -64,7 +98,6 @@ export async function handleModelCommand(args: string): Promise { }; } - // /model list [all|] if (subcommand === "list") { const filter = parts[1] || "all"; try { @@ -76,7 +109,7 @@ export async function handleModelCommand(args: string): Promise { }; } const lines = models.map( - (m) => ` ${m.id}${m.name ? ` (${m.name})` : ""} [${m.provider}]` + (m) => ` ${m.id}${m.name ? ` (${m.name})` : ""} [${m.provider}]`, ); return { action: "handled", @@ -90,36 +123,38 @@ export async function handleModelCommand(args: string): Promise { } } - // /model [--save] const modelName = subcommand; const save = parts.includes("--save"); - - // Resolve alias const resolved = MODEL_ALIASES[modelName.toLowerCase()] ?? modelName; const provider = inferProvider(resolved); if (!provider) { return { action: "handled", - lines: [`Cannot infer provider for "${resolved}". Specify full model name or use a known alias.`], + lines: [ + `Cannot infer provider for "${resolved}". Specify full model name or use a known alias.`, + ], }; } try { const config = await updateConfig({ model: resolved, - provider: provider, + provider, }); appState.update((s) => ({ ...s, provider: config.provider, model: config.model, + zaiPlan: config.zai_plan, })); const lines = [`Switched to ${config.provider}/${config.model}`]; if (save) { - // save_settings would be called here when backend supports it + await saveSettings( + buildProviderDefaultModelSettings(config.provider, config.model), + ); lines.push("(Settings saved)"); } diff --git a/openplanter-desktop/frontend/src/commands/reasoning.test.ts b/openplanter-desktop/frontend/src/commands/reasoning.test.ts index cfd5743f..6df705f1 100644 --- a/openplanter-desktop/frontend/src/commands/reasoning.test.ts +++ b/openplanter-desktop/frontend/src/commands/reasoning.test.ts @@ -38,6 +38,7 @@ describe("handleReasoningCommand", () => { return { provider: "anthropic", model: "claude-opus-4-6", + zai_plan: "paygo", reasoning_effort: "low", workspace: ".", session_id: null, @@ -57,6 +58,7 @@ describe("handleReasoningCommand", () => { __setHandler("update_config", ({ partial }: any) => ({ provider: "anthropic", model: "claude-opus-4-6", + zai_plan: "coding", reasoning_effort: "high", workspace: ".", session_id: null, @@ -77,6 +79,7 @@ describe("handleReasoningCommand", () => { return { provider: "anthropic", model: "claude-opus-4-6", + zai_plan: "paygo", reasoning_effort: null, workspace: ".", session_id: null, @@ -103,6 +106,7 @@ describe("handleReasoningCommand", () => { return { provider: "anthropic", model: "claude-opus-4-6", + zai_plan: "coding", reasoning_effort: "high", workspace: ".", session_id: null, @@ -122,6 +126,7 @@ describe("handleReasoningCommand", () => { __setHandler("update_config", ({ partial }: any) => ({ provider: "anthropic", model: "claude-opus-4-6", + zai_plan: "coding", reasoning_effort: "high", workspace: ".", session_id: null, @@ -130,6 +135,9 @@ describe("handleReasoningCommand", () => { max_steps_per_call: 100, demo: false, })); + __setHandler("save_settings", ({ settings }: any) => { + expect(settings.default_reasoning_effort).toBe("high"); + }); const result = await handleReasoningCommand("high --save"); expect(result.action).toBe("handled"); diff --git a/openplanter-desktop/frontend/src/commands/reasoning.ts b/openplanter-desktop/frontend/src/commands/reasoning.ts index 1e2c1ad6..22b568f9 100644 --- a/openplanter-desktop/frontend/src/commands/reasoning.ts +++ b/openplanter-desktop/frontend/src/commands/reasoning.ts @@ -1,5 +1,5 @@ /** /reasoning slash command handler. */ -import { updateConfig } from "../api/invoke"; +import { saveSettings, updateConfig } from "../api/invoke"; import { appState } from "../state/store"; import type { CommandResult } from "./model"; @@ -41,10 +41,14 @@ export async function handleReasoningCommand(args: string): Promise ({ ...s, reasoningEffort: config.reasoning_effort, + zaiPlan: config.zai_plan, })); const lines = [`Reasoning effort set to: ${config.reasoning_effort ?? "off"}`]; if (save) { + await saveSettings({ + default_reasoning_effort: config.reasoning_effort, + }); lines.push("(Settings saved)"); } diff --git a/openplanter-desktop/frontend/src/commands/slash.test.ts b/openplanter-desktop/frontend/src/commands/slash.test.ts index 495d0974..e95062f4 100644 --- a/openplanter-desktop/frontend/src/commands/slash.test.ts +++ b/openplanter-desktop/frontend/src/commands/slash.test.ts @@ -17,6 +17,8 @@ describe("dispatchSlashCommand", () => { ...originalState, provider: "anthropic", model: "claude-opus-4-6", + zaiPlan: "paygo", + webSearchProvider: "exa", sessionId: "20260101-120000-deadbeef", reasoningEffort: "medium", }); @@ -70,6 +72,18 @@ describe("dispatchSlashCommand", () => { expect(result!.lines.some((l) => l.includes("Session:"))).toBe(true); }); + it("status shows web search provider", async () => { + const result = await dispatchSlashCommand("/status"); + expect(result).not.toBeNull(); + expect(result!.lines.some((l) => l.includes("Web search:"))).toBe(true); + }); + + it("status shows zai plan", async () => { + const result = await dispatchSlashCommand("/status"); + expect(result).not.toBeNull(); + expect(result!.lines.some((l) => l.includes("Z.AI plan:"))).toBe(true); + }); + it("unknown command", async () => { const result = await dispatchSlashCommand("/foobar"); expect(result).not.toBeNull(); @@ -112,6 +126,20 @@ describe("dispatchSlashCommand", () => { ).toBe(true); }); + it("web search dispatches", async () => { + const result = await dispatchSlashCommand("/web-search"); + expect(result).not.toBeNull(); + expect(result!.action).toBe("handled"); + expect(result!.lines.some((l) => l.includes("Web search provider:"))).toBe(true); + }); + + it("zai plan dispatches", async () => { + const result = await dispatchSlashCommand("/zai-plan"); + expect(result).not.toBeNull(); + expect(result!.action).toBe("handled"); + expect(result!.lines.some((l) => l.includes("Z.AI plan:"))).toBe(true); + }); + it("new creates session", async () => { __setHandler( "open_session", diff --git a/openplanter-desktop/frontend/src/commands/slash.ts b/openplanter-desktop/frontend/src/commands/slash.ts index 1da7cb66..d46675dd 100644 --- a/openplanter-desktop/frontend/src/commands/slash.ts +++ b/openplanter-desktop/frontend/src/commands/slash.ts @@ -3,6 +3,8 @@ import { appState } from "../state/store"; import { openSession } from "../api/invoke"; import { handleModelCommand, type CommandResult } from "./model"; import { handleReasoningCommand } from "./reasoning"; +import { handleWebSearchCommand } from "./webSearch"; +import { handleZaiPlanCommand } from "./zaiPlan"; /** Dispatch a slash command. Returns null if not a slash command. */ export async function dispatchSlashCommand(input: string): Promise { @@ -28,6 +30,12 @@ export async function dispatchSlashCommand(input: string): Promise Switch model (auto-detects provider)", " /model --save Switch and persist", " /model list [provider] List available models", + " /zai-plan Show current Z.AI endpoint family", + " /zai-plan Set Z.AI endpoint family (paygo, coding)", + " /zai-plan --save Set and persist", + " /web-search Show current web search provider", + " /web-search Set web search provider (exa, firecrawl)", + " /web-search --save Set and persist", " /reasoning Show/set reasoning effort", " /reasoning Set level (low, medium, high, off)", ], @@ -75,6 +83,8 @@ export async function dispatchSlashCommand(input: string): Promise { + const mock = await import("../__mocks__/tauri"); + return { invoke: mock.invoke }; +}); + +import { appState } from "../state/store"; +import { handleWebSearchCommand } from "./webSearch"; + +describe("handleWebSearchCommand", () => { + const originalState = appState.get(); + + beforeEach(() => { + appState.set({ + ...originalState, + webSearchProvider: "exa", + }); + }); + + afterEach(() => { + __clearHandlers(); + appState.set(originalState); + }); + + it("no args shows current provider", async () => { + const result = await handleWebSearchCommand(""); + expect(result.lines).toContain("Web search provider: exa"); + }); + + it("switches provider for the current session", async () => { + __setHandler("update_config", ({ partial }: { partial: Record }) => { + expect(partial.web_search_provider).toBe("firecrawl"); + return { + provider: "anthropic", + model: "claude-opus-4-6", + zai_plan: "paygo", + workspace: ".", + session_id: null, + recursive: true, + max_depth: 4, + max_steps_per_call: 100, + reasoning_effort: "high", + web_search_provider: "firecrawl", + demo: false, + }; + }); + + const result = await handleWebSearchCommand("firecrawl"); + expect(result.lines).toContain("Web search provider set to: firecrawl"); + expect(appState.get().webSearchProvider).toBe("firecrawl"); + }); + + it("save persists the selected provider", async () => { + __setHandler("update_config", () => ({ + provider: "anthropic", + model: "claude-opus-4-6", + zai_plan: "coding", + workspace: ".", + session_id: null, + recursive: true, + max_depth: 4, + max_steps_per_call: 100, + reasoning_effort: "high", + web_search_provider: "firecrawl", + demo: false, + })); + __setHandler("save_settings", ({ settings }: { settings: Record }) => { + expect(settings.web_search_provider).toBe("firecrawl"); + }); + + const result = await handleWebSearchCommand("firecrawl --save"); + expect(result.lines).toContain("(Settings saved)"); + }); +}); diff --git a/openplanter-desktop/frontend/src/commands/webSearch.ts b/openplanter-desktop/frontend/src/commands/webSearch.ts new file mode 100644 index 00000000..5a475eb0 --- /dev/null +++ b/openplanter-desktop/frontend/src/commands/webSearch.ts @@ -0,0 +1,58 @@ +/** /web-search slash command handler. */ +import { saveSettings, updateConfig } from "../api/invoke"; +import { appState } from "../state/store"; +import type { CommandResult } from "./model"; + +const VALID_WEB_SEARCH_PROVIDERS = ["exa", "firecrawl"]; + +/** Handle /web-search [provider] [--save]. */ +export async function handleWebSearchCommand(args: string): Promise { + const parts = args.trim().split(/\s+/).filter(Boolean); + const requestedProvider = parts[0]?.toLowerCase() ?? ""; + const save = parts.includes("--save"); + + if (!requestedProvider) { + const current = appState.get().webSearchProvider || "exa"; + return { + action: "handled", + lines: [ + `Web search provider: ${current}`, + `Valid providers: ${VALID_WEB_SEARCH_PROVIDERS.join(", ")}`, + ], + }; + } + + if (!VALID_WEB_SEARCH_PROVIDERS.includes(requestedProvider)) { + return { + action: "handled", + lines: [ + `Invalid web search provider "${requestedProvider}". Expected: ${VALID_WEB_SEARCH_PROVIDERS.join(", ")}`, + ], + }; + } + + try { + const config = await updateConfig({ + web_search_provider: requestedProvider, + }); + + appState.update((s) => ({ + ...s, + zaiPlan: config.zai_plan, + webSearchProvider: config.web_search_provider, + })); + + const lines = [`Web search provider set to: ${config.web_search_provider}`]; + if (save) { + await saveSettings({ web_search_provider: config.web_search_provider }); + lines.push("(Settings saved)"); + } + + return { action: "handled", lines }; + } catch (e) { + return { + action: "handled", + lines: [`Failed to set web search provider: ${e}`], + }; + } +} diff --git a/openplanter-desktop/frontend/src/commands/zaiPlan.test.ts b/openplanter-desktop/frontend/src/commands/zaiPlan.test.ts new file mode 100644 index 00000000..2021900f --- /dev/null +++ b/openplanter-desktop/frontend/src/commands/zaiPlan.test.ts @@ -0,0 +1,79 @@ +import { vi, describe, it, expect, beforeEach, afterEach } from "vitest"; +import { __setHandler, __clearHandlers } from "../__mocks__/tauri"; + +vi.mock("@tauri-apps/api/core", async () => { + const mock = await import("../__mocks__/tauri"); + return { invoke: mock.invoke }; +}); + +import { appState } from "../state/store"; +import { handleZaiPlanCommand } from "./zaiPlan"; + +describe("handleZaiPlanCommand", () => { + const originalState = appState.get(); + + beforeEach(() => { + appState.set({ + ...originalState, + provider: "zai", + model: "glm-5", + zaiPlan: "paygo", + }); + }); + + afterEach(() => { + __clearHandlers(); + appState.set(originalState); + }); + + it("no args shows current plan", async () => { + const result = await handleZaiPlanCommand(""); + expect(result.lines).toContain("Z.AI plan: paygo"); + }); + + it("switches plan for the current session", async () => { + __setHandler("update_config", ({ partial }: { partial: Record }) => { + expect(partial.zai_plan).toBe("coding"); + return { + provider: "zai", + model: "glm-5", + zai_plan: "coding", + workspace: ".", + session_id: null, + recursive: true, + max_depth: 4, + max_steps_per_call: 100, + reasoning_effort: "high", + web_search_provider: "firecrawl", + demo: false, + }; + }); + + const result = await handleZaiPlanCommand("coding"); + expect(result.lines).toContain("Z.AI plan set to: coding"); + expect(result.lines).toContain("Endpoint family: https://api.z.ai/api/coding/paas/v4"); + expect(appState.get().zaiPlan).toBe("coding"); + }); + + it("save persists the selected plan", async () => { + __setHandler("update_config", () => ({ + provider: "zai", + model: "glm-5", + zai_plan: "paygo", + workspace: ".", + session_id: null, + recursive: true, + max_depth: 4, + max_steps_per_call: 100, + reasoning_effort: "high", + web_search_provider: "firecrawl", + demo: false, + })); + __setHandler("save_settings", ({ settings }: { settings: Record }) => { + expect(settings.zai_plan).toBe("paygo"); + }); + + const result = await handleZaiPlanCommand("paygo --save"); + expect(result.lines).toContain("(Settings saved)"); + }); +}); diff --git a/openplanter-desktop/frontend/src/commands/zaiPlan.ts b/openplanter-desktop/frontend/src/commands/zaiPlan.ts new file mode 100644 index 00000000..c29006c6 --- /dev/null +++ b/openplanter-desktop/frontend/src/commands/zaiPlan.ts @@ -0,0 +1,62 @@ +/** /zai-plan slash command handler. */ +import { saveSettings, updateConfig } from "../api/invoke"; +import { appState } from "../state/store"; +import type { CommandResult } from "./model"; + +const VALID_ZAI_PLANS = ["paygo", "coding"]; + +/** Handle /zai-plan [plan] [--save]. */ +export async function handleZaiPlanCommand(args: string): Promise { + const parts = args.trim().split(/\s+/).filter(Boolean); + const requestedPlan = parts[0]?.toLowerCase() ?? ""; + const save = parts.includes("--save"); + + if (!requestedPlan) { + const current = appState.get().zaiPlan || "paygo"; + return { + action: "handled", + lines: [ + `Z.AI plan: ${current}`, + `Valid plans: ${VALID_ZAI_PLANS.join(", ")}`, + ], + }; + } + + if (!VALID_ZAI_PLANS.includes(requestedPlan)) { + return { + action: "handled", + lines: [ + `Invalid Z.AI plan "${requestedPlan}". Expected: ${VALID_ZAI_PLANS.join(", ")}`, + ], + }; + } + + try { + const config = await updateConfig({ + zai_plan: requestedPlan, + }); + + appState.update((s) => ({ + ...s, + zaiPlan: config.zai_plan, + provider: config.provider, + model: config.model, + })); + + const lines = [ + `Z.AI plan set to: ${config.zai_plan}`, + `Endpoint family: ${config.zai_plan === "coding" ? "https://api.z.ai/api/coding/paas/v4" : "https://api.z.ai/api/paas/v4"}`, + ]; + if (save) { + await saveSettings({ zai_plan: config.zai_plan }); + lines.push("(Settings saved)"); + } + + return { action: "handled", lines }; + } catch (e) { + return { + action: "handled", + lines: [`Failed to set Z.AI plan: ${e}`], + }; + } +} diff --git a/openplanter-desktop/frontend/src/components/App.test.ts b/openplanter-desktop/frontend/src/components/App.test.ts index 5e5e399c..d641b919 100644 --- a/openplanter-desktop/frontend/src/components/App.test.ts +++ b/openplanter-desktop/frontend/src/components/App.test.ts @@ -48,7 +48,7 @@ describe("createApp", () => { __setHandler("list_sessions", () => [SESSION_B, SESSION_A]); __setHandler("get_credentials_status", () => ({ openai: true, anthropic: true, openrouter: false, - cerebras: false, ollama: true, exa: false, + cerebras: false, zai: true, ollama: true, exa: false, firecrawl: true, })); __setHandler("open_session", () => ({ id: "20260227-120000-cccc3333", @@ -79,13 +79,21 @@ describe("createApp", () => { }); it("renders settings display", () => { - appState.update((s) => ({ ...s, provider: "anthropic", model: "claude-opus-4-6" })); + appState.update((s) => ({ + ...s, + provider: "zai", + model: "glm-5", + zaiPlan: "coding", + webSearchProvider: "firecrawl", + })); const root = document.createElement("div"); createApp(root); const settings = root.querySelector(".settings-display"); expect(settings).not.toBeNull(); - expect(settings!.textContent).toContain("anthropic"); - expect(settings!.textContent).toContain("claude-opus-4-6"); + expect(settings!.textContent).toContain("zai"); + expect(settings!.textContent).toContain("glm-5"); + expect(settings!.textContent).toContain("coding"); + expect(settings!.textContent).toContain("firecrawl"); }); it("renders credential status", async () => { @@ -95,7 +103,7 @@ describe("createApp", () => { await vi.waitFor(() => { const creds = root.querySelector(".cred-status"); - expect(creds!.children.length).toBe(6); + expect(creds!.children.length).toBe(8); expect(creds!.querySelector(".cred-ok")!.textContent).toContain("openai"); expect(creds!.querySelector(".cred-missing")!.textContent).toContain("openrouter"); }); diff --git a/openplanter-desktop/frontend/src/components/App.ts b/openplanter-desktop/frontend/src/components/App.ts index 22047445..9f5ef663 100644 --- a/openplanter-desktop/frontend/src/components/App.ts +++ b/openplanter-desktop/frontend/src/components/App.ts @@ -67,6 +67,8 @@ export function createApp(root: HTMLElement): void { settingsDisplay.innerHTML = [ `
provider: ${s.provider || "auto"}
`, `
model: ${s.model || "\u2014"}
`, + `
z.ai plan: ${s.zaiPlan || "paygo"}
`, + `
web search: ${s.webSearchProvider || "exa"}
`, `
reasoning: ${s.reasoningEffort ?? "off"}
`, `
mode: ${s.recursive ? "recursive" : "flat"}
`, ].join(""); @@ -300,7 +302,7 @@ async function loadCredentials(container: HTMLElement): Promise { try { const status = await getCredentialsStatus(); container.innerHTML = ""; - const providers = ["openai", "anthropic", "openrouter", "cerebras", "ollama", "exa"]; + const providers = ["openai", "anthropic", "openrouter", "cerebras", "zai", "ollama", "exa", "firecrawl"]; for (const p of providers) { const row = document.createElement("div"); const hasKey = status[p] ?? false; diff --git a/openplanter-desktop/frontend/src/components/StatusBar.test.ts b/openplanter-desktop/frontend/src/components/StatusBar.test.ts index 507ea6cd..34aba5ca 100644 --- a/openplanter-desktop/frontend/src/components/StatusBar.test.ts +++ b/openplanter-desktop/frontend/src/components/StatusBar.test.ts @@ -24,6 +24,7 @@ describe("createStatusBar", () => { expect(bar.querySelector(".provider")).not.toBeNull(); expect(bar.querySelector(".model")).not.toBeNull(); expect(bar.querySelector(".reasoning")).not.toBeNull(); + expect(bar.querySelector(".zai-plan")).not.toBeNull(); expect(bar.querySelector(".mode")).not.toBeNull(); expect(bar.querySelector(".session")).not.toBeNull(); expect(bar.querySelector(".tokens")).not.toBeNull(); @@ -54,6 +55,18 @@ describe("createStatusBar", () => { expect(bar.querySelector(".reasoning")!.textContent).toBe(""); }); + it("renders Z.AI plan when provider is zai", () => { + appState.update((s) => ({ ...s, provider: "zai", zaiPlan: "coding" })); + const bar = createStatusBar(); + expect(bar.querySelector(".zai-plan")!.textContent).toBe("zai:coding"); + }); + + it("hides Z.AI plan when provider is not zai", () => { + appState.update((s) => ({ ...s, provider: "anthropic", zaiPlan: "coding" })); + const bar = createStatusBar(); + expect(bar.querySelector(".zai-plan")!.textContent).toBe(""); + }); + it("renders recursive mode", () => { appState.update((s) => ({ ...s, recursive: true })); const bar = createStatusBar(); diff --git a/openplanter-desktop/frontend/src/components/StatusBar.ts b/openplanter-desktop/frontend/src/components/StatusBar.ts index c45963e0..f2f119ad 100644 --- a/openplanter-desktop/frontend/src/components/StatusBar.ts +++ b/openplanter-desktop/frontend/src/components/StatusBar.ts @@ -14,6 +14,9 @@ export function createStatusBar(): HTMLElement { const reasoningEl = document.createElement("span"); reasoningEl.className = "reasoning"; + const zaiPlanEl = document.createElement("span"); + zaiPlanEl.className = "zai-plan"; + const modeEl = document.createElement("span"); modeEl.className = "mode"; @@ -26,6 +29,7 @@ export function createStatusBar(): HTMLElement { bar.appendChild(providerEl); bar.appendChild(modelEl); bar.appendChild(reasoningEl); + bar.appendChild(zaiPlanEl); bar.appendChild(modeEl); bar.appendChild(sessionEl); bar.appendChild(tokensEl); @@ -37,6 +41,8 @@ export function createStatusBar(): HTMLElement { reasoningEl.textContent = s.reasoningEffort ? `reasoning:${s.reasoningEffort}` : ""; + zaiPlanEl.textContent = + s.provider === "zai" ? `zai:${s.zaiPlan || "paygo"}` : ""; modeEl.textContent = s.recursive ? "recursive" : "flat"; sessionEl.textContent = s.sessionId ? `session ${s.sessionId.slice(0, 8)}` : ""; diff --git a/openplanter-desktop/frontend/src/main.ts b/openplanter-desktop/frontend/src/main.ts index c797da10..ad9ac303 100644 --- a/openplanter-desktop/frontend/src/main.ts +++ b/openplanter-desktop/frontend/src/main.ts @@ -35,6 +35,8 @@ async function init() { ...s, provider: config.provider, model: config.model, + zaiPlan: config.zai_plan, + webSearchProvider: config.web_search_provider, sessionId: config.session_id, reasoningEffort: config.reasoning_effort, recursive: config.recursive, @@ -66,6 +68,8 @@ async function init() { content: [ `provider: ${provider || "auto"}`, `model: ${model || "—"}`, + `z.ai plan: ${state.zaiPlan || "paygo"}`, + `web search: ${state.webSearchProvider || "exa"}`, `reasoning: ${reasoningLabel}`, `mode: ${modeLabel}`, `workspace: ${state.workspace || "."}`, diff --git a/openplanter-desktop/frontend/src/state/store.ts b/openplanter-desktop/frontend/src/state/store.ts index 1cd3d3fb..eafa4c8a 100644 --- a/openplanter-desktop/frontend/src/state/store.ts +++ b/openplanter-desktop/frontend/src/state/store.ts @@ -61,6 +61,8 @@ export interface ChatMessage { export interface AppState { provider: string; model: string; + zaiPlan: string; + webSearchProvider: string; sessionId: string | null; inputTokens: number; outputTokens: number; @@ -80,6 +82,8 @@ export interface AppState { export const appState = new Store({ provider: "", model: "", + zaiPlan: "paygo", + webSearchProvider: "exa", sessionId: null, inputTokens: 0, outputTokens: 0, diff --git a/openplanter-desktop/package.json b/openplanter-desktop/package.json new file mode 100644 index 00000000..36278c4a --- /dev/null +++ b/openplanter-desktop/package.json @@ -0,0 +1,10 @@ +{ + "name": "openplanter-desktop", + "private": true, + "scripts": { + "dev": "npm --prefix frontend run dev", + "build": "npm --prefix frontend run build", + "test": "npm --prefix frontend run test", + "test:e2e": "npm --prefix frontend run test:e2e" + } +} diff --git a/tests/test_coverage_gaps.py b/tests/test_coverage_gaps.py index 22e4dfe7..23c49947 100644 --- a/tests/test_coverage_gaps.py +++ b/tests/test_coverage_gaps.py @@ -162,16 +162,26 @@ def test_defaults_from_clean_env(self) -> None: with patch.dict(os.environ, {}, clear=True): cfg = AgentConfig.from_env("/tmp/test-ws") self.assertEqual(cfg.provider, "auto") - self.assertEqual(cfg.model, "claude-opus-4-6") + self.assertEqual(cfg.model, "anthropic-foundry/claude-opus-4-6") self.assertEqual(cfg.reasoning_effort, "high") self.assertEqual(cfg.max_depth, 4) self.assertEqual(cfg.max_steps_per_call, 100) self.assertEqual(cfg.shell, "/bin/sh") + self.assertEqual( + cfg.openai_base_url, + "https://foundry-proxy.cheetah-koi.ts.net/openai/v1", + ) + self.assertEqual( + cfg.anthropic_base_url, + "https://foundry-proxy.cheetah-koi.ts.net/anthropic/v1", + ) + self.assertEqual(cfg.openai_api_key, "dont-worry-this-key-will-be-auto-injected") + self.assertEqual(cfg.anthropic_api_key, "dont-worry-it-will-be-injected") def test_custom_env_overrides(self) -> None: env = { "OPENPLANTER_PROVIDER": "anthropic", - "OPENPLANTER_MODEL": "claude-opus-4-6", + "OPENPLANTER_MODEL": "anthropic-foundry/claude-opus-4-6", "OPENPLANTER_REASONING_EFFORT": "low", "OPENPLANTER_MAX_DEPTH": "5", "OPENPLANTER_MAX_STEPS": "20", @@ -180,12 +190,51 @@ def test_custom_env_overrides(self) -> None: with patch.dict(os.environ, env, clear=True): cfg = AgentConfig.from_env("/tmp/test-ws") self.assertEqual(cfg.provider, "anthropic") - self.assertEqual(cfg.model, "claude-opus-4-6") + self.assertEqual(cfg.model, "anthropic-foundry/claude-opus-4-6") self.assertEqual(cfg.reasoning_effort, "low") self.assertEqual(cfg.max_depth, 5) self.assertEqual(cfg.max_steps_per_call, 20) self.assertEqual(cfg.shell, "/bin/bash") + def test_rate_limit_and_zai_stream_retries_from_env(self) -> None: + env = { + "OPENPLANTER_RATE_LIMIT_MAX_RETRIES": "7", + "OPENPLANTER_RATE_LIMIT_BACKOFF_BASE_SEC": "0.5", + "OPENPLANTER_RATE_LIMIT_BACKOFF_MAX_SEC": "10.0", + "OPENPLANTER_RATE_LIMIT_RETRY_AFTER_CAP_SEC": "30.0", + "OPENPLANTER_ZAI_STREAM_MAX_RETRIES": "8", + } + with patch.dict(os.environ, env, clear=True): + cfg = AgentConfig.from_env("/tmp/test-ws") + self.assertEqual(cfg.rate_limit_max_retries, 7) + self.assertEqual(cfg.rate_limit_backoff_base_sec, 0.5) + self.assertEqual(cfg.rate_limit_backoff_max_sec, 10.0) + self.assertEqual(cfg.rate_limit_retry_after_cap_sec, 30.0) + self.assertEqual(cfg.zai_stream_max_retries, 8) + + def test_zai_plan_selects_endpoint(self) -> None: + env = { + "OPENPLANTER_ZAI_PLAN": "coding", + "OPENPLANTER_ZAI_PAYGO_BASE_URL": "https://paygo.example/v4", + "OPENPLANTER_ZAI_CODING_BASE_URL": "https://coding.example/v4", + } + with patch.dict(os.environ, env, clear=True): + cfg = AgentConfig.from_env("/tmp/test-ws") + self.assertEqual(cfg.zai_plan, "coding") + self.assertEqual(cfg.zai_base_url, "https://coding.example/v4") + + def test_zai_base_url_override_wins_over_plan(self) -> None: + env = { + "OPENPLANTER_ZAI_PLAN": "paygo", + "OPENPLANTER_ZAI_BASE_URL": "https://override.example/v4", + "OPENPLANTER_ZAI_PAYGO_BASE_URL": "https://paygo.example/v4", + "OPENPLANTER_ZAI_CODING_BASE_URL": "https://coding.example/v4", + } + with patch.dict(os.environ, env, clear=True): + cfg = AgentConfig.from_env("/tmp/test-ws") + self.assertEqual(cfg.zai_plan, "paygo") + self.assertEqual(cfg.zai_base_url, "https://override.example/v4") + def test_api_keys_from_env(self) -> None: env = { "OPENAI_API_KEY": "oa", @@ -200,6 +249,16 @@ def test_api_keys_from_env(self) -> None: self.assertEqual(cfg.openrouter_api_key, "or") self.assertEqual(cfg.exa_api_key, "exa") + def test_foundry_placeholder_keys_disabled_for_public_endpoints(self) -> None: + env = { + "OPENPLANTER_OPENAI_BASE_URL": "https://api.openai.com/v1", + "OPENPLANTER_ANTHROPIC_BASE_URL": "https://api.anthropic.com/v1", + } + with patch.dict(os.environ, env, clear=True): + cfg = AgentConfig.from_env("/tmp/test-ws") + self.assertIsNone(cfg.openai_api_key) + self.assertIsNone(cfg.anthropic_api_key) + def test_workspace_resolved(self) -> None: with patch.dict(os.environ, {}, clear=True): cfg = AgentConfig.from_env("/tmp/test-ws") @@ -274,19 +333,25 @@ def test_explicit_model_returned(self) -> None: def test_empty_model_uses_provider_default(self) -> None: cfg = AgentConfig(workspace=Path("/tmp"), provider="openai", model="") - self.assertEqual(_resolve_model_name(cfg), "gpt-5.2") + self.assertEqual(_resolve_model_name(cfg), "azure-foundry/gpt-5.3-codex") def test_empty_model_anthropic_default(self) -> None: cfg = AgentConfig(workspace=Path("/tmp"), provider="anthropic", model="") - self.assertEqual(_resolve_model_name(cfg), "claude-opus-4-6") + self.assertEqual(_resolve_model_name(cfg), "anthropic-foundry/claude-opus-4-6") def test_unknown_provider_fallback(self) -> None: cfg = AgentConfig(workspace=Path("/tmp"), provider="custom", model="") result = _resolve_model_name(cfg) - self.assertEqual(result, "claude-opus-4-6") + self.assertEqual(result, "anthropic-foundry/claude-opus-4-6") def test_newest_without_key_raises(self) -> None: - cfg = AgentConfig(workspace=Path("/tmp"), provider="openai", model="newest") + cfg = AgentConfig( + workspace=Path("/tmp"), + provider="openai", + model="newest", + openai_base_url="https://api.openai.com/v1", + openai_api_key=None, + ) with self.assertRaises(ModelError): _resolve_model_name(cfg) @@ -302,7 +367,7 @@ def test_openai_provider_with_key(self) -> None: cfg = AgentConfig( workspace=Path(tmpdir), provider="openai", - model="gpt-5.2", + model="azure-foundry/gpt-5.3-codex", openai_api_key="test-key", ) engine = build_engine(cfg) @@ -313,7 +378,7 @@ def test_anthropic_provider_with_key(self) -> None: cfg = AgentConfig( workspace=Path(tmpdir), provider="anthropic", - model="claude-opus-4-6", + model="anthropic-foundry/claude-opus-4-6", anthropic_api_key="test-key", ) engine = build_engine(cfg) @@ -324,8 +389,10 @@ def test_no_key_fallback_to_echo(self) -> None: cfg = AgentConfig( workspace=Path(tmpdir), provider="openai", - model="gpt-5.2", - ) + model="azure-foundry/gpt-5.3-codex", + openai_base_url="https://api.openai.com/v1", + openai_api_key=None, + ) engine = build_engine(cfg) self.assertIsInstance(engine.model, EchoFallbackModel) @@ -340,6 +407,33 @@ def test_openrouter_provider_with_key(self) -> None: engine = build_engine(cfg) self.assertIsInstance(engine.model, OpenAICompatibleModel) + def test_zai_stream_retries_propagated(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + cfg = AgentConfig( + workspace=Path(tmpdir), + provider="zai", + model="glm-5", + zai_api_key="test-key", + zai_stream_max_retries=10, + ) + engine = build_engine(cfg) + self.assertIsInstance(engine.model, OpenAICompatibleModel) + self.assertEqual(engine.model.stream_max_retries, 10) + + def test_zai_coding_plan_sets_coding_endpoint(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + env = { + "OPENPLANTER_PROVIDER": "zai", + "OPENPLANTER_MODEL": "glm-5", + "OPENPLANTER_ZAI_PLAN": "coding", + } + with patch.dict(os.environ, env, clear=True): + cfg = AgentConfig.from_env(Path(tmpdir)) + cfg.zai_api_key = "test-key" + engine = build_engine(cfg) + self.assertIsInstance(engine.model, OpenAICompatibleModel) + self.assertEqual(engine.model.base_url, cfg.zai_coding_base_url) + def test_model_provider_mismatch_raises(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: cfg = AgentConfig( diff --git a/tests/test_credentials.py b/tests/test_credentials.py index 79886207..29538747 100644 --- a/tests/test_credentials.py +++ b/tests/test_credentials.py @@ -22,7 +22,9 @@ def test_parse_env_file_extracts_supported_keys(self) -> None: "OPENAI_API_KEY=oa-key", "ANTHROPIC_API_KEY=an-key", "OPENROUTER_API_KEY=or-key", + "ZAI_API_KEY=zai-key", "EXA_API_KEY=exa-key", + "FIRECRAWL_API_KEY=fc-key", ] ), encoding="utf-8", @@ -31,7 +33,9 @@ def test_parse_env_file_extracts_supported_keys(self) -> None: self.assertEqual(creds.openai_api_key, "oa-key") self.assertEqual(creds.anthropic_api_key, "an-key") self.assertEqual(creds.openrouter_api_key, "or-key") + self.assertEqual(creds.zai_api_key, "zai-key") self.assertEqual(creds.exa_api_key, "exa-key") + self.assertEqual(creds.firecrawl_api_key, "fc-key") def test_store_roundtrip(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -41,7 +45,9 @@ def test_store_roundtrip(self) -> None: openai_api_key="oa", anthropic_api_key="an", openrouter_api_key="or", + zai_api_key="zai", exa_api_key="exa", + firecrawl_api_key="fc", ) store.save(creds) loaded = store.load() diff --git a/tests/test_engine_complex.py b/tests/test_engine_complex.py index e5bb29b7..a2e0f2de 100644 --- a/tests/test_engine_complex.py +++ b/tests/test_engine_complex.py @@ -8,7 +8,7 @@ from conftest import _tc from agent.config import AgentConfig from agent.engine import RLMEngine, ExternalContext -from agent.model import ModelTurn, ScriptedModel +from agent.model import Conversation, ModelTurn, RateLimitError, ScriptedModel, ToolResult from agent.tools import WorkspaceTools @@ -640,6 +640,117 @@ def test_think_tool_observation(self) -> None: self.assertEqual(result, "done") self.assertIn("Thought noted: my thought", returned_ctx.observations[0]) + # ------------------------------------------------------------------ + # 30. Rate-limit retries succeed without consuming extra step budget + # ------------------------------------------------------------------ + def test_rate_limit_retries_then_succeeds(self) -> None: + class RetryThenSuccessModel: + def __init__(self) -> None: + self.calls = 0 + + def create_conversation(self, system_prompt: str, initial_user_message: str) -> Conversation: + return Conversation(_provider_messages=[{"role": "user", "content": initial_user_message}]) + + def complete(self, conversation: Conversation) -> ModelTurn: + self.calls += 1 + if self.calls == 1: + raise RateLimitError( + "rate limit", + status_code=429, + provider_code="1302", + ) + return ModelTurn(text="done", stop_reason="end_turn") + + def append_assistant_turn(self, conversation: Conversation, turn: ModelTurn) -> None: + pass + + def append_tool_results(self, conversation: Conversation, results: list[ToolResult]) -> None: + pass + + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + cfg = AgentConfig( + workspace=root, + max_depth=1, + max_steps_per_call=1, + rate_limit_max_retries=3, + rate_limit_backoff_base_sec=0.0, + rate_limit_backoff_max_sec=0.0, + rate_limit_retry_after_cap_sec=0.0, + ) + tools = WorkspaceTools(root=root) + model = RetryThenSuccessModel() + engine = RLMEngine(model=model, tools=tools, config=cfg) + with patch("agent.engine.random.uniform", return_value=0.0): + result = engine.solve("retry test") + self.assertEqual(result, "done") + self.assertEqual(model.calls, 2) + + # ------------------------------------------------------------------ + # 31. Exhausted rate-limit retries surfaces model error + # ------------------------------------------------------------------ + def test_rate_limit_retries_exhausted_returns_model_error(self) -> None: + class AlwaysRateLimitModel: + def create_conversation(self, system_prompt: str, initial_user_message: str) -> Conversation: + return Conversation(_provider_messages=[{"role": "user", "content": initial_user_message}]) + + def complete(self, conversation: Conversation) -> ModelTurn: + raise RateLimitError("still rate limited", status_code=429, provider_code="1302") + + def append_assistant_turn(self, conversation: Conversation, turn: ModelTurn) -> None: + pass + + def append_tool_results(self, conversation: Conversation, results: list[ToolResult]) -> None: + pass + + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + cfg = AgentConfig( + workspace=root, + max_depth=1, + max_steps_per_call=1, + rate_limit_max_retries=2, + rate_limit_backoff_base_sec=0.0, + rate_limit_backoff_max_sec=0.0, + rate_limit_retry_after_cap_sec=0.0, + ) + tools = WorkspaceTools(root=root) + engine = RLMEngine(model=AlwaysRateLimitModel(), tools=tools, config=cfg) + with patch("agent.engine.random.uniform", return_value=0.0): + result = engine.solve("retry test") + self.assertIn("Model error at depth 0, step 1", result) + + # ------------------------------------------------------------------ + # 32. Deadline exits gracefully during rate-limit wait + # ------------------------------------------------------------------ + def test_rate_limit_wait_respects_deadline(self) -> None: + class SlowRateLimitModel: + def create_conversation(self, system_prompt: str, initial_user_message: str) -> Conversation: + return Conversation(_provider_messages=[{"role": "user", "content": initial_user_message}]) + + def complete(self, conversation: Conversation) -> ModelTurn: + raise RateLimitError("wait", status_code=429, retry_after_sec=10.0) + + def append_assistant_turn(self, conversation: Conversation, turn: ModelTurn) -> None: + pass + + def append_tool_results(self, conversation: Conversation, results: list[ToolResult]) -> None: + pass + + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + cfg = AgentConfig( + workspace=root, + max_depth=1, + max_steps_per_call=1, + max_solve_seconds=1, + rate_limit_max_retries=3, + ) + tools = WorkspaceTools(root=root) + engine = RLMEngine(model=SlowRateLimitModel(), tools=tools, config=cfg) + result = engine.solve("deadline retry test") + self.assertIn("Time limit exceeded", result) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_model.py b/tests/test_model.py index 19b1540e..0631eb19 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -4,7 +4,7 @@ from unittest.mock import patch from conftest import mock_anthropic_stream, mock_openai_stream -from agent.model import AnthropicModel, ModelError, OpenAICompatibleModel +from agent.model import AnthropicModel, HTTPModelError, ModelError, OpenAICompatibleModel, RateLimitError class ModelPayloadTests(unittest.TestCase): @@ -36,6 +36,62 @@ def fake_http_json(url, method, headers, payload=None, timeout_sec=90): # type: self.assertEqual(turn.text, "ok") self.assertEqual(captured["payload"]["reasoning_effort"], "high") + def test_openai_payload_strips_foundry_prefix(self) -> None: + captured: dict = {} + + def fake_http_json(url, method, headers, payload=None, timeout_sec=90): # type: ignore[no-untyped-def] + captured["payload"] = payload + return { + "choices": [ + { + "message": { + "content": "ok", + "tool_calls": None, + }, + "finish_reason": "stop", + } + ] + } + + with patch("agent.model._http_stream_sse", mock_openai_stream(fake_http_json)): + model = OpenAICompatibleModel( + model="azure-foundry/gpt-5.3-codex", + api_key="k", + reasoning_effort="high", + ) + conv = model.create_conversation("system", "user msg") + turn = model.complete(conv) + self.assertEqual(turn.text, "ok") + self.assertEqual(captured["payload"]["model"], "gpt-5.3-codex") + + def test_openai_payload_includes_thinking_type(self) -> None: + captured: dict = {} + + def fake_http_json(url, method, headers, payload=None, timeout_sec=90): # type: ignore[no-untyped-def] + captured["payload"] = payload + return { + "choices": [ + { + "message": { + "content": "ok", + "tool_calls": None, + }, + "finish_reason": "stop", + } + ] + } + + with patch("agent.model._http_stream_sse", mock_openai_stream(fake_http_json)): + model = OpenAICompatibleModel( + model="glm-5", + api_key="k", + thinking_type="enabled", + ) + conv = model.create_conversation("system", "user msg") + turn = model.complete(conv) + self.assertEqual(turn.text, "ok") + self.assertEqual(captured["payload"]["thinking"], {"type": "enabled"}) + def test_anthropic_payload_includes_thinking_budget(self) -> None: """Non-Opus-4.6 models use manual thinking with budget_tokens.""" captured: dict = {} @@ -58,6 +114,27 @@ def fake_http_json(url, method, headers, payload=None, timeout_sec=90): # type: self.assertEqual(turn.text, "ok") self.assertEqual(captured["payload"]["thinking"]["budget_tokens"], 4096) + def test_anthropic_payload_strips_foundry_prefix(self) -> None: + captured: dict = {} + + def fake_http_json(url, method, headers, payload=None, timeout_sec=90): # type: ignore[no-untyped-def] + captured["payload"] = payload + return { + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + } + + with patch("agent.model._http_stream_sse", mock_anthropic_stream(fake_http_json)): + model = AnthropicModel( + model="anthropic-foundry/claude-opus-4-6", + api_key="k", + reasoning_effort="high", + ) + conv = model.create_conversation("system", "user msg") + turn = model.complete(conv) + self.assertEqual(turn.text, "ok") + self.assertEqual(captured["payload"]["model"], "claude-opus-4-6") + def test_anthropic_opus46_uses_adaptive_thinking(self) -> None: """Opus 4.6 uses adaptive thinking with output_config effort.""" captured: dict = {} @@ -142,6 +219,124 @@ def fake_http_json(url, method, headers, payload=None, timeout_sec=90): # type: self.assertIn("thinking", calls[0]) self.assertNotIn("thinking", calls[1]) + def test_openai_reasoning_content_forwards_as_thinking(self) -> None: + deltas: list[tuple[str, str]] = [] + + def fake_stream_sse(url, method, headers, payload, first_byte_timeout=10, stream_timeout=120, max_retries=3, on_sse_event=None): # type: ignore[no-untyped-def] + events = [ + ("", {"choices": [{"delta": {"reasoning_content": "thinking text"}, "finish_reason": None}]}), + ("", {"choices": [{"delta": {"content": "final text"}, "finish_reason": None}]}), + ("", {"choices": [{"delta": {}, "finish_reason": "stop"}]}), + ] + if on_sse_event: + for event_type, data in events: + on_sse_event(event_type, data) + return events + + with patch("agent.model._http_stream_sse", fake_stream_sse): + model = OpenAICompatibleModel( + model="glm-5", + api_key="k", + on_content_delta=lambda delta_type, text: deltas.append((delta_type, text)), + ) + conv = model.create_conversation("system", "user msg") + turn = model.complete(conv) + self.assertEqual(turn.text, "final text") + self.assertIn(("thinking", "thinking text"), deltas) + self.assertIn(("text", "final text"), deltas) + + def test_openai_finish_reason_rate_limit_raises_rate_limit_error(self) -> None: + def fake_http_json(url, method, headers, payload=None, timeout_sec=90): # type: ignore[no-untyped-def] + return { + "choices": [ + { + "message": {"content": "partial", "tool_calls": None}, + "finish_reason": "rate_limit", + } + ] + } + + with patch("agent.model._http_stream_sse", mock_openai_stream(fake_http_json)): + model = OpenAICompatibleModel(model="glm-5", api_key="k") + conv = model.create_conversation("system", "user msg") + with self.assertRaises(RateLimitError): + model.complete(conv) + + def test_zai_uses_configured_endpoint_without_auto_switch(self) -> None: + calls: list[str] = [] + + def fake_stream_sse(url, method, headers, payload, first_byte_timeout=10, stream_timeout=120, max_retries=3, on_sse_event=None): # type: ignore[no-untyped-def] + calls.append(url) + if "/api/paas/v4/" in url: + raise HTTPModelError( + f"HTTP 404 calling {url}: not found", + status_code=404, + body='{"error":{"message":"not found"}}', + ) + return [] + + with patch("agent.model._http_stream_sse", fake_stream_sse): + model = OpenAICompatibleModel( + model="glm-5", + api_key="k", + base_url="https://api.z.ai/api/paas/v4", + provider="zai", + ) + conv = model.create_conversation("system", "user msg") + with self.assertRaises(HTTPModelError): + model.complete(conv) + self.assertEqual(model.base_url, "https://api.z.ai/api/paas/v4") + self.assertEqual(len(calls), 1) + + def test_openai_stream_retries_respected(self) -> None: + captured: dict[str, int] = {} + + def fake_stream_sse(url, method, headers, payload, first_byte_timeout=10, stream_timeout=120, max_retries=3, on_sse_event=None): # type: ignore[no-untyped-def] + captured["max_retries"] = max_retries + events = [ + ("", {"choices": [{"delta": {"content": "ok"}, "finish_reason": None}]}), + ("", {"choices": [{"delta": {}, "finish_reason": "stop"}]}), + ] + if on_sse_event: + for event_type, data in events: + on_sse_event(event_type, data) + return events + + with patch("agent.model._http_stream_sse", fake_stream_sse): + model = OpenAICompatibleModel( + model="gpt-4.1-mini", + api_key="k", + stream_max_retries=7, + ) + conv = model.create_conversation("system", "user msg") + model.complete(conv) + self.assertEqual(captured.get("max_retries"), 7) + + def test_zai_stream_retries_respected(self) -> None: + captured: dict[str, int] = {} + + def fake_stream_sse(url, method, headers, payload, first_byte_timeout=10, stream_timeout=120, max_retries=3, on_sse_event=None): # type: ignore[no-untyped-def] + captured["max_retries"] = max_retries + events = [ + ("", {"choices": [{"delta": {"content": "ok"}, "finish_reason": None}]}), + ("", {"choices": [{"delta": {}, "finish_reason": "stop"}]}), + ] + if on_sse_event: + for event_type, data in events: + on_sse_event(event_type, data) + return events + + with patch("agent.model._http_stream_sse", fake_stream_sse): + model = OpenAICompatibleModel( + model="glm-5", + api_key="k", + provider="zai", + stream_max_retries=10, + ) + conv = model.create_conversation("system", "user msg") + model.complete(conv) + self.assertEqual(captured.get("max_retries"), 10) + class OllamaPayloadTests(unittest.TestCase): def test_ollama_uses_openai_compatible_format(self) -> None: diff --git a/tests/test_settings.py b/tests/test_settings.py index 2f85fa12..d39c08ea 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -40,6 +40,7 @@ def test_per_provider_model_roundtrip(self) -> None: default_model_openai="gpt-4.1-mini", default_model_anthropic="claude-opus-4-6", default_model_openrouter="anthropic/claude-sonnet-4-5", + default_model_zai="glm-5", ) store.save(settings) loaded = store.load() @@ -47,6 +48,7 @@ def test_per_provider_model_roundtrip(self) -> None: self.assertEqual(loaded.default_model_openai, "gpt-4.1-mini") self.assertEqual(loaded.default_model_anthropic, "claude-opus-4-6") self.assertEqual(loaded.default_model_openrouter, "anthropic/claude-sonnet-4-5") + self.assertEqual(loaded.default_model_zai, "glm-5") def test_default_model_for_provider_specific(self) -> None: settings = PersistentSettings( @@ -66,6 +68,7 @@ def test_default_model_for_provider_none(self) -> None: self.assertIsNone(settings.default_model_for_provider("anthropic")) self.assertIsNone(settings.default_model_for_provider("openrouter")) self.assertIsNone(settings.default_model_for_provider("cerebras")) + self.assertIsNone(settings.default_model_for_provider("zai")) def test_per_provider_model_ollama(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -85,6 +88,13 @@ def test_default_model_for_provider_ollama(self) -> None: ) self.assertEqual(settings.default_model_for_provider("ollama"), "llama3.2") + def test_default_model_for_provider_zai(self) -> None: + settings = PersistentSettings( + default_model="global-model", + default_model_zai="glm-5", + ) + self.assertEqual(settings.default_model_for_provider("zai"), "glm-5") + def test_backward_compat_old_settings(self) -> None: """Old settings.json without per-provider keys still loads fine.""" import json @@ -100,6 +110,7 @@ def test_backward_compat_old_settings(self) -> None: self.assertIsNone(loaded.default_model_openai) self.assertIsNone(loaded.default_model_anthropic) self.assertIsNone(loaded.default_model_openrouter) + self.assertIsNone(loaded.default_model_zai) class ComputeSuggestionsTests(unittest.TestCase): @@ -155,11 +166,19 @@ def test_claude_is_anthropic(self) -> None: self.assertEqual(infer_provider_for_model("claude-opus-4-6"), "anthropic") self.assertEqual(infer_provider_for_model("claude-sonnet-4-5-20250929"), "anthropic") self.assertEqual(infer_provider_for_model("Claude-3-Haiku"), "anthropic") + self.assertEqual( + infer_provider_for_model("anthropic-foundry/claude-opus-4-6"), + "anthropic", + ) def test_gpt_is_openai(self) -> None: self.assertEqual(infer_provider_for_model("gpt-5.2"), "openai") self.assertEqual(infer_provider_for_model("gpt-4.1-mini"), "openai") self.assertEqual(infer_provider_for_model("GPT-4o"), "openai") + self.assertEqual( + infer_provider_for_model("azure-foundry/gpt-5.3-codex"), + "openai", + ) def test_o_series_is_openai(self) -> None: self.assertEqual(infer_provider_for_model("o1-mini"), "openai") @@ -190,6 +209,10 @@ def test_cerebras_qwen3_not_ollama(self) -> None: """qwen-3 models go to Cerebras, not Ollama.""" self.assertEqual(infer_provider_for_model("qwen-3-235b-a22b-instruct-2507"), "cerebras") + def test_zai_models(self) -> None: + self.assertEqual(infer_provider_for_model("glm-5"), "zai") + self.assertEqual(infer_provider_for_model("GLM-4.5"), "zai") + def test_unknown_returns_none(self) -> None: self.assertIsNone(infer_provider_for_model("my-custom-model")) self.assertIsNone(infer_provider_for_model("some-random-model")) @@ -200,6 +223,7 @@ def test_matching_provider_passes(self) -> None: _validate_model_provider("gpt-5.2", "openai") _validate_model_provider("claude-opus-4-6", "anthropic") _validate_model_provider("anthropic/claude-sonnet-4-5", "openrouter") + _validate_model_provider("glm-5", "zai") def test_mismatch_raises(self) -> None: with self.assertRaises(ModelError): diff --git a/tests/test_streaming.py b/tests/test_streaming.py index ac031f85..293f6e5d 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -8,7 +8,9 @@ from unittest.mock import MagicMock, patch from agent.model import ( + HTTPModelError, ModelError, + RateLimitError, _accumulate_anthropic_stream, _accumulate_openai_stream, _http_stream_sse, @@ -77,6 +79,15 @@ def test_anthropic_error_event_raises(self) -> None: _read_sse_events(resp) self.assertIn("Overloaded", str(ctx.exception)) + def test_openai_style_rate_limit_error_event_raises(self) -> None: + resp = self._make_resp([ + 'data: {"error":{"code":"1302","message":"Rate limit reached for requests"}}', + '', + ]) + with self.assertRaises(RateLimitError) as ctx: + _read_sse_events(resp) + self.assertIn("Rate limit", str(ctx.exception)) + def test_done_terminates_early(self) -> None: resp = self._make_resp([ 'data: {"choices":[{"delta":{"content":"a"}}]}', @@ -270,6 +281,77 @@ def fake_urlopen(req, timeout=None): # Should only be called once — no retries on HTTP errors self.assertEqual(call_count, 1) + def test_http_429_raises_rate_limit_error(self) -> None: + call_count = 0 + + def fake_urlopen(req, timeout=None): + nonlocal call_count + call_count += 1 + import urllib.error + raise urllib.error.HTTPError( + url="http://test", + code=429, + msg="Too Many Requests", + hdrs={"Retry-After": "2"}, + fp=io.BytesIO(b'{"error":{"message":"Too many requests","code":"rate_limit_exceeded"}}'), + ) + + with patch("agent.model.urllib.request.urlopen", fake_urlopen): + with self.assertRaises(RateLimitError) as ctx: + _http_stream_sse( + url="http://test/v1/chat/completions", + method="POST", + headers={}, + payload={"model": "test"}, + max_retries=3, + ) + self.assertEqual(ctx.exception.status_code, 429) + self.assertEqual(call_count, 1) + + def test_http_400_with_code_1302_raises_rate_limit_error(self) -> None: + def fake_urlopen(req, timeout=None): + import urllib.error + raise urllib.error.HTTPError( + url="http://test", + code=400, + msg="Bad Request", + hdrs={}, + fp=io.BytesIO(b'{"error":{"message":"Rate limit reached for requests","code":"1302"}}'), + ) + + with patch("agent.model.urllib.request.urlopen", fake_urlopen): + with self.assertRaises(RateLimitError) as ctx: + _http_stream_sse( + url="http://test/v1/chat/completions", + method="POST", + headers={}, + payload={"model": "test"}, + max_retries=3, + ) + self.assertEqual(ctx.exception.provider_code, "1302") + + def test_http_400_non_rate_limit_raises_http_model_error(self) -> None: + def fake_urlopen(req, timeout=None): + import urllib.error + raise urllib.error.HTTPError( + url="http://test", + code=400, + msg="Bad Request", + hdrs={}, + fp=io.BytesIO(b'{"error":{"message":"bad request","code":"invalid_request"}}'), + ) + + with patch("agent.model.urllib.request.urlopen", fake_urlopen): + with self.assertRaises(HTTPModelError) as ctx: + _http_stream_sse( + url="http://test/v1/chat/completions", + method="POST", + headers={}, + payload={"model": "test"}, + max_retries=3, + ) + self.assertEqual(ctx.exception.status_code, 400) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_tools.py b/tests/test_tools.py index 844722e0..a5590a56 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -77,6 +77,7 @@ def test_web_search_with_mocked_exa_response(self) -> None: with patch.object(WorkspaceTools, "_exa_request", return_value=mocked): raw = tools.web_search("test query", num_results=3, include_text=True) parsed = json.loads(raw) + self.assertEqual(parsed["provider"], "exa") self.assertEqual(parsed["query"], "test query") self.assertEqual(parsed["total"], 1) self.assertEqual(parsed["results"][0]["url"], "https://example.com") @@ -98,6 +99,57 @@ def test_fetch_url_with_mocked_exa_response(self) -> None: with patch.object(WorkspaceTools, "_exa_request", return_value=mocked): raw = tools.fetch_url(["https://example.com"]) parsed = json.loads(raw) + self.assertEqual(parsed["provider"], "exa") + self.assertEqual(parsed["total"], 1) + self.assertEqual(parsed["pages"][0]["url"], "https://example.com") + self.assertEqual(parsed["pages"][0]["text"], "Page body") + + def test_web_search_with_mocked_firecrawl_response(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + tools = WorkspaceTools( + root=root, + web_search_provider="firecrawl", + firecrawl_api_key="fc-key", + ) + mocked = { + "data": [ + { + "url": "https://example.com", + "title": "Example", + "description": "Snippet", + "markdown": "Long text body", + } + ] + } + with patch.object(WorkspaceTools, "_firecrawl_request", return_value=mocked): + raw = tools.web_search("test query", num_results=3, include_text=True) + parsed = json.loads(raw) + self.assertEqual(parsed["provider"], "firecrawl") + self.assertEqual(parsed["query"], "test query") + self.assertEqual(parsed["total"], 1) + self.assertEqual(parsed["results"][0]["url"], "https://example.com") + self.assertIn("text", parsed["results"][0]) + + def test_fetch_url_with_mocked_firecrawl_response(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + tools = WorkspaceTools( + root=root, + web_search_provider="firecrawl", + firecrawl_api_key="fc-key", + ) + mocked = { + "data": { + "url": "https://example.com", + "metadata": {"title": "Example"}, + "markdown": "Page body", + } + } + with patch.object(WorkspaceTools, "_firecrawl_request", return_value=mocked): + raw = tools.fetch_url(["https://example.com"]) + parsed = json.loads(raw) + self.assertEqual(parsed["provider"], "firecrawl") self.assertEqual(parsed["total"], 1) self.assertEqual(parsed["pages"][0]["url"], "https://example.com") self.assertEqual(parsed["pages"][0]["text"], "Page body") @@ -109,6 +161,13 @@ def test_web_search_without_exa_key(self) -> None: out = tools.web_search("test") self.assertIn("EXA_API_KEY not configured", out) + def test_web_search_without_firecrawl_key(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + tools = WorkspaceTools(root=root, web_search_provider="firecrawl", firecrawl_api_key=None) + out = tools.web_search("test") + self.assertIn("FIRECRAWL_API_KEY not configured", out) + def test_repo_map_python_symbols(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) diff --git a/tests/test_tools_complex.py b/tests/test_tools_complex.py index bfd85ddf..edb258a5 100644 --- a/tests/test_tools_complex.py +++ b/tests/test_tools_complex.py @@ -126,6 +126,20 @@ def test_web_search_clamps_num_results(self) -> None: payload = mock_exa.call_args[0][1] self.assertEqual(payload["numResults"], 20) + def test_web_search_clamps_num_results_firecrawl(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + tools = WorkspaceTools( + root=Path(tmpdir), web_search_provider="firecrawl", firecrawl_api_key="test-key" + ) + mock_response = {"data": []} + with patch.object( + WorkspaceTools, "_firecrawl_request", return_value=mock_response + ) as mock_fc: + tools.web_search("test query", num_results=50) + mock_fc.assert_called_once() + payload = mock_fc.call_args[0][1] + self.assertEqual(payload["limit"], 20) + # 12 def test_fetch_url_non_list_returns_error(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -164,6 +178,13 @@ def test_exa_request_no_key_raises(self) -> None: tools._exa_request("/search", {"query": "test"}) self.assertIn("EXA_API_KEY not configured", str(ctx.exception)) + def test_firecrawl_request_no_key_raises(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + tools = WorkspaceTools(root=Path(tmpdir), firecrawl_api_key=None) + with self.assertRaises(ToolError) as ctx: + tools._firecrawl_request("/search", {"query": "test"}) + self.assertIn("FIRECRAWL_API_KEY not configured", str(ctx.exception)) + # 16 def test_write_file_creates_nested_dirs(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: From 8b85e770ec03f8f90a9ab0158f6716661ad6a1fb Mon Sep 17 00:00:00 2001 From: Drake Date: Wed, 11 Mar 2026 16:15:22 -0400 Subject: [PATCH 02/58] chore: add codex fork sync workflow --- .github/prompts/codex-fork-sync.prompt.md | 30 +++++ .github/workflows/codex-fork-sync.yml | 139 ++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 .github/prompts/codex-fork-sync.prompt.md create mode 100644 .github/workflows/codex-fork-sync.yml diff --git a/.github/prompts/codex-fork-sync.prompt.md b/.github/prompts/codex-fork-sync.prompt.md new file mode 100644 index 00000000..105e84d6 --- /dev/null +++ b/.github/prompts/codex-fork-sync.prompt.md @@ -0,0 +1,30 @@ +You are running inside GitHub Actions for the fork of the `OpenPlanter` repository. + +Your job is to sync the fork with upstream and rebase active fork branches on top of the latest upstream main branch. + +Repository layout: +- `origin` is the fork: `ThomsenDrake/OpenPlanter` +- `upstream` is the source repo: `ShinMegamiBoson/OpenPlanter` + +Constraints: +- Operate only on refs that have already been fetched locally. +- Do not run network commands. +- Do not edit product code, docs, or workflow files. +- Do not add untracked files. +- Only manipulate git branches and commits. +- Leave the repository on the local `main` branch with a clean working tree and no staged changes. + +Required outcome: +1. If `origin/main` already matches `upstream/main`, make no changes and say so. +2. Otherwise, move local `main` to exactly `upstream/main`. +3. For every fork branch that exists as `origin/chore/*`: + - Create or refresh a matching local `chore/*` branch from the remote branch. + - Determine whether it has commits not already contained in `upstream/main`. + - If it has unique commits, rebase those commits onto `upstream/main`. + - If it is already fully contained in `upstream/main`, leave it alone. +4. If any rebase hits conflicts, stop immediately and report the branch name plus the conflicting files. + +Guidance: +- Because this is a clean CI checkout, it is acceptable to force local branch pointers when needed. +- Favor deterministic git commands over exploratory edits. +- Keep a short summary of what you changed, including branch names and resulting commit SHAs. diff --git a/.github/workflows/codex-fork-sync.yml b/.github/workflows/codex-fork-sync.yml new file mode 100644 index 00000000..a9e1f602 --- /dev/null +++ b/.github/workflows/codex-fork-sync.yml @@ -0,0 +1,139 @@ +name: Codex Fork Sync + +on: + schedule: + - cron: "17 * * * *" + workflow_dispatch: + inputs: + force: + description: Run even if upstream/main has not moved + required: false + default: false + type: boolean + +permissions: + contents: write + +concurrency: + group: codex-fork-sync + cancel-in-progress: false + +jobs: + sync: + runs-on: ubuntu-latest + + steps: + - name: Check out fork + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: main + + - name: Ensure Codex secret exists + if: ${{ secrets.OPENAI_API_KEY == '' }} + run: | + echo "Set the OPENAI_API_KEY repository secret to enable Codex fork sync." >> "$GITHUB_STEP_SUMMARY" + echo "Missing OPENAI_API_KEY secret." >&2 + exit 1 + + - name: Prepare git state + id: prepare + run: | + set -euo pipefail + + git remote add upstream https://github.com/ShinMegamiBoson/OpenPlanter.git 2>/dev/null || \ + git remote set-url upstream https://github.com/ShinMegamiBoson/OpenPlanter.git + + git fetch --prune --no-tags origin '+refs/heads/*:refs/remotes/origin/*' + git fetch --prune --no-tags upstream '+refs/heads/*:refs/remotes/upstream/*' + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + upstream_sha="$(git rev-parse upstream/main)" + fork_sha="$(git rev-parse origin/main)" + changed=false + + if [ "$upstream_sha" != "$fork_sha" ]; then + changed=true + fi + + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.force }}" = "true" ]; then + changed=true + fi + + echo "upstream_sha=$upstream_sha" >> "$GITHUB_OUTPUT" + echo "fork_sha=$fork_sha" >> "$GITHUB_OUTPUT" + echo "changed=$changed" >> "$GITHUB_OUTPUT" + + - name: Report no-op + if: steps.prepare.outputs.changed != 'true' + run: | + { + echo "### Codex Fork Sync" + echo + echo "No sync needed." + echo + echo "- upstream/main: \`${{ steps.prepare.outputs.upstream_sha }}\`" + echo "- fork/main: \`${{ steps.prepare.outputs.fork_sha }}\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run Codex in GitHub Actions + if: steps.prepare.outputs.changed == 'true' + id: codex + uses: openai/codex-action@v1 + with: + openai-api-key: ${{ secrets.OPENAI_API_KEY }} + prompt-file: .github/prompts/codex-fork-sync.prompt.md + output-file: .codex-fork-sync-summary.md + working-directory: . + sandbox: danger-full-access + safety-strategy: drop-sudo + allow-bots: true + + - name: Require clean worktree after Codex + if: steps.prepare.outputs.changed == 'true' && success() + run: | + set -euo pipefail + git diff --quiet + git diff --cached --quiet + + - name: Push synced branches + if: steps.prepare.outputs.changed == 'true' && success() + run: | + set -euo pipefail + + git push origin main:main + + while IFS= read -r branch; do + remote_ref="refs/remotes/origin/${branch}" + if ! git show-ref --verify --quiet "$remote_ref"; then + continue + fi + + local_sha="$(git rev-parse "$branch")" + remote_sha="$(git rev-parse "origin/${branch}")" + + if [ "$local_sha" != "$remote_sha" ]; then + git push --force-with-lease origin "${branch}:${branch}" + fi + done < <(git for-each-ref --format='%(refname:short)' refs/heads/chore/) + + - name: Publish Codex summary + if: steps.prepare.outputs.changed == 'true' && always() + run: | + { + echo "### Codex Fork Sync" + echo + echo "- upstream/main before sync: \`${{ steps.prepare.outputs.upstream_sha }}\`" + echo "- fork/main before sync: \`${{ steps.prepare.outputs.fork_sha }}\`" + echo + } >> "$GITHUB_STEP_SUMMARY" + + if [ -f .codex-fork-sync-summary.md ]; then + cat .codex-fork-sync-summary.md >> "$GITHUB_STEP_SUMMARY" + elif [ "${{ steps.codex.outcome }}" = "success" ]; then + echo "Codex completed without a written summary." >> "$GITHUB_STEP_SUMMARY" + else + echo "Codex did not complete successfully." >> "$GITHUB_STEP_SUMMARY" + fi From ec2fe8d73d454f0bf253f317b88bd3a8b2f6849a Mon Sep 17 00:00:00 2001 From: Drake Date: Thu, 12 Mar 2026 10:17:25 -0400 Subject: [PATCH 03/58] chore: checkpoint current docs and code changes --- .../crates/op-core/src/builder.rs | 32 +- .../crates/op-core/src/config.rs | 12 +- .../crates/op-core/src/engine/curator.rs | 19 +- .../crates/op-core/src/engine/mod.rs | 237 +++++++++-- .../crates/op-core/src/tools/defs.rs | 6 +- .../crates/op-core/src/tools/mod.rs | 171 +++++++- .../crates/op-tauri/src/bridge.rs | 195 ++++++++- .../crates/op-tauri/src/commands/agent.rs | 12 + .../crates/op-tauri/src/commands/config.rs | 10 +- .../crates/op-tauri/src/commands/session.rs | 3 +- .../crates/op-tauri/src/main.rs | 5 +- .../crates/op-tauri/src/state.rs | 387 +++++++++++++++++- .../frontend/e2e/streaming.spec.ts | 52 +++ .../frontend/src/styles/main.css | 29 +- uv.lock | 204 +++++++++ 15 files changed, 1286 insertions(+), 88 deletions(-) create mode 100644 uv.lock diff --git a/openplanter-desktop/crates/op-core/src/builder.rs b/openplanter-desktop/crates/op-core/src/builder.rs index 1be274c7..786e4c10 100644 --- a/openplanter-desktop/crates/op-core/src/builder.rs +++ b/openplanter-desktop/crates/op-core/src/builder.rs @@ -151,14 +151,17 @@ pub fn resolve_endpoint(cfg: &AgentConfig, provider: &str) -> Result<(String, St match provider { "anthropic" => { let key = resolve_anthropic_api_key( - cfg.anthropic_api_key.clone().or_else(|| cfg.api_key.clone()), + cfg.anthropic_api_key + .clone() + .or_else(|| cfg.api_key.clone()), &cfg.anthropic_base_url, ) .ok_or_else(|| { - ModelError::Message( - "No Anthropic API key. Set ANTHROPIC_API_KEY or OPENPLANTER_ANTHROPIC_API_KEY.".into(), - ) - })?; + ModelError::Message( + "No Anthropic API key. Set ANTHROPIC_API_KEY or OPENPLANTER_ANTHROPIC_API_KEY." + .into(), + ) + })?; // Anthropic base URL does NOT include /v1 suffix for /messages endpoint — // the model adapter appends /messages itself. The config stores it with /v1. Ok((cfg.anthropic_base_url.clone(), key)) @@ -169,11 +172,10 @@ pub fn resolve_endpoint(cfg: &AgentConfig, provider: &str) -> Result<(String, St &cfg.openai_base_url, ) .ok_or_else(|| { - ModelError::Message( - "No OpenAI API key. Set OPENAI_API_KEY or OPENPLANTER_OPENAI_API_KEY." - .into(), - ) - })?; + ModelError::Message( + "No OpenAI API key. Set OPENAI_API_KEY or OPENPLANTER_OPENAI_API_KEY.".into(), + ) + })?; Ok((cfg.openai_base_url.clone(), key)) } "openrouter" => { @@ -384,7 +386,10 @@ mod tests { provider: "openai".into(), ..Default::default() }; - assert_eq!(resolve_model_name(&cfg).unwrap(), "azure-foundry/gpt-5.3-codex"); + assert_eq!( + resolve_model_name(&cfg).unwrap(), + "azure-foundry/gpt-5.3-codex" + ); } #[test] @@ -394,7 +399,10 @@ mod tests { provider: "openai".into(), ..Default::default() }; - assert_eq!(resolve_model_name(&cfg).unwrap(), "azure-foundry/gpt-5.3-codex"); + assert_eq!( + resolve_model_name(&cfg).unwrap(), + "azure-foundry/gpt-5.3-codex" + ); } // ── resolve_provider ── diff --git a/openplanter-desktop/crates/op-core/src/config.rs b/openplanter-desktop/crates/op-core/src/config.rs index 06ff4c86..b25abbe0 100644 --- a/openplanter-desktop/crates/op-core/src/config.rs +++ b/openplanter-desktop/crates/op-core/src/config.rs @@ -7,14 +7,11 @@ use serde::{Deserialize, Serialize}; pub const AZURE_FOUNDRY_MODEL_PREFIX: &str = "azure-foundry/"; pub const ANTHROPIC_FOUNDRY_MODEL_PREFIX: &str = "anthropic-foundry/"; -pub const FOUNDRY_OPENAI_BASE_URL: &str = - "https://foundry-proxy.cheetah-koi.ts.net/openai/v1"; +pub const FOUNDRY_OPENAI_BASE_URL: &str = "https://foundry-proxy.cheetah-koi.ts.net/openai/v1"; pub const FOUNDRY_ANTHROPIC_BASE_URL: &str = "https://foundry-proxy.cheetah-koi.ts.net/anthropic/v1"; -pub const FOUNDRY_OPENAI_API_KEY_PLACEHOLDER: &str = - "dont-worry-this-key-will-be-auto-injected"; -pub const FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER: &str = - "dont-worry-it-will-be-injected"; +pub const FOUNDRY_OPENAI_API_KEY_PLACEHOLDER: &str = "dont-worry-this-key-will-be-auto-injected"; +pub const FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER: &str = "dont-worry-it-will-be-injected"; pub const ZAI_PAYGO_BASE_URL: &str = "https://api.z.ai/api/paas/v4"; pub const ZAI_CODING_BASE_URL: &str = "https://api.z.ai/api/coding/paas/v4"; @@ -294,7 +291,8 @@ impl AgentConfig { let openai_base_url = env_opt("OPENPLANTER_OPENAI_BASE_URL") .or_else(|| env_opt("OPENPLANTER_BASE_URL")) .unwrap_or_else(|| FOUNDRY_OPENAI_BASE_URL.into()); - let anthropic_base_url = env_or("OPENPLANTER_ANTHROPIC_BASE_URL", FOUNDRY_ANTHROPIC_BASE_URL); + let anthropic_base_url = + env_or("OPENPLANTER_ANTHROPIC_BASE_URL", FOUNDRY_ANTHROPIC_BASE_URL); let openai_api_key = resolve_openai_api_key(openai_api_key, &openai_base_url); let anthropic_api_key = resolve_anthropic_api_key(anthropic_api_key, &anthropic_base_url); diff --git a/openplanter-desktop/crates/op-core/src/engine/curator.rs b/openplanter-desktop/crates/op-core/src/engine/curator.rs index 7d50a61b..ea683acd 100644 --- a/openplanter-desktop/crates/op-core/src/engine/curator.rs +++ b/openplanter-desktop/crates/op-core/src/engine/curator.rs @@ -31,8 +31,9 @@ Your ONLY job is to update the wiki at .openplanter/wiki/ based on the main agen 6. Use EXACT source names in Cross-Reference sections to power the knowledge graph 7. If nothing in the step context is wiki-relevant, respond with ONLY: "No wiki updates needed" 8. Keep entries factual and concise — document what was found, not speculation -9. Never modify files outside .openplanter/wiki/ -10. Maximum 8 tool calls — be efficient +9. Never modify files outside .openplanter/wiki/ — this is enforced at runtime +10. Only use write_file or edit_file for mutations +11. Maximum 8 tool calls — be efficient == WIKI ENTRY TEMPLATE == When creating a new entry, use this format: @@ -126,8 +127,6 @@ pub const CURATOR_TOOL_NAMES: &[&str] = &[ "read_file", "write_file", "edit_file", - "apply_patch", - "hashline_edit", "think", ]; @@ -152,7 +151,7 @@ pub async fn run_curator( let provider = model.provider_name().to_string(); let tool_defs = build_curator_tool_defs(&provider); - let mut tools = WorkspaceTools::new(config); + let mut tools = WorkspaceTools::new_curator(config); let mut messages = vec![ Message::System { @@ -227,11 +226,7 @@ pub async fn run_curator( let result = tools.execute(&tc.name, &tc.arguments).await; // Track file modifications - if matches!( - tc.name.as_str(), - "write_file" | "edit_file" | "apply_patch" | "hashline_edit" - ) && !result.is_error - { + if matches!(tc.name.as_str(), "write_file" | "edit_file") && !result.is_error { files_changed += 1; // Extract path for summary if let Ok(args) = serde_json::from_str::(&tc.arguments) { @@ -365,7 +360,9 @@ mod tests { "run_shell", "run_shell_bg", "check_shell_bg", - "kill_shell_bg" + "kill_shell_bg", + "apply_patch", + "hashline_edit" ] .contains(name), "Curator should not have access to {name}" diff --git a/openplanter-desktop/crates/op-core/src/engine/mod.rs b/openplanter-desktop/crates/op-core/src/engine/mod.rs index f19e38b1..0df7fe9d 100644 --- a/openplanter-desktop/crates/op-core/src/engine/mod.rs +++ b/openplanter-desktop/crates/op-core/src/engine/mod.rs @@ -30,10 +30,50 @@ enum CuratorOutcome { Error(String), } -/// Abort all in-flight curator tasks. -fn abort_curators(handles: &mut Vec>) { - for h in handles.drain(..) { - h.abort(); +fn spawn_curator_task( + context: String, + tx: mpsc::UnboundedSender, + config: AgentConfig, + cancel: CancellationToken, +) -> JoinHandle<()> { + tokio::spawn(async move { + let outcome = match run_curator(&context, &config, cancel).await { + Ok(result) => CuratorOutcome::Done(result), + Err(err) => CuratorOutcome::Error(err), + }; + let _ = tx.send(outcome); + }) +} + +fn schedule_curator_context( + has_running_curator: bool, + queued_context: &mut Option, + context: String, +) -> Option { + if has_running_curator { + *queued_context = Some(context); + None + } else { + Some(context) + } +} + +fn take_queued_context_if_idle( + has_running_curator: bool, + queued_context: &mut Option, +) -> Option { + if has_running_curator { + None + } else { + queued_context.take() + } +} + +/// Abort any active curator and clear pending work. +fn abort_curators(running: &mut Option>, queued_context: &mut Option) { + queued_context.take(); + if let Some(handle) = running.take() { + handle.abort(); } } @@ -67,34 +107,97 @@ fn drain_curator_results( /// Wait for in-flight curators (up to timeout), drain final results, abort rest. async fn finish_curators( - handles: &mut Vec>, + running: &mut Option>, + queued_context: &mut Option, + tx: &mpsc::UnboundedSender, + config: &AgentConfig, + cancel: &CancellationToken, rx: &mut mpsc::UnboundedReceiver, messages: &mut Vec, emitter: &dyn SolveEmitter, ) { - if handles.is_empty() { + if running.is_none() && queued_context.is_none() { return; } emitter.emit_trace(&format!( - "[curator] waiting for {} in-flight curator(s)...", - handles.len() + "[curator] waiting for {} pending curator task(s)...", + usize::from(running.is_some()) + usize::from(queued_context.is_some()) )); // Wait up to 30 seconds total for all curators to finish let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); - for h in handles.iter_mut() { - let remaining = deadline - tokio::time::Instant::now(); + loop { + if running.is_none() { + if let Some(context) = take_queued_context_if_idle(false, queued_context) { + emitter.emit_trace("[curator] spawning queued update"); + *running = Some(spawn_curator_task( + context, + tx.clone(), + config.clone(), + cancel.clone(), + )); + } else { + break; + } + } + + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); if remaining.is_zero() { break; } - let _ = tokio::time::timeout(remaining, h).await; + + if let Some(mut handle) = running.take() { + match tokio::time::timeout(remaining, &mut handle).await { + Ok(_) => { + drain_curator_results(rx, messages, emitter); + } + Err(_) => { + *running = Some(handle); + break; + } + } + } } // Final drain drain_curator_results(rx, messages, emitter); // Abort any still running - abort_curators(handles); + abort_curators(running, queued_context); +} + +async fn poll_curator_state( + running: &mut Option>, + queued_context: &mut Option, + tx: &mpsc::UnboundedSender, + config: &AgentConfig, + cancel: &CancellationToken, + rx: &mut mpsc::UnboundedReceiver, + messages: &mut Vec, + emitter: &dyn SolveEmitter, +) { + drain_curator_results(rx, messages, emitter); + + let should_join = running + .as_ref() + .map(|handle| handle.is_finished()) + .unwrap_or(false); + if should_join { + if let Some(mut handle) = running.take() { + let _ = (&mut handle).await; + } + drain_curator_results(rx, messages, emitter); + } + + if let Some(context) = take_queued_context_if_idle(running.is_some(), queued_context) { + emitter.emit_trace("[curator] spawning queued update"); + *running = Some(spawn_curator_task( + context, + tx.clone(), + config.clone(), + cancel.clone(), + )); + } } // Abstraction for emitting solve events. @@ -195,6 +298,11 @@ fn estimate_tokens(messages: &[Message]) -> usize { / 4 } +fn safe_prefix(text: &str, max_chars: usize) -> &str { + let end = text.floor_char_boundary(text.len().min(max_chars)); + &text[..end] +} + /// Compact conversation context when it grows too large. /// /// Keeps the system prompt, user objective, and the most recent messages @@ -212,7 +320,7 @@ fn compact_messages(messages: &mut Vec, max_tokens: usize) { for i in 2..protected_tail { if let Message::Tool { content, .. } = &mut messages[i] { if content.len() > 200 { - let preview = &content[..content.len().min(150)]; + let preview = safe_prefix(content, 150); *content = format!("{preview}\n...[truncated — older tool result]"); } } @@ -336,19 +444,29 @@ pub async fn solve( // 3. Background curator channel let (curator_tx, mut curator_rx) = mpsc::unbounded_channel::(); - let mut curator_handles: Vec> = Vec::new(); + let mut running_curator: Option> = None; + let mut queued_curator_context: Option = None; // 4. Agentic loop for step in 1..=max_steps { if cancel.is_cancelled() { emitter.emit_error("Cancelled"); tools.cleanup(); - abort_curators(&mut curator_handles); + abort_curators(&mut running_curator, &mut queued_curator_context); return; } - // Drain completed curator results and inject as system messages - drain_curator_results(&mut curator_rx, &mut messages, emitter); + poll_curator_state( + &mut running_curator, + &mut queued_curator_context, + &curator_tx, + config, + &cancel, + &mut curator_rx, + &mut messages, + emitter, + ) + .await; let step_start = std::time::Instant::now(); @@ -372,7 +490,7 @@ pub async fn solve( Err(e) => { let msg = e.to_string(); tools.cleanup(); - abort_curators(&mut curator_handles); + abort_curators(&mut running_curator, &mut queued_curator_context); if msg == "Cancelled" { emitter.emit_error("Cancelled"); } else { @@ -411,7 +529,11 @@ pub async fn solve( tools.cleanup(); // Wait for in-flight curators before exiting finish_curators( - &mut curator_handles, + &mut running_curator, + &mut queued_curator_context, + &curator_tx, + config, + &cancel, &mut curator_rx, &mut messages, emitter, @@ -425,7 +547,7 @@ pub async fn solve( if cancel.is_cancelled() { emitter.emit_error("Cancelled"); tools.cleanup(); - abort_curators(&mut curator_handles); + abort_curators(&mut running_curator, &mut queued_curator_context); return; } @@ -436,7 +558,7 @@ pub async fn solve( emitter.emit_trace(&format!( "Tool {} error: {}", tc.name, - &result.content[..result.content.len().min(200)] + safe_prefix(&result.content, 200) )); } @@ -464,17 +586,21 @@ pub async fn solve( // Spawn background curator after each non-final step let context = extract_step_context(&messages); if !context.is_empty() { - let tx = curator_tx.clone(); - let curator_cfg = config.clone(); - let curator_cancel = cancel.clone(); - emitter.emit_trace(&format!("[curator] spawning for step {step}")); - curator_handles.push(tokio::spawn(async move { - let outcome = match run_curator(&context, &curator_cfg, curator_cancel).await { - Ok(result) => CuratorOutcome::Done(result), - Err(e) => CuratorOutcome::Error(e), - }; - let _ = tx.send(outcome); - })); + if let Some(context_to_spawn) = schedule_curator_context( + running_curator.is_some(), + &mut queued_curator_context, + context, + ) { + emitter.emit_trace(&format!("[curator] spawning for step {step}")); + running_curator = Some(spawn_curator_task( + context_to_spawn, + curator_tx.clone(), + config.clone(), + cancel.clone(), + )); + } else { + emitter.emit_trace(&format!("[curator] queued latest refresh from step {step}")); + } } // Budget warnings @@ -493,7 +619,11 @@ pub async fn solve( // Budget exhausted tools.cleanup(); finish_curators( - &mut curator_handles, + &mut running_curator, + &mut queued_curator_context, + &curator_tx, + config, + &cancel, &mut curator_rx, &mut messages, emitter, @@ -759,6 +889,47 @@ mod tests { assert!(complete_text.contains("Spawned test")); } + #[test] + fn test_schedule_curator_context_spawns_when_idle() { + let mut queued = None; + let spawn = schedule_curator_context(false, &mut queued, "ctx-1".to_string()); + assert_eq!(spawn, Some("ctx-1".to_string())); + assert!(queued.is_none()); + } + + #[test] + fn test_schedule_curator_context_keeps_latest_when_busy() { + let mut queued = Some("older".to_string()); + let spawn = schedule_curator_context(true, &mut queued, "newer".to_string()); + assert!(spawn.is_none()); + assert_eq!(queued, Some("newer".to_string())); + } + + #[test] + fn test_take_queued_context_if_idle_only_releases_when_idle() { + let mut queued = Some("latest".to_string()); + assert_eq!(take_queued_context_if_idle(true, &mut queued), None); + assert_eq!(queued, Some("latest".to_string())); + assert_eq!( + take_queued_context_if_idle(false, &mut queued), + Some("latest".to_string()) + ); + assert!(queued.is_none()); + } + + #[tokio::test] + async fn test_abort_curators_clears_running_and_queue() { + let mut running = Some(tokio::spawn(async { + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + })); + let mut queued = Some("queued".to_string()); + + abort_curators(&mut running, &mut queued); + + assert!(running.is_none()); + assert!(queued.is_none()); + } + #[test] fn test_estimate_tokens() { let messages = vec![ diff --git a/openplanter-desktop/crates/op-core/src/tools/defs.rs b/openplanter-desktop/crates/op-core/src/tools/defs.rs index e0fe40cf..a29ceabe 100644 --- a/openplanter-desktop/crates/op-core/src/tools/defs.rs +++ b/openplanter-desktop/crates/op-core/src/tools/defs.rs @@ -541,7 +541,7 @@ mod tests { #[test] fn test_curator_tool_defs_openai() { let tools = build_curator_tool_defs("openai"); - assert_eq!(tools.len(), 8, "curator should have exactly 8 tools"); + assert_eq!(tools.len(), 6, "curator should have exactly 6 tools"); let names: Vec = tools .iter() @@ -555,6 +555,8 @@ mod tests { assert!(names.contains(&"list_files".to_string())); assert!(names.contains(&"search_files".to_string())); assert!(names.contains(&"think".to_string())); + assert!(!names.contains(&"apply_patch".to_string())); + assert!(!names.contains(&"hashline_edit".to_string())); // Should NOT include web, shell, or bg job tools assert!(!names.contains(&"web_search".to_string())); @@ -568,7 +570,7 @@ mod tests { #[test] fn test_curator_tool_defs_anthropic() { let tools = build_curator_tool_defs("anthropic"); - assert_eq!(tools.len(), 8); + assert_eq!(tools.len(), 6); // Anthropic format: flat with input_schema assert!(tools[0].get("input_schema").is_some()); diff --git a/openplanter-desktop/crates/op-core/src/tools/mod.rs b/openplanter-desktop/crates/op-core/src/tools/mod.rs index a5e4589b..a44fc2e5 100644 --- a/openplanter-desktop/crates/op-core/src/tools/mod.rs +++ b/openplanter-desktop/crates/op-core/src/tools/mod.rs @@ -36,9 +36,16 @@ impl ToolResult { } } +#[derive(Debug, Clone)] +enum ToolScope { + FullWorkspace, + CuratorWikiOnly { allowed_root: PathBuf }, +} + /// Central dispatcher for workspace tools. pub struct WorkspaceTools { root: PathBuf, + scope: ToolScope, shell_path: String, command_timeout_sec: u64, max_shell_output_chars: usize, @@ -55,10 +62,20 @@ pub struct WorkspaceTools { bg_jobs: shell::BgJobs, } +fn clip(text: &str, max_chars: usize) -> String { + if text.len() <= max_chars { + return text.to_string(); + } + let end = text.floor_char_boundary(max_chars); + let omitted = text.len() - end; + format!("{}\n\n...[truncated {omitted} chars]...", &text[..end]) +} + impl WorkspaceTools { pub fn new(config: &AgentConfig) -> Self { Self { root: config.workspace.clone(), + scope: ToolScope::FullWorkspace, shell_path: config.shell.clone(), command_timeout_sec: config.command_timeout_sec as u64, max_shell_output_chars: config.max_shell_output_chars as usize, @@ -76,6 +93,49 @@ impl WorkspaceTools { } } + pub fn new_curator(config: &AgentConfig) -> Self { + let allowed_root = filesystem::resolve_path( + &config.workspace, + &format!("{}/wiki", config.session_root_dir), + ) + .unwrap_or_else(|_| config.workspace.join(&config.session_root_dir).join("wiki")); + Self { + root: config.workspace.clone(), + scope: ToolScope::CuratorWikiOnly { allowed_root }, + shell_path: config.shell.clone(), + command_timeout_sec: config.command_timeout_sec as u64, + max_shell_output_chars: config.max_shell_output_chars as usize, + max_file_chars: config.max_file_chars as usize, + max_files_listed: config.max_files_listed as usize, + max_search_hits: config.max_search_hits as usize, + max_observation_chars: config.max_observation_chars as usize, + web_search_provider: normalize_web_search_provider(Some(&config.web_search_provider)), + exa_api_key: config.exa_api_key.clone(), + exa_base_url: config.exa_base_url.clone(), + firecrawl_api_key: config.firecrawl_api_key.clone(), + firecrawl_base_url: config.firecrawl_base_url.clone(), + files_read: HashSet::new(), + bg_jobs: shell::BgJobs::new(), + } + } + + fn enforce_write_scope(&self, raw_path: &str) -> Result<(), ToolResult> { + match &self.scope { + ToolScope::FullWorkspace => Ok(()), + ToolScope::CuratorWikiOnly { allowed_root } => { + let resolved = + filesystem::resolve_path(&self.root, raw_path).map_err(ToolResult::error)?; + if resolved == *allowed_root || resolved.starts_with(allowed_root) { + Ok(()) + } else { + Err(ToolResult::error( + "Curator writes are restricted to .openplanter/wiki/**".to_string(), + )) + } + } + } + } + /// Execute a tool by name with JSON arguments string. /// Returns the tool result, clipped to max_observation_chars. pub async fn execute(&mut self, name: &str, args_json: &str) -> ToolResult { @@ -101,12 +161,18 @@ impl WorkspaceTools { "write_file" => { let path = args.get("path").and_then(|v| v.as_str()).unwrap_or(""); let content = args.get("content").and_then(|v| v.as_str()).unwrap_or(""); + if let Err(result) = self.enforce_write_scope(path) { + return result; + } filesystem::write_file(&self.root, path, content, &mut self.files_read) } "edit_file" => { let path = args.get("path").and_then(|v| v.as_str()).unwrap_or(""); let old_text = args.get("old_text").and_then(|v| v.as_str()).unwrap_or(""); let new_text = args.get("new_text").and_then(|v| v.as_str()).unwrap_or(""); + if let Err(result) = self.enforce_write_scope(path) { + return result; + } filesystem::edit_file(&self.root, path, old_text, new_text, &mut self.files_read) } "list_files" => { @@ -232,12 +298,8 @@ impl WorkspaceTools { // Clip observation to max_observation_chars if result.content.len() > self.max_observation_chars { - let omitted = result.content.len() - self.max_observation_chars; ToolResult { - content: format!( - "{}\n\n...[truncated {omitted} chars]...", - &result.content[..self.max_observation_chars] - ), + content: clip(&result.content, self.max_observation_chars), is_error: result.is_error, } } else { @@ -250,3 +312,102 @@ impl WorkspaceTools { self.bg_jobs.cleanup(); } } + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + fn test_config(root: &std::path::Path) -> AgentConfig { + AgentConfig::from_env(root) + } + + #[tokio::test] + async fn test_curator_scope_allows_wiki_writes() { + let tmp = tempdir().unwrap(); + let cfg = test_config(tmp.path()); + let mut tools = WorkspaceTools::new_curator(&cfg); + + let result = tools + .execute( + "write_file", + r#"{"path":".openplanter/wiki/source.md","content":"hello"}"#, + ) + .await; + + assert!(!result.is_error, "unexpected error: {}", result.content); + assert_eq!( + std::fs::read_to_string(tmp.path().join(".openplanter/wiki/source.md")).unwrap(), + "hello" + ); + } + + #[tokio::test] + async fn test_curator_scope_rejects_non_wiki_writes() { + let tmp = tempdir().unwrap(); + let cfg = test_config(tmp.path()); + let mut tools = WorkspaceTools::new_curator(&cfg); + + let result = tools + .execute("write_file", r#"{"path":"notes.md","content":"nope"}"#) + .await; + + assert!(result.is_error); + assert!(result.content.contains(".openplanter/wiki")); + assert!(!tmp.path().join("notes.md").exists()); + } + + #[tokio::test] + async fn test_curator_scope_rejects_traversal() { + let tmp = tempdir().unwrap(); + let cfg = test_config(tmp.path()); + let mut tools = WorkspaceTools::new_curator(&cfg); + + let result = tools + .execute( + "write_file", + r#"{"path":".openplanter/wiki/../../escape.md","content":"nope"}"#, + ) + .await; + + assert!(result.is_error); + assert!(!tmp.path().join("escape.md").exists()); + } + + #[tokio::test] + async fn test_full_workspace_scope_unchanged() { + let tmp = tempdir().unwrap(); + let cfg = test_config(tmp.path()); + let mut tools = WorkspaceTools::new(&cfg); + + let result = tools + .execute("write_file", r#"{"path":"notes.md","content":"allowed"}"#) + .await; + + assert!(!result.is_error, "unexpected error: {}", result.content); + assert_eq!( + std::fs::read_to_string(tmp.path().join("notes.md")).unwrap(), + "allowed" + ); + } + + #[tokio::test] + async fn test_execute_clips_observations_on_char_boundary() { + let tmp = tempdir().unwrap(); + let mut cfg = test_config(tmp.path()); + cfg.max_observation_chars = 6000; + let mut tools = WorkspaceTools::new(&cfg); + + let mut content = "a".repeat(5999); + content.push('─'); + std::fs::write(tmp.path().join("unicode.txt"), content).unwrap(); + + let result = tools + .execute("read_file", r#"{"path":"unicode.txt","hashline":false}"#) + .await; + + assert!(!result.is_error, "unexpected error: {}", result.content); + assert!(result.content.contains("[truncated")); + assert!(std::str::from_utf8(result.content.as_bytes()).is_ok()); + } +} diff --git a/openplanter-desktop/crates/op-tauri/src/bridge.rs b/openplanter-desktop/crates/op-tauri/src/bridge.rs index bc8aa326..e522dbdc 100644 --- a/openplanter-desktop/crates/op-tauri/src/bridge.rs +++ b/openplanter-desktop/crates/op-tauri/src/bridge.rs @@ -15,6 +15,47 @@ use op_core::events::{ }; use op_core::session::replay::{ReplayEntry, ReplayLogger, StepToolCallEntry}; +const MAX_STEP_MODEL_PREVIEW_CHARS: usize = 4 * 1024; +const MAX_TOOL_ARGS_CAPTURE_CHARS: usize = 16 * 1024; +const MAX_DELTA_LOG_CHARS: usize = 120; + +fn preview_text(text: &str, max_chars: usize) -> String { + if text.len() <= max_chars { + return text.to_string(); + } + + let end = text.floor_char_boundary(max_chars); + format!("{}...[truncated {} chars]", &text[..end], text.len() - end) +} + +fn append_with_cap(buffer: &mut String, text: &str, max_chars: usize, truncated: &mut bool) { + if *truncated { + return; + } + if buffer.len() >= max_chars { + *truncated = true; + return; + } + + let remaining = max_chars - buffer.len(); + let end = text.floor_char_boundary(text.len().min(remaining)); + buffer.push_str(&text[..end]); + if end < text.len() { + *truncated = true; + } +} + +fn format_model_preview(buffer: &str, truncated: bool) -> Option { + let trimmed = buffer.trim(); + if trimmed.is_empty() { + None + } else if truncated { + Some(format!("{trimmed}\n...[truncated]")) + } else { + Some(trimmed.to_string()) + } +} + pub struct TauriEmitter { handle: AppHandle, } @@ -37,10 +78,24 @@ impl SolveEmitter for TauriEmitter { } fn emit_delta(&self, event: DeltaEvent) { - eprintln!( - "[bridge] delta: kind={:?} text={:?}", - event.kind, event.text - ); + match event.kind { + DeltaKind::ToolCallArgs => eprintln!( + "[bridge] delta: kind={:?} len={} preview={:?}", + event.kind, + event.text.len(), + preview_text(&event.text, MAX_DELTA_LOG_CHARS) + ), + _ if event.text.len() > MAX_DELTA_LOG_CHARS => eprintln!( + "[bridge] delta: kind={:?} len={} preview={:?}", + event.kind, + event.text.len(), + preview_text(&event.text, MAX_DELTA_LOG_CHARS) + ), + _ => eprintln!( + "[bridge] delta: kind={:?} text={:?}", + event.kind, event.text + ), + } let _ = self.handle.emit("agent:delta", event); } @@ -93,12 +148,16 @@ pub struct LoggingEmitter { replay: Arc>, /// Accumulated streaming text for the current step (std::sync for non-async ops). streaming_buf: Mutex, + /// Whether the current step preview was truncated. + streaming_truncated: Mutex, /// Tool calls accumulated during the current step. step_tool_calls: Mutex>, /// Name of the tool currently being generated. current_tool: Mutex, /// Accumulated args JSON for the current tool. current_args_buf: Mutex, + /// Whether the current tool args buffer was truncated. + current_args_truncated: Mutex, } /// A tool call being accumulated during streaming. @@ -130,9 +189,11 @@ impl LoggingEmitter { inner, replay: Arc::new(tokio::sync::Mutex::new(replay)), streaming_buf: Mutex::new(String::new()), + streaming_truncated: Mutex::new(false), step_tool_calls: Mutex::new(Vec::new()), current_tool: Mutex::new(String::new()), current_args_buf: Mutex::new(String::new()), + current_args_truncated: Mutex::new(false), } } } @@ -146,12 +207,19 @@ impl SolveEmitter for LoggingEmitter { // Accumulate streaming data for step summary logging (sync — no I/O) match event.kind { DeltaKind::Text => { - self.streaming_buf.lock().unwrap().push_str(&event.text); + let mut truncated = self.streaming_truncated.lock().unwrap(); + append_with_cap( + &mut self.streaming_buf.lock().unwrap(), + &event.text, + MAX_STEP_MODEL_PREVIEW_CHARS, + &mut truncated, + ); } DeltaKind::ToolCallStart => { let tool_name = event.text.clone(); *self.current_tool.lock().unwrap() = tool_name.clone(); *self.current_args_buf.lock().unwrap() = String::new(); + *self.current_args_truncated.lock().unwrap() = false; self.step_tool_calls.lock().unwrap().push(PendingToolCall { name: tool_name, key_arg: String::new(), @@ -160,7 +228,13 @@ impl SolveEmitter for LoggingEmitter { } DeltaKind::ToolCallArgs => { let mut buf = self.current_args_buf.lock().unwrap(); - buf.push_str(&event.text); + let mut truncated = self.current_args_truncated.lock().unwrap(); + append_with_cap( + &mut buf, + &event.text, + MAX_TOOL_ARGS_CAPTURE_CHARS, + &mut truncated, + ); let tool_name = self.current_tool.lock().unwrap().clone(); if let Some(key_arg) = extract_key_arg(&tool_name, &buf) { let mut calls = self.step_tool_calls.lock().unwrap(); @@ -179,12 +253,7 @@ impl SolveEmitter for LoggingEmitter { // Collect accumulated data (sync) let model_preview = { let buf = self.streaming_buf.lock().unwrap(); - let trimmed = buf.trim().to_string(); - if trimmed.is_empty() { - None - } else { - Some(trimmed) - } + format_model_preview(&buf, *self.streaming_truncated.lock().unwrap()) }; let step_tools: Vec = { @@ -230,7 +299,11 @@ impl SolveEmitter for LoggingEmitter { // Reset buffers for next step self.streaming_buf.lock().unwrap().clear(); + *self.streaming_truncated.lock().unwrap() = false; self.step_tool_calls.lock().unwrap().clear(); + self.current_tool.lock().unwrap().clear(); + self.current_args_buf.lock().unwrap().clear(); + *self.current_args_truncated.lock().unwrap() = false; self.inner.emit_step(event); } @@ -418,4 +491,102 @@ mod tests { assert_eq!(entry.seq, (i + 1) as u64); } } + + #[derive(Default)] + struct CapturingEmitter { + deltas: Arc>>, + } + + impl SolveEmitter for CapturingEmitter { + fn emit_trace(&self, _: &str) {} + fn emit_delta(&self, event: DeltaEvent) { + self.deltas.lock().unwrap().push(event); + } + fn emit_step(&self, _: StepEvent) {} + fn emit_complete(&self, _: &str) {} + fn emit_error(&self, _: &str) {} + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_logging_emitter_caps_model_preview_and_preserves_deltas() { + let tmp = tempdir().unwrap(); + let replay = ReplayLogger::new(tmp.path()); + let inner = CapturingEmitter::default(); + let deltas = inner.deltas.clone(); + let emitter = LoggingEmitter::new(inner, replay); + let big_text = "x".repeat(MAX_STEP_MODEL_PREVIEW_CHARS + 256); + + emitter.emit_delta(DeltaEvent { + kind: DeltaKind::Text, + text: big_text.clone(), + }); + emitter.emit_step(StepEvent { + depth: 0, + step: 1, + tool_name: None, + tokens: Default::default(), + elapsed_ms: 1, + is_final: false, + }); + + let entries = ReplayLogger::read_all(tmp.path()).await.unwrap(); + let step = entries + .iter() + .find(|entry| entry.role == "step-summary") + .unwrap(); + let preview = step.step_model_preview.as_ref().unwrap(); + assert!(preview.contains("[truncated]")); + assert!(preview.len() < big_text.len()); + + let captured = deltas.lock().unwrap(); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0].text, big_text); + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_logging_emitter_caps_tool_args_buffer_and_keeps_key_arg() { + let tmp = tempdir().unwrap(); + let replay = ReplayLogger::new(tmp.path()); + let inner = CapturingEmitter::default(); + let deltas = inner.deltas.clone(); + let emitter = LoggingEmitter::new(inner, replay); + let filler = "x".repeat(MAX_TOOL_ARGS_CAPTURE_CHARS + 512); + + emitter.emit_delta(DeltaEvent { + kind: DeltaKind::ToolCallStart, + text: "read_file".to_string(), + }); + emitter.emit_delta(DeltaEvent { + kind: DeltaKind::ToolCallArgs, + text: "{\"path\":\"foo.md\",\"other\":\"".to_string(), + }); + emitter.emit_delta(DeltaEvent { + kind: DeltaKind::ToolCallArgs, + text: filler.clone(), + }); + + assert!(emitter.current_args_buf.lock().unwrap().len() <= MAX_TOOL_ARGS_CAPTURE_CHARS); + assert!(*emitter.current_args_truncated.lock().unwrap()); + + emitter.emit_step(StepEvent { + depth: 0, + step: 1, + tool_name: Some("read_file".into()), + tokens: Default::default(), + elapsed_ms: 1, + is_final: false, + }); + + let entries = ReplayLogger::read_all(tmp.path()).await.unwrap(); + let step = entries + .iter() + .find(|entry| entry.role == "step-summary") + .unwrap(); + let tool_calls = step.step_tool_calls.as_ref().unwrap(); + assert_eq!(tool_calls[0].key_arg, "foo.md"); + + let captured = deltas.lock().unwrap(); + assert_eq!(captured.len(), 3); + assert_eq!(captured[2].text, filler); + } } diff --git a/openplanter-desktop/crates/op-tauri/src/commands/agent.rs b/openplanter-desktop/crates/op-tauri/src/commands/agent.rs index fc40649b..201ab9df 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/agent.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/agent.rs @@ -4,6 +4,7 @@ use tokio_util::sync::CancellationToken; use crate::bridge::{LoggingEmitter, TauriEmitter}; use crate::commands::session::sessions_dir; use crate::state::AppState; +use op_core::engine::SolveEmitter; use op_core::session::replay::{ReplayEntry, ReplayLogger}; /// Start solving an objective. Result streamed via events. @@ -55,6 +56,17 @@ pub async fn solve( } let emitter = LoggingEmitter::new(TauriEmitter::new(app), replay); + let cwd = std::env::current_dir() + .map(|dir| dir.display().to_string()) + .unwrap_or_else(|_| "".to_string()); + emitter.emit_trace(&format!( + "[solve] pid={} cwd={} workspace={} session={}", + std::process::id(), + cwd, + cfg.workspace.display(), + session_id + )); + emitter.emit_trace(&format!("[startup:info] {}", state.startup_trace())); tokio::spawn(async move { let result = tokio::spawn(async move { diff --git a/openplanter-desktop/crates/op-tauri/src/commands/config.rs b/openplanter-desktop/crates/op-tauri/src/commands/config.rs index 7bfecbc6..cf3f0edb 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/config.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/config.rs @@ -105,12 +105,18 @@ fn known_models_for_provider(provider: &str) -> Vec { ("azure-foundry/Kimi-K2.5", "Kimi K2.5 (Foundry)"), ], "anthropic" => vec![ - ("anthropic-foundry/claude-opus-4-6", "Claude Opus 4.6 (Foundry)"), + ( + "anthropic-foundry/claude-opus-4-6", + "Claude Opus 4.6 (Foundry)", + ), ( "anthropic-foundry/claude-sonnet-4-6", "Claude Sonnet 4.6 (Foundry)", ), - ("anthropic-foundry/claude-haiku-4-5", "Claude Haiku 4.5 (Foundry)"), + ( + "anthropic-foundry/claude-haiku-4-5", + "Claude Haiku 4.5 (Foundry)", + ), ], "openrouter" => vec![ ("anthropic/claude-sonnet-4-5", "Claude Sonnet 4.5 (OR)"), diff --git a/openplanter-desktop/crates/op-tauri/src/commands/session.rs b/openplanter-desktop/crates/op-tauri/src/commands/session.rs index 5cc8348c..d504afbc 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/session.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/session.rs @@ -170,7 +170,8 @@ pub async fn update_session_metadata( .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; info.turn_count += 1; info.last_objective = Some(if objective.len() > 100 { - format!("{}...", &objective[..97]) + let end = objective.floor_char_boundary(97); + format!("{}...", &objective[..end]) } else { objective.to_string() }); diff --git a/openplanter-desktop/crates/op-tauri/src/main.rs b/openplanter-desktop/crates/op-tauri/src/main.rs index 20088713..edf948cf 100644 --- a/openplanter-desktop/crates/op-tauri/src/main.rs +++ b/openplanter-desktop/crates/op-tauri/src/main.rs @@ -8,9 +8,12 @@ mod state; use state::AppState; fn main() { + let state = AppState::new(); + eprintln!("[startup:info] {}", state.startup_trace()); + tauri::Builder::default() .plugin(tauri_plugin_shell::init()) - .manage(AppState::new()) + .manage(state) .invoke_handler(tauri::generate_handler![ commands::agent::solve, commands::agent::cancel, diff --git a/openplanter-desktop/crates/op-tauri/src/state.rs b/openplanter-desktop/crates/op-tauri/src/state.rs index 3109c5c0..6c9fc8ff 100644 --- a/openplanter-desktop/crates/op-tauri/src/state.rs +++ b/openplanter-desktop/crates/op-tauri/src/state.rs @@ -6,10 +6,36 @@ use op_core::credentials::{ }; use op_core::settings::{PersistentSettings, SettingsStore}; use std::env; +use std::fs; +use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; +const WORKSPACE_ENV_KEY: &str = "OPENPLANTER_WORKSPACE"; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum WorkspaceSource { + EnvOverride, + GitRoot, + CurrentDir, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ResolvedWorkspace { + path: PathBuf, + source: WorkspaceSource, + invalid_override: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct LegacyMigrationReport { + source: Option, + copied_files: u64, + skipped_existing: u64, + errors: Vec, +} + /// Merge credentials into an AgentConfig. /// Priority: existing config value > env_creds > file_creds. pub fn merge_credentials_into_config( @@ -87,16 +113,232 @@ fn apply_settings_to_config(cfg: &mut AgentConfig, settings: &PersistentSettings } } +fn canonicalize_or_self(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) +} + +fn find_git_root(start: &Path) -> Option { + let mut current = Some(canonicalize_or_self(start)); + while let Some(dir) = current { + if dir.join(".git").exists() { + return Some(dir); + } + current = dir.parent().map(|parent| parent.to_path_buf()); + } + None +} + +fn resolve_startup_workspace_from( + current_dir: &Path, + env_override: Option<&str>, +) -> ResolvedWorkspace { + let mut invalid_override = None; + + if let Some(raw_override) = env_override + .map(str::trim) + .filter(|value| !value.is_empty()) + { + let candidate = PathBuf::from(raw_override); + if candidate.exists() { + return ResolvedWorkspace { + path: canonicalize_or_self(&candidate), + source: WorkspaceSource::EnvOverride, + invalid_override: None, + }; + } + invalid_override = Some(raw_override.to_string()); + } + + if let Some(git_root) = find_git_root(current_dir) { + return ResolvedWorkspace { + path: git_root, + source: WorkspaceSource::GitRoot, + invalid_override, + }; + } + + ResolvedWorkspace { + path: canonicalize_or_self(current_dir), + source: WorkspaceSource::CurrentDir, + invalid_override, + } +} + +fn resolve_desktop_workspace() -> ResolvedWorkspace { + let current_dir = env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + let env_override = env::var(WORKSPACE_ENV_KEY).ok(); + resolve_startup_workspace_from(¤t_dir, env_override.as_deref()) +} + +fn legacy_state_candidates(workspace: &Path, session_root_dir: &str) -> Vec { + vec![ + workspace + .join("openplanter-desktop") + .join("crates") + .join("op-tauri") + .join(session_root_dir), + workspace + .join("crates") + .join("op-tauri") + .join(session_root_dir), + ] +} + +fn copy_missing_file(src: &Path, dst: &Path, report: &mut LegacyMigrationReport) { + if !src.exists() || !src.is_file() { + return; + } + + if dst.exists() { + report.skipped_existing += 1; + return; + } + + if let Some(parent) = dst.parent() { + if let Err(err) = fs::create_dir_all(parent) { + report + .errors + .push(format!("failed to create {}: {err}", parent.display())); + return; + } + } + + match fs::copy(src, dst) { + Ok(_) => report.copied_files += 1, + Err(err) => report.errors.push(format!( + "failed to copy {} -> {}: {err}", + src.display(), + dst.display() + )), + } +} + +fn copy_missing_tree(src: &Path, dst: &Path, report: &mut LegacyMigrationReport) { + if !src.exists() { + return; + } + if src.is_file() { + copy_missing_file(src, dst, report); + return; + } + if !src.is_dir() { + return; + } + + if let Err(err) = fs::create_dir_all(dst) { + report + .errors + .push(format!("failed to create {}: {err}", dst.display())); + return; + } + + let entries = match fs::read_dir(src) { + Ok(entries) => entries, + Err(err) => { + report + .errors + .push(format!("failed to read {}: {err}", src.display())); + return; + } + }; + + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(err) => { + report.errors.push(format!( + "failed to read entry under {}: {err}", + src.display() + )); + continue; + } + }; + let src_path = entry.path(); + let dst_path = dst.join(entry.file_name()); + if src_path.is_dir() { + copy_missing_tree(&src_path, &dst_path, report); + } else { + copy_missing_file(&src_path, &dst_path, report); + } + } +} + +fn migrate_legacy_desktop_state(workspace: &Path, session_root_dir: &str) -> LegacyMigrationReport { + let mut report = LegacyMigrationReport::default(); + let destination_root = workspace.join(session_root_dir); + + for candidate in legacy_state_candidates(workspace, session_root_dir) { + if !candidate.exists() { + continue; + } + + report.source = Some(candidate.clone()); + copy_missing_file( + &candidate.join("settings.json"), + &destination_root.join("settings.json"), + &mut report, + ); + copy_missing_file( + &candidate.join("credentials.json"), + &destination_root.join("credentials.json"), + &mut report, + ); + copy_missing_tree( + &candidate.join("sessions"), + &destination_root.join("sessions"), + &mut report, + ); + break; + } + + report +} + +fn format_startup_trace( + current_dir: &Path, + resolved: &ResolvedWorkspace, + migration: &LegacyMigrationReport, +) -> String { + let source = match resolved.source { + WorkspaceSource::EnvOverride => "env_override", + WorkspaceSource::GitRoot => "git_root", + WorkspaceSource::CurrentDir => "current_dir", + }; + let invalid_override = resolved.invalid_override.as_deref().unwrap_or(""); + let migration_source = migration + .source + .as_ref() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| "".to_string()); + + format!( + "pid={} cwd={} workspace={} source={} invalid_override={} migration_source={} migration_copied={} migration_skipped={} migration_errors={}", + std::process::id(), + current_dir.display(), + resolved.path.display(), + source, + invalid_override, + migration_source, + migration.copied_files, + migration.skipped_existing, + migration.errors.len() + ) +} + /// Application state shared across Tauri commands. pub struct AppState { pub config: Arc>, pub session_id: Arc>>, pub cancel_token: Arc>, + startup_trace: String, } impl AppState { pub fn new() -> Self { - let mut cfg = AgentConfig::from_env("."); + let current_dir = env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + let resolved_workspace = resolve_desktop_workspace(); + let mut cfg = AgentConfig::from_env(&resolved_workspace.path); + let migration = migrate_legacy_desktop_state(&cfg.workspace, &cfg.session_root_dir); // Load .env files and merge credentials into config let env_creds = credentials_from_env(); @@ -119,14 +361,20 @@ impl AppState { config: Arc::new(Mutex::new(cfg)), session_id: Arc::new(Mutex::new(None)), cancel_token: Arc::new(Mutex::new(CancellationToken::new())), + startup_trace: format_startup_trace(¤t_dir, &resolved_workspace, &migration), } } + + pub fn startup_trace(&self) -> &str { + &self.startup_trace + } } #[cfg(test)] mod tests { use super::*; use std::env; + use tempfile::tempdir; fn empty_cfg() -> AgentConfig { let mut cfg = AgentConfig::from_env("/nonexistent"); @@ -247,4 +495,141 @@ mod tests { } } } + + #[test] + fn test_resolve_startup_workspace_prefers_env_override() { + let temp = tempdir().unwrap(); + let repo = temp.path().join("repo"); + fs::create_dir_all(repo.join(".git")).unwrap(); + let override_dir = temp.path().join("override"); + fs::create_dir_all(&override_dir).unwrap(); + + let resolved = resolve_startup_workspace_from(&repo, Some(override_dir.to_str().unwrap())); + + assert_eq!(resolved.source, WorkspaceSource::EnvOverride); + assert_eq!(resolved.path, canonicalize_or_self(&override_dir)); + assert!(resolved.invalid_override.is_none()); + } + + #[test] + fn test_resolve_startup_workspace_finds_git_root_from_nested_dir() { + let temp = tempdir().unwrap(); + let repo = temp.path().join("repo"); + fs::create_dir_all(repo.join(".git")).unwrap(); + let nested = repo + .join("openplanter-desktop") + .join("crates") + .join("op-tauri"); + fs::create_dir_all(&nested).unwrap(); + + let resolved = resolve_startup_workspace_from(&nested, None); + + assert_eq!(resolved.source, WorkspaceSource::GitRoot); + assert_eq!(resolved.path, canonicalize_or_self(&repo)); + } + + #[test] + fn test_resolve_startup_workspace_falls_back_to_current_dir() { + let temp = tempdir().unwrap(); + + let resolved = + resolve_startup_workspace_from(temp.path(), Some("/definitely/missing/path")); + + assert_eq!(resolved.source, WorkspaceSource::CurrentDir); + assert_eq!(resolved.path, canonicalize_or_self(temp.path())); + assert_eq!( + resolved.invalid_override, + Some("/definitely/missing/path".to_string()) + ); + } + + #[test] + fn test_migrate_legacy_desktop_state_copies_missing_and_preserves_existing() { + let temp = tempdir().unwrap(); + let workspace = temp.path().join("repo"); + let legacy = workspace + .join("openplanter-desktop") + .join("crates") + .join("op-tauri") + .join(".openplanter"); + let destination = workspace.join(".openplanter"); + + fs::create_dir_all(legacy.join("sessions").join("session-a")).unwrap(); + fs::write(legacy.join("settings.json"), "{\"legacy\":true}").unwrap(); + fs::write(legacy.join("credentials.json"), "{\"key\":\"legacy\"}").unwrap(); + fs::write( + legacy + .join("sessions") + .join("session-a") + .join("replay.jsonl"), + "legacy-session", + ) + .unwrap(); + + fs::create_dir_all(&destination).unwrap(); + fs::write(destination.join("settings.json"), "{\"keep\":true}").unwrap(); + + let report = migrate_legacy_desktop_state(&workspace, ".openplanter"); + + assert_eq!(report.source, Some(legacy)); + assert_eq!( + fs::read_to_string(destination.join("settings.json")).unwrap(), + "{\"keep\":true}" + ); + assert_eq!( + fs::read_to_string(destination.join("credentials.json")).unwrap(), + "{\"key\":\"legacy\"}" + ); + assert_eq!( + fs::read_to_string( + destination + .join("sessions") + .join("session-a") + .join("replay.jsonl") + ) + .unwrap(), + "legacy-session" + ); + assert_eq!(report.copied_files, 2); + assert_eq!(report.skipped_existing, 1); + assert!(report.errors.is_empty()); + } + + #[test] + fn test_startup_trace_uses_informational_migration_labels() { + let temp = tempdir().unwrap(); + let workspace = temp.path().join("repo"); + let current_dir = workspace + .join("openplanter-desktop") + .join("crates") + .join("op-tauri"); + fs::create_dir_all(workspace.join(".git")).unwrap(); + fs::create_dir_all(¤t_dir).unwrap(); + + let resolved = resolve_startup_workspace_from(¤t_dir, None); + let migration = LegacyMigrationReport { + source: Some(workspace.join("legacy-state")), + copied_files: 2, + skipped_existing: 3, + errors: vec!["copy failed".to_string()], + }; + + let trace = format_startup_trace(¤t_dir, &resolved, &migration); + + assert!(trace.contains("pid=")); + assert!(trace.contains(&format!("cwd={}", current_dir.display()))); + assert!(trace.contains(&format!("workspace={}", resolved.path.display()))); + assert!(trace.contains("source=git_root")); + assert!(trace.contains("invalid_override=")); + assert!(trace.contains(&format!( + "migration_source={}", + workspace.join("legacy-state").display() + ))); + assert!(trace.contains("migration_copied=2")); + assert!(trace.contains("migration_skipped=3")); + assert!(trace.contains("migration_errors=1")); + assert!(!trace.contains(" copied=")); + assert!(!trace.contains(" skipped=")); + assert!(!trace.contains(" errors=")); + } } diff --git a/openplanter-desktop/frontend/e2e/streaming.spec.ts b/openplanter-desktop/frontend/e2e/streaming.spec.ts index 9afe59a4..e328249f 100644 --- a/openplanter-desktop/frontend/e2e/streaming.spec.ts +++ b/openplanter-desktop/frontend/e2e/streaming.spec.ts @@ -113,6 +113,27 @@ async function sendStep( ); } +async function expectGraphPaneVisibleAndStable(page: Page) { + const graphPane = page.locator(".graph-pane"); + await expect(graphPane).toBeVisible(); + + const box = await graphPane.boundingBox(); + expect(box).not.toBeNull(); + + const viewport = page.viewportSize(); + expect(viewport).not.toBeNull(); + + expect(box!.width).toBeGreaterThan(150); + expect(box!.x).toBeGreaterThanOrEqual(0); + expect(box!.x + box!.width).toBeLessThanOrEqual(viewport!.width + 1); + + const appMetrics = await page.locator("#app").evaluate((el) => ({ + clientWidth: el.clientWidth, + scrollWidth: el.scrollWidth, + })); + expect(appMetrics.scrollWidth).toBeLessThanOrEqual(appMetrics.clientWidth + 1); +} + test.describe("Streaming Display", () => { test.beforeEach(async ({ page }) => { await injectTauriMocks(page); @@ -293,4 +314,35 @@ test.describe("Streaming Display", () => { path: "e2e/screenshots/35-activity-elapsed.png", }); }); + + test("long streamed preview text does not push graph pane off-screen", async ({ + page, + }) => { + const longPreview = `Investigating_${"CentralFlorida".repeat(120)}`; + + await sendDelta(page, "thinking", longPreview); + await expect(page.locator(".activity-preview")).toContainText("Investigating_"); + + await expectGraphPaneVisibleAndStable(page); + }); + + test("long tool call rows do not push graph pane off-screen", async ({ + page, + }) => { + const longCommand = `find_${"central_florida_workspace".repeat(80)}`; + + await sendDelta(page, "tool_call_start", "run_shell"); + await sendDelta( + page, + "tool_call_args", + JSON.stringify({ command: longCommand }) + ); + await sendStep(page, 1, 6400, 1200); + + const toolLine = page.locator(".step-tool-line").first(); + await expect(toolLine).toBeVisible(); + await expect(toolLine).toContainText("run_shell"); + + await expectGraphPaneVisibleAndStable(page); + }); }); diff --git a/openplanter-desktop/frontend/src/styles/main.css b/openplanter-desktop/frontend/src/styles/main.css index 8f3fa2b6..eae58ebc 100644 --- a/openplanter-desktop/frontend/src/styles/main.css +++ b/openplanter-desktop/frontend/src/styles/main.css @@ -19,7 +19,7 @@ html, body { #app { display: grid; grid-template-rows: var(--statusbar-height) 1fr; - grid-template-columns: var(--sidebar-width) 3fr 2fr; + grid-template-columns: var(--sidebar-width) minmax(0, 3fr) minmax(0, 2fr); height: 100vh; gap: 1px; background: var(--border); @@ -151,11 +151,13 @@ html, body { font-family: var(--font-mono); font-size: 13px; line-height: 1.5; + min-width: 0; min-height: 0; } .chat-messages { flex: 1; + min-width: 0; min-height: 0; overflow-y: auto; padding: 12px 16px; @@ -166,9 +168,11 @@ html, body { .message { width: 100%; + min-width: 0; padding: 2px 0; white-space: pre-wrap; word-break: break-word; + overflow-wrap: anywhere; } .message.user { @@ -271,6 +275,8 @@ html, body { border-radius: var(--radius); padding: 8px 12px; margin: 6px 0; + max-width: 100%; + min-width: 0; overflow-x: auto; } @@ -332,6 +338,7 @@ html, body { font-size: 12px; color: var(--text-secondary); white-space: nowrap; + min-width: 0; overflow: hidden; text-overflow: ellipsis; } @@ -339,10 +346,15 @@ html, body { .tool-call-block .tool-fn { color: var(--warning); font-weight: 600; + flex-shrink: 0; } .tool-call-block .tool-arg { + flex: 1 1 auto; + min-width: 0; color: var(--text-muted); + overflow: hidden; + text-overflow: ellipsis; } .tool-result-wrapper { @@ -357,6 +369,7 @@ html, body { color: var(--text-muted); white-space: pre-wrap; word-break: break-word; + overflow-wrap: anywhere; max-height: 6em; overflow: hidden; } @@ -401,6 +414,7 @@ html, body { .activity-indicator { padding: 4px 0; margin: 2px 0; + min-width: 0; } .activity-row { @@ -408,6 +422,7 @@ html, body { align-items: center; gap: 8px; font-size: 12px; + min-width: 0; } .activity-icon { @@ -473,6 +488,7 @@ html, body { padding-left: 20px; white-space: pre-wrap; word-break: break-word; + overflow-wrap: anywhere; max-height: 3.6em; overflow: hidden; line-height: 1.2; @@ -501,6 +517,7 @@ html, body { padding: 2px 0; white-space: pre-wrap; word-break: break-word; + overflow-wrap: anywhere; max-height: 3em; overflow: hidden; } @@ -512,8 +529,11 @@ html, body { } .step-tool-line { + display: flex; + align-items: baseline; color: var(--text-secondary); white-space: nowrap; + min-width: 0; overflow: hidden; text-overflow: ellipsis; line-height: 1.7; @@ -522,15 +542,21 @@ html, body { .step-tool-line .tool-fn { color: var(--warning); font-weight: 600; + flex-shrink: 0; } .step-tool-line .tool-arg { + flex: 1 1 auto; + min-width: 0; color: var(--text-muted); + overflow: hidden; + text-overflow: ellipsis; } .step-tool-line .tool-elapsed { color: var(--text-muted); font-size: 11px; + flex-shrink: 0; } /* Graph pane */ @@ -542,6 +568,7 @@ html, body { overflow: hidden; display: flex; flex-direction: column; + min-width: 0; } /* Graph toolbar */ diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..b8aac500 --- /dev/null +++ b/uv.lock @@ -0,0 +1,204 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.11'", + "python_full_version < '3.11'", +] + +[[package]] +name = "linkify-it-py" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "uc-micro-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/c9/06ea13676ef354f0af6169587ae292d3e2406e212876a413bf9eece4eb23/linkify_it_py-2.1.0.tar.gz", hash = "sha256:43360231720999c10e9328dc3691160e27a718e280673d444c38d7d3aaa3b98b", size = 29158, upload-time = "2026-03-01T07:48:47.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/de/88b3be5c31b22333b3ca2f6ff1de4e863d8fe45aaea7485f591970ec1d3e/linkify_it_py-2.1.0-py3-none-any.whl", hash = "sha256:0d252c1594ecba2ecedc444053db5d3a9b7ec1b0dd929c8f1d74dce89f86c05e", size = 19878, upload-time = "2026-03-01T07:48:46.098Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[package.optional-dependencies] +linkify = [ + { name = "linkify-it-py" }, +] + +[[package]] +name = "mdit-py-plugins" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/86/dd6e5db36df29e76c7a7699123569a4a18c1623ce68d826ed96c62643cae/mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f", size = 57205, upload-time = "2025-08-11T07:25:47.597Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "networkx" +version = "3.4.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263, upload-time = "2024-10-21T12:39:36.247Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "openplanter-agent" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "prompt-toolkit" }, + { name = "pyfiglet" }, + { name = "rich" }, +] + +[package.optional-dependencies] +textual = [ + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "textual" }, +] + +[package.metadata] +requires-dist = [ + { name = "networkx", marker = "extra == 'textual'", specifier = ">=3.2" }, + { name = "prompt-toolkit", specifier = ">=3.0" }, + { name = "pyfiglet", specifier = ">=1.0" }, + { name = "rich", specifier = ">=13.0" }, + { name = "textual", marker = "extra == 'textual'", specifier = ">=0.89" }, +] +provides-extras = ["textual"] + +[[package]] +name = "platformdirs" +version = "4.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/04/fea538adf7dbbd6d186f551d595961e564a3b6715bdf276b477460858672/platformdirs-4.9.2.tar.gz", hash = "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291", size = 28394, upload-time = "2026-02-16T03:56:10.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/31/05e764397056194206169869b50cf2fee4dbbbc71b344705b9c0d878d4d8/platformdirs-4.9.2-py3-none-any.whl", hash = "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd", size = 21168, upload-time = "2026-02-16T03:56:08.891Z" }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, +] + +[[package]] +name = "pyfiglet" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/e3/0a86276ad2c383ce08d76110a8eec2fe22e7051c4b8ba3fa163a0b08c428/pyfiglet-1.0.4.tar.gz", hash = "sha256:db9c9940ed1bf3048deff534ed52ff2dafbbc2cd7610b17bb5eca1df6d4278ef", size = 1560615, upload-time = "2025-08-15T18:32:47.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/5c/fe9f95abd5eaedfa69f31e450f7e2768bef121dbdf25bcddee2cd3087a16/pyfiglet-1.0.4-py3-none-any.whl", hash = "sha256:65b57b7a8e1dff8a67dc8e940a117238661d5e14c3e49121032bd404d9b2b39f", size = 1806118, upload-time = "2025-08-15T18:32:45.556Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "rich" +version = "14.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/c6/f3b320c27991c46f43ee9d856302c70dc2d0fb2dba4842ff739d5f46b393/rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b", size = 230582, upload-time = "2026-02-19T17:23:12.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, +] + +[[package]] +name = "textual" +version = "8.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py", extra = ["linkify"] }, + { name = "mdit-py-plugins" }, + { name = "platformdirs" }, + { name = "pygments" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/08/c6bcb1e3c4c9528ec9049f4ac685afdafc72866664270f0deb416ccbba2a/textual-8.0.2.tar.gz", hash = "sha256:7b342f3ee9a5f2f1bd42d7b598cae00ff1275da68536769510db4b7fe8cabf5d", size = 6099270, upload-time = "2026-03-03T20:23:46.858Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/bc/0cd17f96f00b6e8bfbca64c574088c85f3c614912b3030f313752e30a099/textual-8.0.2-py3-none-any.whl", hash = "sha256:4ceadbe0e8a30eb80f9995000f4d031f711420a31b02da38f3482957b7c50ce4", size = 719174, upload-time = "2026-03-03T20:23:50.46Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "uc-micro-py" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/78/67/9a363818028526e2d4579334460df777115bdec1bb77c08f9db88f6389f2/uc_micro_py-2.0.0.tar.gz", hash = "sha256:c53691e495c8db60e16ffc4861a35469b0ba0821fe409a8a7a0a71864d33a811", size = 6611, upload-time = "2026-03-01T06:31:27.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/73/d21edf5b204d1467e06500080a50f79d49ef2b997c79123a536d4a17d97c/uc_micro_py-2.0.0-py3-none-any.whl", hash = "sha256:3603a3859af53e5a39bc7677713c78ea6589ff188d70f4fee165db88e22b242c", size = 6383, upload-time = "2026-03-01T06:31:26.257Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, +] From 6e25209ddc47da7e9b7b31e39c6108f3aad8e16b Mon Sep 17 00:00:00 2001 From: Drake Date: Thu, 12 Mar 2026 10:53:31 -0400 Subject: [PATCH 04/58] Add Brave search option --- README.md | 4 +- agent/__main__.py | 13 +- agent/builder.py | 2 + agent/config.py | 7 +- agent/credentials.py | 11 + agent/tool_defs.py | 2 +- agent/tools.py | 208 +++++++++- .../crates/op-core/src/config.rs | 23 +- .../crates/op-core/src/credentials.rs | 15 +- .../crates/op-core/src/tools/defs.rs | 4 +- .../crates/op-core/src/tools/mod.rs | 8 + .../crates/op-core/src/tools/web.rs | 385 +++++++++++++++++- .../crates/op-tauri/src/commands/config.rs | 14 +- .../crates/op-tauri/src/state.rs | 10 +- .../frontend/src/api/invoke.test.ts | 2 + .../src/commands/completionRegistry.test.ts | 4 +- .../src/commands/completionRegistry.ts | 1 + .../frontend/src/commands/slash.ts | 2 +- .../frontend/src/commands/webSearch.test.ts | 16 +- .../frontend/src/commands/webSearch.ts | 2 +- .../frontend/src/components/App.test.ts | 4 +- .../frontend/src/components/App.ts | 2 +- tests/test_coverage_gaps.py | 9 +- tests/test_credentials.py | 3 + tests/test_tools.py | 56 +++ tests/test_tools_complex.py | 21 + 26 files changed, 791 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 53a01029..bfede85c 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ export OPENPLANTER_RATE_LIMIT_RETRY_AFTER_CAP_SEC=120.0 export OPENPLANTER_ZAI_STREAM_MAX_RETRIES=10 ``` -Additional service keys: `EXA_API_KEY` (web search), `VOYAGE_API_KEY` (embeddings). +Additional service keys: `EXA_API_KEY`, `FIRECRAWL_API_KEY`, `BRAVE_API_KEY` (web search), `VOYAGE_API_KEY` (embeddings). All keys can also be set with an `OPENPLANTER_` prefix (e.g. `OPENPLANTER_OPENAI_API_KEY`), via `.env` files in the workspace, or via CLI flags. @@ -160,7 +160,7 @@ The agent has access to 19 tools, organized around its investigation workflow: **Shell execution** — `run_shell`, `run_shell_bg`, `check_shell_bg`, `kill_shell_bg` — run analysis scripts, data pipelines, and validation checks. -**Web** — `web_search` (Exa), `fetch_url` — pull public records, verify entities, and retrieve supplementary data. +**Web** — `web_search` (Exa, Firecrawl, or Brave), `fetch_url` — pull public records, verify entities, and retrieve supplementary data. **Planning & delegation** — `think`, `subtask`, `execute`, `list_artifacts`, `read_artifact` — decompose investigations into focused sub-tasks, each with acceptance criteria and independent verification. diff --git a/agent/__main__.py b/agent/__main__.py index f3c29eca..728397a5 100644 --- a/agent/__main__.py +++ b/agent/__main__.py @@ -100,9 +100,10 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument("--exa-api-key", help="Exa API key override.") parser.add_argument("--firecrawl-api-key", help="Firecrawl API key override.") + parser.add_argument("--brave-api-key", help="Brave Search API key override.") parser.add_argument( "--web-search-provider", - choices=["exa", "firecrawl"], + choices=["exa", "firecrawl", "brave"], help="Web search backend provider.", ) parser.add_argument("--voyage-api-key", help="Voyage API key override.") @@ -241,6 +242,7 @@ def _load_credentials( zai_api_key=user_creds.zai_api_key, exa_api_key=user_creds.exa_api_key, firecrawl_api_key=user_creds.firecrawl_api_key, + brave_api_key=user_creds.brave_api_key, voyage_api_key=user_creds.voyage_api_key, ) @@ -260,6 +262,8 @@ def _load_credentials( creds.exa_api_key = stored.exa_api_key if stored.firecrawl_api_key: creds.firecrawl_api_key = stored.firecrawl_api_key + if stored.brave_api_key: + creds.brave_api_key = stored.brave_api_key if stored.voyage_api_key: creds.voyage_api_key = stored.voyage_api_key @@ -278,6 +282,8 @@ def _load_credentials( creds.exa_api_key = env_creds.exa_api_key if env_creds.firecrawl_api_key: creds.firecrawl_api_key = env_creds.firecrawl_api_key + if env_creds.brave_api_key: + creds.brave_api_key = env_creds.brave_api_key if env_creds.voyage_api_key: creds.voyage_api_key = env_creds.voyage_api_key @@ -301,6 +307,8 @@ def _load_credentials( creds.exa_api_key = args.exa_api_key.strip() or creds.exa_api_key if args.firecrawl_api_key: creds.firecrawl_api_key = args.firecrawl_api_key.strip() or creds.firecrawl_api_key + if args.brave_api_key: + creds.brave_api_key = args.brave_api_key.strip() or creds.brave_api_key if args.voyage_api_key: creds.voyage_api_key = args.voyage_api_key.strip() or creds.voyage_api_key @@ -349,6 +357,7 @@ def _apply_runtime_overrides(cfg: AgentConfig, args: argparse.Namespace, creds: cfg.zai_api_key = creds.zai_api_key cfg.exa_api_key = creds.exa_api_key cfg.firecrawl_api_key = creds.firecrawl_api_key + cfg.brave_api_key = creds.brave_api_key cfg.voyage_api_key = creds.voyage_api_key cfg.api_key = cfg.openai_api_key @@ -386,7 +395,7 @@ def _apply_runtime_overrides(cfg: AgentConfig, args: argparse.Namespace, creds: cfg.model = args.model if args.web_search_provider: cfg.web_search_provider = args.web_search_provider - if cfg.web_search_provider not in {"exa", "firecrawl"}: + if cfg.web_search_provider not in {"exa", "firecrawl", "brave"}: cfg.web_search_provider = "exa" if args.reasoning_effort: cfg.reasoning_effort = None if args.reasoning_effort == "none" else args.reasoning_effort diff --git a/agent/builder.py b/agent/builder.py index 1a07bf56..89671221 100644 --- a/agent/builder.py +++ b/agent/builder.py @@ -235,6 +235,8 @@ def build_engine(cfg: AgentConfig) -> RLMEngine: exa_base_url=cfg.exa_base_url, firecrawl_api_key=cfg.firecrawl_api_key, firecrawl_base_url=cfg.firecrawl_base_url, + brave_api_key=cfg.brave_api_key, + brave_base_url=cfg.brave_base_url, ) try: diff --git a/agent/config.py b/agent/config.py index 527c0d2c..50290176 100644 --- a/agent/config.py +++ b/agent/config.py @@ -104,6 +104,7 @@ class AgentConfig: ollama_base_url: str = "http://localhost:11434/v1" exa_base_url: str = "https://api.exa.ai" firecrawl_base_url: str = "https://api.firecrawl.dev/v1" + brave_base_url: str = "https://api.search.brave.com/res/v1" openai_api_key: str | None = None anthropic_api_key: str | None = None openrouter_api_key: str | None = None @@ -111,6 +112,7 @@ class AgentConfig: zai_api_key: str | None = None exa_api_key: str | None = None firecrawl_api_key: str | None = None + brave_api_key: str | None = None web_search_provider: str = "exa" voyage_api_key: str | None = None max_depth: int = 4 @@ -157,6 +159,7 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": zai_api_key = os.getenv("OPENPLANTER_ZAI_API_KEY") or os.getenv("ZAI_API_KEY") exa_api_key = os.getenv("OPENPLANTER_EXA_API_KEY") or os.getenv("EXA_API_KEY") firecrawl_api_key = os.getenv("OPENPLANTER_FIRECRAWL_API_KEY") or os.getenv("FIRECRAWL_API_KEY") + brave_api_key = os.getenv("OPENPLANTER_BRAVE_API_KEY") or os.getenv("BRAVE_API_KEY") voyage_api_key = os.getenv("OPENPLANTER_VOYAGE_API_KEY") or os.getenv("VOYAGE_API_KEY") openai_base_url = os.getenv("OPENPLANTER_OPENAI_BASE_URL") or os.getenv( "OPENPLANTER_BASE_URL", @@ -181,7 +184,7 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": ) ) web_search_provider = (os.getenv("OPENPLANTER_WEB_SEARCH_PROVIDER", "exa").strip().lower() or "exa") - if web_search_provider not in {"exa", "firecrawl"}: + if web_search_provider not in {"exa", "firecrawl", "brave"}: web_search_provider = "exa" return cls( workspace=ws, @@ -201,6 +204,7 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": ollama_base_url=os.getenv("OPENPLANTER_OLLAMA_BASE_URL", "http://localhost:11434/v1"), exa_base_url=os.getenv("OPENPLANTER_EXA_BASE_URL", "https://api.exa.ai"), firecrawl_base_url=os.getenv("OPENPLANTER_FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v1"), + brave_base_url=os.getenv("OPENPLANTER_BRAVE_BASE_URL", "https://api.search.brave.com/res/v1"), openai_api_key=openai_api_key, anthropic_api_key=anthropic_api_key, openrouter_api_key=openrouter_api_key, @@ -208,6 +212,7 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": zai_api_key=zai_api_key, exa_api_key=exa_api_key, firecrawl_api_key=firecrawl_api_key, + brave_api_key=brave_api_key, web_search_provider=web_search_provider, voyage_api_key=voyage_api_key, max_depth=int(os.getenv("OPENPLANTER_MAX_DEPTH", "4")), diff --git a/agent/credentials.py b/agent/credentials.py index 275a8106..a714f82d 100644 --- a/agent/credentials.py +++ b/agent/credentials.py @@ -18,6 +18,7 @@ class CredentialBundle: zai_api_key: str | None = None exa_api_key: str | None = None firecrawl_api_key: str | None = None + brave_api_key: str | None = None voyage_api_key: str | None = None def has_any(self) -> bool: @@ -29,6 +30,7 @@ def has_any(self) -> bool: or (self.zai_api_key and self.zai_api_key.strip()) or (self.exa_api_key and self.exa_api_key.strip()) or (self.firecrawl_api_key and self.firecrawl_api_key.strip()) + or (self.brave_api_key and self.brave_api_key.strip()) or (self.voyage_api_key and self.voyage_api_key.strip()) ) @@ -47,6 +49,8 @@ def merge_missing(self, other: "CredentialBundle") -> None: self.exa_api_key = other.exa_api_key if not self.firecrawl_api_key and other.firecrawl_api_key: self.firecrawl_api_key = other.firecrawl_api_key + if not self.brave_api_key and other.brave_api_key: + self.brave_api_key = other.brave_api_key if not self.voyage_api_key and other.voyage_api_key: self.voyage_api_key = other.voyage_api_key @@ -66,6 +70,8 @@ def to_json(self) -> dict[str, str]: out["exa_api_key"] = self.exa_api_key if self.firecrawl_api_key: out["firecrawl_api_key"] = self.firecrawl_api_key + if self.brave_api_key: + out["brave_api_key"] = self.brave_api_key if self.voyage_api_key: out["voyage_api_key"] = self.voyage_api_key return out @@ -82,6 +88,7 @@ def from_json(cls, payload: dict[str, str] | None) -> "CredentialBundle": zai_api_key=(payload.get("zai_api_key") or "").strip() or None, exa_api_key=(payload.get("exa_api_key") or "").strip() or None, firecrawl_api_key=(payload.get("firecrawl_api_key") or "").strip() or None, + brave_api_key=(payload.get("brave_api_key") or "").strip() or None, voyage_api_key=(payload.get("voyage_api_key") or "").strip() or None, ) @@ -127,6 +134,7 @@ def parse_env_file(path: Path) -> CredentialBundle: exa_api_key=(env.get("EXA_API_KEY") or env.get("OPENPLANTER_EXA_API_KEY") or "").strip() or None, firecrawl_api_key=(env.get("FIRECRAWL_API_KEY") or env.get("OPENPLANTER_FIRECRAWL_API_KEY") or "").strip() or None, + brave_api_key=(env.get("BRAVE_API_KEY") or env.get("OPENPLANTER_BRAVE_API_KEY") or "").strip() or None, voyage_api_key=(env.get("VOYAGE_API_KEY") or env.get("OPENPLANTER_VOYAGE_API_KEY") or "").strip() or None, ) @@ -160,6 +168,7 @@ def credentials_from_env() -> CredentialBundle: os.getenv("OPENPLANTER_FIRECRAWL_API_KEY") or os.getenv("FIRECRAWL_API_KEY") or "" ).strip() or None, + brave_api_key=(os.getenv("OPENPLANTER_BRAVE_API_KEY") or os.getenv("BRAVE_API_KEY") or "").strip() or None, voyage_api_key=(os.getenv("OPENPLANTER_VOYAGE_API_KEY") or os.getenv("VOYAGE_API_KEY") or "").strip() or None, ) @@ -257,6 +266,7 @@ def prompt_for_credentials( zai_api_key=existing.zai_api_key, exa_api_key=existing.exa_api_key, firecrawl_api_key=existing.firecrawl_api_key, + brave_api_key=existing.brave_api_key, voyage_api_key=existing.voyage_api_key, ) @@ -292,6 +302,7 @@ def _ask(label: str, existing_value: str | None) -> str | None: current.zai_api_key = _ask("Z.AI", current.zai_api_key) current.exa_api_key = _ask("Exa", current.exa_api_key) current.firecrawl_api_key = _ask("Firecrawl", current.firecrawl_api_key) + current.brave_api_key = _ask("Brave", current.brave_api_key) current.voyage_api_key = _ask("Voyage", current.voyage_api_key) if not force and current.has_any() and not existing.has_any(): changed = True diff --git a/agent/tool_defs.py b/agent/tool_defs.py index 79bdb496..63d4765f 100644 --- a/agent/tool_defs.py +++ b/agent/tool_defs.py @@ -63,7 +63,7 @@ }, { "name": "web_search", - "description": "Search the web using the configured provider (Exa or Firecrawl). Returns URLs, titles, and optional page text.", + "description": "Search the web using the configured provider (Exa, Firecrawl, or Brave). Returns URLs, titles, and optional page text.", "parameters": { "type": "object", "properties": { diff --git a/agent/tools.py b/agent/tools.py index a9d6d4ef..102d4863 100644 --- a/agent/tools.py +++ b/agent/tools.py @@ -3,6 +3,7 @@ import ast import base64 import fnmatch +import html as _html import json import os import signal @@ -11,11 +12,13 @@ import tempfile import threading import urllib.error +import urllib.parse import urllib.request import re as _re import zlib from contextlib import contextmanager from dataclasses import dataclass, field +from html.parser import HTMLParser from pathlib import Path from typing import Any @@ -36,6 +39,66 @@ _INTERACTIVE_RE = _re.compile(r"(^|[;&|]\s*)(vim|nano|less|more|top|htop|man)\b") +class _HTMLTextExtractor(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=False) + self._title_parts: list[str] = [] + self._text_parts: list[str] = [] + self._skip_depth = 0 + self._in_title = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + lowered = tag.lower() + if lowered in {"script", "style"}: + self._skip_depth += 1 + return + if self._skip_depth: + return + if lowered == "title": + self._in_title = True + return + if lowered in {"article", "br", "div", "footer", "h1", "h2", "h3", "h4", "h5", "h6", "header", "li", "main", "p", "section", "td", "th", "tr"}: + self._text_parts.append("\n") + + def handle_endtag(self, tag: str) -> None: + lowered = tag.lower() + if lowered in {"script", "style"}: + if self._skip_depth: + self._skip_depth -= 1 + return + if self._skip_depth: + return + if lowered == "title": + self._in_title = False + return + if lowered in {"article", "div", "footer", "h1", "h2", "h3", "h4", "h5", "h6", "header", "li", "main", "p", "section", "td", "th", "tr"}: + self._text_parts.append("\n") + + def handle_data(self, data: str) -> None: + if self._skip_depth or not data: + return + if self._in_title: + self._title_parts.append(data) + self._text_parts.append(data) + + def title(self) -> str: + return _WS_RE.sub(" ", _html.unescape("".join(self._title_parts))).strip() + + def text(self) -> str: + return _WS_RE.sub(" ", _html.unescape(" ".join(self._text_parts))).strip() + + +def _extract_html_text(raw_html: str) -> tuple[str, str]: + parser = _HTMLTextExtractor() + try: + parser.feed(raw_html) + parser.close() + return parser.title(), parser.text() + except Exception: + stripped = _WS_RE.sub(" ", _re.sub(r"(?is)<[^>]+>", " ", raw_html)).strip() + return "", _html.unescape(stripped) + + def _line_hash(line: str) -> str: """2-char hex hash, whitespace-invariant.""" return format(zlib.crc32(_WS_RE.sub("", line).encode("utf-8")) & 0xFF, "02x") @@ -59,6 +122,8 @@ class WorkspaceTools: exa_base_url: str = "https://api.exa.ai" firecrawl_api_key: str | None = None firecrawl_base_url: str = "https://api.firecrawl.dev/v1" + brave_api_key: str | None = None + brave_base_url: str = "https://api.search.brave.com/res/v1" def __post_init__(self) -> None: self.root = self.root.expanduser().resolve() @@ -839,6 +904,84 @@ def _firecrawl_request(self, endpoint: str, payload: dict[str, Any]) -> dict[str raise ToolError(f"Firecrawl API returned non-object response: {type(parsed)!r}") return parsed + def _brave_request(self, endpoint: str, params: dict[str, Any]) -> dict[str, Any]: + if not (self.brave_api_key and self.brave_api_key.strip()): + raise ToolError("BRAVE_API_KEY not configured") + query = urllib.parse.urlencode(params, doseq=True) + url = self.brave_base_url.rstrip("/") + endpoint + if query: + url = f"{url}?{query}" + req = urllib.request.Request( + url=url, + headers={ + "Accept": "application/json", + "X-Subscription-Token": self.brave_api_key, + }, + method="GET", + ) + try: + with urllib.request.urlopen(req, timeout=self.command_timeout_sec) as resp: + raw = resp.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise ToolError(f"Brave API HTTP {exc.code}: {body}") from exc + except urllib.error.URLError as exc: + raise ToolError(f"Brave API connection error: {exc}") from exc + except OSError as exc: + raise ToolError(f"Brave API network error: {exc}") from exc + + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise ToolError(f"Brave API returned non-JSON payload: {raw[:500]}") from exc + if not isinstance(parsed, dict): + raise ToolError(f"Brave API returned non-object response: {type(parsed)!r}") + return parsed + + def _fetch_url_direct(self, url: str) -> dict[str, str]: + req = urllib.request.Request( + url=url, + headers={ + "Accept": "text/html,application/xhtml+xml,application/json,text/plain;q=0.9,*/*;q=0.8", + "User-Agent": "OpenPlanter/1.0", + }, + method="GET", + ) + try: + with urllib.request.urlopen(req, timeout=self.command_timeout_sec) as resp: + resolved_url = resp.geturl() + charset = resp.headers.get_content_charset() or "utf-8" + raw = resp.read().decode(charset, errors="replace") + content_type = (resp.headers.get("Content-Type") or "").lower() + except urllib.error.HTTPError as exc: + return { + "url": url, + "title": "", + "text": f"Direct fetch failed: HTTP {exc.code}", + } + except urllib.error.URLError as exc: + return { + "url": url, + "title": "", + "text": f"Direct fetch failed: {exc}", + } + except OSError as exc: + return { + "url": url, + "title": "", + "text": f"Direct fetch failed: {exc}", + } + + if "html" in content_type: + title, text = _extract_html_text(raw) + else: + title, text = "", raw + return { + "url": resolved_url, + "title": title, + "text": self._clip(text or raw, 8000), + } + def web_search( self, query: str, @@ -850,7 +993,7 @@ def web_search( return "web_search requires non-empty query" clamped_results = max(1, min(int(num_results), 20)) provider = (self.web_search_provider or "exa").strip().lower() - if provider not in {"exa", "firecrawl"}: + if provider not in {"exa", "firecrawl", "brave"}: provider = "exa" if provider == "firecrawl": @@ -902,6 +1045,58 @@ def web_search( } return self._clip(json.dumps(output, indent=2, ensure_ascii=True), self.max_file_chars) + if provider == "brave": + params: dict[str, Any] = { + "q": query, + "count": clamped_results, + } + if include_text: + params["extra_snippets"] = "true" + + try: + parsed = self._brave_request("/web/search", params) + except Exception as exc: + return f"Web search failed: {exc}" + + rows: list[Any] = [] + web = parsed.get("web") + if isinstance(web, dict): + web_rows = web.get("results") + if isinstance(web_rows, list): + rows = web_rows + elif isinstance(parsed.get("results"), list): + rows = parsed["results"] + + out_results: list[dict[str, Any]] = [] + for row in rows: + if not isinstance(row, dict): + continue + description = str(row.get("description", "") or row.get("snippet", "")) + extra_snippets = row.get("extra_snippets") + extra_texts = [ + snippet + for snippet in extra_snippets + if isinstance(snippet, str) and snippet + ] if isinstance(extra_snippets, list) else [] + item: dict[str, Any] = { + "url": str(row.get("url", "")), + "title": str(row.get("title", "")), + "snippet": description or (extra_texts[0] if extra_texts else ""), + } + if include_text: + text_parts = [part for part in [description, *extra_texts] if part] + if text_parts: + item["text"] = self._clip("\n\n".join(text_parts), 4000) + out_results.append(item) + + output = { + "query": query, + "provider": provider, + "results": out_results, + "total": len(out_results), + } + return self._clip(json.dumps(output, indent=2, ensure_ascii=True), self.max_file_chars) + payload: dict[str, Any] = { "query": query, "numResults": clamped_results, @@ -949,7 +1144,7 @@ def fetch_url(self, urls: list[str]) -> str: return "fetch_url requires at least one valid URL" normalized = normalized[:10] provider = (self.web_search_provider or "exa").strip().lower() - if provider not in {"exa", "firecrawl"}: + if provider not in {"exa", "firecrawl", "brave"}: provider = "exa" if provider == "firecrawl": @@ -985,6 +1180,15 @@ def fetch_url(self, urls: list[str]) -> str: } return self._clip(json.dumps(output, indent=2, ensure_ascii=True), self.max_file_chars) + if provider == "brave": + pages = [self._fetch_url_direct(url) for url in normalized] + output = { + "provider": provider, + "pages": pages, + "total": len(pages), + } + return self._clip(json.dumps(output, indent=2, ensure_ascii=True), self.max_file_chars) + payload: dict[str, Any] = { "ids": normalized, "text": {"maxCharacters": 8000}, diff --git a/openplanter-desktop/crates/op-core/src/config.rs b/openplanter-desktop/crates/op-core/src/config.rs index b25abbe0..e75b21a1 100644 --- a/openplanter-desktop/crates/op-core/src/config.rs +++ b/openplanter-desktop/crates/op-core/src/config.rs @@ -14,6 +14,7 @@ pub const FOUNDRY_OPENAI_API_KEY_PLACEHOLDER: &str = "dont-worry-this-key-will-b pub const FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER: &str = "dont-worry-it-will-be-injected"; pub const ZAI_PAYGO_BASE_URL: &str = "https://api.z.ai/api/paas/v4"; pub const ZAI_CODING_BASE_URL: &str = "https://api.z.ai/api/coding/paas/v4"; +pub const BRAVE_BASE_URL: &str = "https://api.search.brave.com/res/v1"; /// Default model for each supported provider. pub static PROVIDER_DEFAULT_MODELS: LazyLock> = @@ -75,6 +76,7 @@ pub fn resolve_zai_base_url(plan: &str, paygo_base_url: &str, coding_base_url: & pub fn normalize_web_search_provider(value: Option<&str>) -> String { match value.unwrap_or_default().trim().to_lowercase().as_str() { "firecrawl" => "firecrawl".to_string(), + "brave" => "brave".to_string(), _ => "exa".to_string(), } } @@ -166,6 +168,7 @@ pub struct AgentConfig { pub ollama_base_url: String, pub exa_base_url: String, pub firecrawl_base_url: String, + pub brave_base_url: String, // API keys pub api_key: Option, @@ -176,6 +179,7 @@ pub struct AgentConfig { pub zai_api_key: Option, pub exa_api_key: Option, pub firecrawl_api_key: Option, + pub brave_api_key: Option, pub web_search_provider: String, pub voyage_api_key: Option, @@ -224,6 +228,7 @@ impl Default for AgentConfig { ollama_base_url: "http://localhost:11434/v1".into(), exa_base_url: "https://api.exa.ai".into(), firecrawl_base_url: "https://api.firecrawl.dev/v1".into(), + brave_base_url: BRAVE_BASE_URL.into(), api_key: Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.into()), openai_api_key: Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.into()), anthropic_api_key: Some(FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER.into()), @@ -232,6 +237,7 @@ impl Default for AgentConfig { zai_api_key: None, exa_api_key: None, firecrawl_api_key: None, + brave_api_key: None, web_search_provider: "exa".into(), voyage_api_key: None, max_depth: 4, @@ -284,6 +290,8 @@ impl AgentConfig { let firecrawl_api_key = env_opt("OPENPLANTER_FIRECRAWL_API_KEY").or_else(|| env_opt("FIRECRAWL_API_KEY")); + let brave_api_key = + env_opt("OPENPLANTER_BRAVE_API_KEY").or_else(|| env_opt("BRAVE_API_KEY")); let voyage_api_key = env_opt("OPENPLANTER_VOYAGE_API_KEY").or_else(|| env_opt("VOYAGE_API_KEY")); @@ -348,6 +356,7 @@ impl AgentConfig { "OPENPLANTER_FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v1", ), + brave_base_url: env_or("OPENPLANTER_BRAVE_BASE_URL", BRAVE_BASE_URL), openai_api_key, anthropic_api_key, openrouter_api_key, @@ -355,6 +364,7 @@ impl AgentConfig { zai_api_key, exa_api_key, firecrawl_api_key, + brave_api_key, web_search_provider, voyage_api_key, max_depth: env_int("OPENPLANTER_MAX_DEPTH", 4), @@ -434,6 +444,8 @@ mod tests { assert_eq!(cfg.zai_plan, "paygo"); assert_eq!(cfg.zai_base_url, ZAI_PAYGO_BASE_URL); assert_eq!(cfg.web_search_provider, "exa"); + assert_eq!(cfg.brave_base_url, BRAVE_BASE_URL); + assert!(cfg.brave_api_key.is_none()); assert_eq!(cfg.rate_limit_max_retries, 12); assert_eq!(cfg.rate_limit_backoff_base_sec, 1.0); assert_eq!(cfg.rate_limit_backoff_max_sec, 60.0); @@ -486,6 +498,9 @@ mod tests { "OPENPLANTER_RECURSIVE", "OPENPLANTER_DEMO", "OPENPLANTER_WEB_SEARCH_PROVIDER", + "OPENPLANTER_BRAVE_API_KEY", + "BRAVE_API_KEY", + "OPENPLANTER_BRAVE_BASE_URL", "OPENPLANTER_ZAI_PLAN", "OPENPLANTER_ZAI_BASE_URL", "OPENPLANTER_RATE_LIMIT_MAX_RETRIES", @@ -521,6 +536,7 @@ mod tests { Some(FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER) ); assert!(cfg.zai_api_key.is_none()); + assert!(cfg.brave_api_key.is_none()); assert_eq!(cfg.openai_base_url, FOUNDRY_OPENAI_BASE_URL); assert_eq!(cfg.anthropic_base_url, FOUNDRY_ANTHROPIC_BASE_URL); assert_eq!(cfg.web_search_provider, "exa"); @@ -539,7 +555,8 @@ mod tests { env::set_var("OPENPLANTER_DEMO", "true"); env::set_var("OPENAI_API_KEY", "sk-test123"); env::set_var("ZAI_API_KEY", "zai-test123"); - env::set_var("OPENPLANTER_WEB_SEARCH_PROVIDER", "firecrawl"); + env::set_var("BRAVE_API_KEY", "brave-test123"); + env::set_var("OPENPLANTER_WEB_SEARCH_PROVIDER", "brave"); env::set_var("OPENPLANTER_RATE_LIMIT_MAX_RETRIES", "5"); env::set_var("OPENPLANTER_RATE_LIMIT_BACKOFF_BASE_SEC", "2.5"); env::set_var("OPENPLANTER_RATE_LIMIT_BACKOFF_MAX_SEC", "30.0"); @@ -557,10 +574,11 @@ mod tests { assert!(cfg.demo); assert_eq!(cfg.openai_api_key, Some("sk-test123".into())); assert_eq!(cfg.zai_api_key, Some("zai-test123".into())); + assert_eq!(cfg.brave_api_key, Some("brave-test123".into())); assert_eq!(cfg.zai_plan, "coding"); assert_eq!(cfg.zai_base_url, ZAI_CODING_BASE_URL); assert_eq!(cfg.zai_stream_max_retries, 7); - assert_eq!(cfg.web_search_provider, "firecrawl"); + assert_eq!(cfg.web_search_provider, "brave"); assert_eq!(cfg.rate_limit_max_retries, 5); assert_eq!(cfg.rate_limit_backoff_base_sec, 2.5); assert_eq!(cfg.rate_limit_backoff_max_sec, 30.0); @@ -589,6 +607,7 @@ mod tests { normalize_web_search_provider(Some("firecrawl")), "firecrawl" ); + assert_eq!(normalize_web_search_provider(Some("brave")), "brave"); assert_eq!(normalize_web_search_provider(Some("other")), "exa"); assert!(is_foundry_openai_base_url(FOUNDRY_OPENAI_BASE_URL)); assert!(is_foundry_anthropic_base_url(FOUNDRY_ANTHROPIC_BASE_URL)); diff --git a/openplanter-desktop/crates/op-core/src/credentials.rs b/openplanter-desktop/crates/op-core/src/credentials.rs index af746019..d7ce3e52 100644 --- a/openplanter-desktop/crates/op-core/src/credentials.rs +++ b/openplanter-desktop/crates/op-core/src/credentials.rs @@ -19,13 +19,14 @@ pub struct CredentialBundle { pub zai_api_key: Option, pub exa_api_key: Option, pub firecrawl_api_key: Option, + pub brave_api_key: Option, pub voyage_api_key: Option, } impl CredentialBundle { /// Returns `true` if any key has a non-empty value. pub fn has_any(&self) -> bool { - let keys: [&Option; 8] = [ + let keys: [&Option; 9] = [ &self.openai_api_key, &self.anthropic_api_key, &self.openrouter_api_key, @@ -33,6 +34,7 @@ impl CredentialBundle { &self.zai_api_key, &self.exa_api_key, &self.firecrawl_api_key, + &self.brave_api_key, &self.voyage_api_key, ]; keys.iter() @@ -55,6 +57,7 @@ impl CredentialBundle { fill!(zai_api_key); fill!(exa_api_key); fill!(firecrawl_api_key); + fill!(brave_api_key); fill!(voyage_api_key); } @@ -75,6 +78,7 @@ impl CredentialBundle { add!(zai_api_key, "zai_api_key"); add!(exa_api_key, "exa_api_key"); add!(firecrawl_api_key, "firecrawl_api_key"); + add!(brave_api_key, "brave_api_key"); add!(voyage_api_key, "voyage_api_key"); out } @@ -95,6 +99,7 @@ impl CredentialBundle { zai_api_key: get_str(payload, "zai_api_key"), exa_api_key: get_str(payload, "exa_api_key"), firecrawl_api_key: get_str(payload, "firecrawl_api_key"), + brave_api_key: get_str(payload, "brave_api_key"), voyage_api_key: get_str(payload, "voyage_api_key"), } } @@ -161,6 +166,7 @@ pub fn parse_env_file(path: &Path) -> CredentialBundle { "FIRECRAWL_API_KEY", "OPENPLANTER_FIRECRAWL_API_KEY", ), + brave_api_key: get_key(&env_map, "BRAVE_API_KEY", "OPENPLANTER_BRAVE_API_KEY"), voyage_api_key: get_key(&env_map, "VOYAGE_API_KEY", "OPENPLANTER_VOYAGE_API_KEY"), } } @@ -183,6 +189,7 @@ pub fn credentials_from_env() -> CredentialBundle { zai_api_key: env_key("OPENPLANTER_ZAI_API_KEY", "ZAI_API_KEY"), exa_api_key: env_key("OPENPLANTER_EXA_API_KEY", "EXA_API_KEY"), firecrawl_api_key: env_key("OPENPLANTER_FIRECRAWL_API_KEY", "FIRECRAWL_API_KEY"), + brave_api_key: env_key("OPENPLANTER_BRAVE_API_KEY", "BRAVE_API_KEY"), voyage_api_key: env_key("OPENPLANTER_VOYAGE_API_KEY", "VOYAGE_API_KEY"), } } @@ -346,6 +353,7 @@ mod tests { anthropic_api_key: None, openrouter_api_key: Some("or-456".into()), firecrawl_api_key: Some("fc-789".into()), + brave_api_key: Some("brave-101".into()), ..Default::default() }; let json = bundle.to_json(); @@ -353,6 +361,7 @@ mod tests { assert!(!json.contains_key("anthropic_api_key")); assert_eq!(json.get("openrouter_api_key").unwrap(), "or-456"); assert_eq!(json.get("firecrawl_api_key").unwrap(), "fc-789"); + assert_eq!(json.get("brave_api_key").unwrap(), "brave-101"); } #[test] @@ -368,6 +377,7 @@ export ANTHROPIC_API_KEY='ant-key' EXA_API_KEY="exa-quoted" ZAI_API_KEY=zai-from-env OPENPLANTER_FIRECRAWL_API_KEY="firecrawl-quoted" +BRAVE_API_KEY=brave-from-env UNRELATED_VAR=foo "#, ) @@ -379,6 +389,7 @@ UNRELATED_VAR=foo assert_eq!(bundle.exa_api_key, Some("exa-quoted".into())); assert_eq!(bundle.zai_api_key, Some("zai-from-env".into())); assert_eq!(bundle.firecrawl_api_key, Some("firecrawl-quoted".into())); + assert_eq!(bundle.brave_api_key, Some("brave-from-env".into())); assert!(bundle.cerebras_api_key.is_none()); } @@ -390,6 +401,7 @@ UNRELATED_VAR=foo openai_api_key: Some("sk-test".into()), anthropic_api_key: Some("ant-test".into()), zai_api_key: Some("zai-test".into()), + brave_api_key: Some("brave-test".into()), ..Default::default() }; store.save(&bundle).unwrap(); @@ -397,6 +409,7 @@ UNRELATED_VAR=foo assert_eq!(loaded.openai_api_key, Some("sk-test".into())); assert_eq!(loaded.anthropic_api_key, Some("ant-test".into())); assert_eq!(loaded.zai_api_key, Some("zai-test".into())); + assert_eq!(loaded.brave_api_key, Some("brave-test".into())); } #[test] diff --git a/openplanter-desktop/crates/op-core/src/tools/defs.rs b/openplanter-desktop/crates/op-core/src/tools/defs.rs index a29ceabe..7b1d5835 100644 --- a/openplanter-desktop/crates/op-core/src/tools/defs.rs +++ b/openplanter-desktop/crates/op-core/src/tools/defs.rs @@ -176,7 +176,7 @@ fn mvp_tool_defs() -> Vec { // ── Web ── ToolDef { name: "web_search", - description: "Search the web using the configured Exa or Firecrawl backend. Returns URLs, titles, snippets, and optional page text.", + description: "Search the web using the configured Exa, Firecrawl, or Brave backend. Returns URLs, titles, snippets, and optional page text.", parameters: json!({ "type": "object", "properties": { @@ -199,7 +199,7 @@ fn mvp_tool_defs() -> Vec { }, ToolDef { name: "fetch_url", - description: "Fetch and return the text content of one or more URLs using the configured Exa or Firecrawl backend.", + description: "Fetch and return the text content of one or more URLs using the configured Exa, Firecrawl, or Brave backend.", parameters: json!({ "type": "object", "properties": { diff --git a/openplanter-desktop/crates/op-core/src/tools/mod.rs b/openplanter-desktop/crates/op-core/src/tools/mod.rs index a44fc2e5..f6220a92 100644 --- a/openplanter-desktop/crates/op-core/src/tools/mod.rs +++ b/openplanter-desktop/crates/op-core/src/tools/mod.rs @@ -58,6 +58,8 @@ pub struct WorkspaceTools { exa_base_url: String, firecrawl_api_key: Option, firecrawl_base_url: String, + brave_api_key: Option, + brave_base_url: String, files_read: HashSet, bg_jobs: shell::BgJobs, } @@ -88,6 +90,8 @@ impl WorkspaceTools { exa_base_url: config.exa_base_url.clone(), firecrawl_api_key: config.firecrawl_api_key.clone(), firecrawl_base_url: config.firecrawl_base_url.clone(), + brave_api_key: config.brave_api_key.clone(), + brave_base_url: config.brave_base_url.clone(), files_read: HashSet::new(), bg_jobs: shell::BgJobs::new(), } @@ -114,6 +118,8 @@ impl WorkspaceTools { exa_base_url: config.exa_base_url.clone(), firecrawl_api_key: config.firecrawl_api_key.clone(), firecrawl_base_url: config.firecrawl_base_url.clone(), + brave_api_key: config.brave_api_key.clone(), + brave_base_url: config.brave_base_url.clone(), files_read: HashSet::new(), bg_jobs: shell::BgJobs::new(), } @@ -241,6 +247,8 @@ impl WorkspaceTools { &self.exa_base_url, self.firecrawl_api_key.as_deref(), &self.firecrawl_base_url, + self.brave_api_key.as_deref(), + &self.brave_base_url, query, num_results, include_text, diff --git a/openplanter-desktop/crates/op-core/src/tools/web.rs b/openplanter-desktop/crates/op-core/src/tools/web.rs index fb67a633..2b36060e 100644 --- a/openplanter-desktop/crates/op-core/src/tools/web.rs +++ b/openplanter-desktop/crates/op-core/src/tools/web.rs @@ -1,12 +1,26 @@ -/// Web tools: Exa / Firecrawl search and fetch_url. +/// Web tools: Exa / Firecrawl / Brave search and fetch_url. use std::time::Duration; +use std::sync::LazyLock; +use regex::Regex; use serde_json::json; use crate::config::normalize_web_search_provider; use super::ToolResult; +static SCRIPT_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)]*>.*?").unwrap()); +static STYLE_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)]*>.*?").unwrap()); +static TITLE_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)]*>(.*?)").unwrap()); +static BLOCK_TAG_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?is)]*>") + .unwrap() +}); +static TAG_RE: LazyLock = LazyLock::new(|| Regex::new(r"(?is)<[^>]+>").unwrap()); + fn clip(text: &str, max_chars: usize) -> String { if text.len() <= max_chars { return text.to_string(); @@ -16,6 +30,34 @@ fn clip(text: &str, max_chars: usize) -> String { format!("{}\n\n...[truncated {omitted} chars]...", &text[..end]) } +fn collapse_ws(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") +} + +fn decode_html_entities(text: &str) -> String { + text.replace(" ", " ") + .replace(""", "\"") + .replace("'", "'") + .replace("'", "'") + .replace("<", "<") + .replace(">", ">") + .replace("&", "&") +} + +fn extract_html_text(html: &str) -> (String, String) { + let title = TITLE_RE + .captures(html) + .and_then(|caps| caps.get(1)) + .map(|m| collapse_ws(&decode_html_entities(m.as_str()))) + .unwrap_or_default(); + let without_scripts = SCRIPT_RE.replace_all(html, " "); + let without_styles = STYLE_RE.replace_all(&without_scripts, " "); + let with_breaks = BLOCK_TAG_RE.replace_all(&without_styles, "\n"); + let plain = TAG_RE.replace_all(&with_breaks, " "); + let text = collapse_ws(&decode_html_entities(&plain)); + (title, text) +} + async fn exa_request( api_key: Option<&str>, exa_base_url: &str, @@ -85,12 +127,119 @@ async fn firecrawl_request( .map_err(|e| format!("Firecrawl API returned non-JSON payload: {e}")) } +async fn brave_request( + api_key: Option<&str>, + brave_base_url: &str, + endpoint: &str, + params: &[(&str, String)], + timeout_sec: u64, +) -> Result { + let api_key = match api_key { + Some(value) if !value.trim().is_empty() => value, + _ => return Err("BRAVE_API_KEY not configured".into()), + }; + + let url = format!("{}{}", brave_base_url.trim_end_matches('/'), endpoint); + let client = reqwest::Client::new(); + let response = client + .get(&url) + .header("Accept", "application/json") + .header("X-Subscription-Token", api_key) + .query(params) + .timeout(Duration::from_secs(timeout_sec)) + .send() + .await + .map_err(|e| format!("Brave API request failed: {e}"))?; + + let response = response + .error_for_status() + .map_err(|e| format!("Brave API request failed: {e}"))?; + + response + .json::() + .await + .map_err(|e| format!("Brave API returned non-JSON payload: {e}")) +} + +async fn fetch_direct_page(url: &str, timeout_sec: u64) -> serde_json::Value { + let client = reqwest::Client::new(); + let response = match client + .get(url) + .header( + "Accept", + "text/html,application/xhtml+xml,application/json,text/plain;q=0.9,*/*;q=0.8", + ) + .header("User-Agent", "OpenPlanter/1.0") + .timeout(Duration::from_secs(timeout_sec)) + .send() + .await + { + Ok(response) => response, + Err(error) => { + return json!({ + "url": url, + "title": "", + "text": format!("Direct fetch failed: {error}"), + }); + } + }; + + let final_url = response.url().to_string(); + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_lowercase(); + + let response = match response.error_for_status() { + Ok(response) => response, + Err(error) => { + return json!({ + "url": url, + "title": "", + "text": format!("Direct fetch failed: {error}"), + }); + } + }; + + let body = match response.text().await { + Ok(body) => body, + Err(error) => { + return json!({ + "url": final_url, + "title": "", + "text": format!("Direct fetch failed: {error}"), + }); + } + }; + + let (title, extracted_text) = if content_type.contains("html") { + extract_html_text(&body) + } else { + (String::new(), body.clone()) + }; + let text = if extracted_text.is_empty() { + body + } else { + extracted_text + }; + + json!({ + "url": final_url, + "title": title, + "text": clip(&text, 8_000), + }) +} + pub async fn web_search( provider: &str, exa_api_key: Option<&str>, exa_base_url: &str, firecrawl_api_key: Option<&str>, firecrawl_base_url: &str, + brave_api_key: Option<&str>, + brave_base_url: &str, query: &str, num_results: i64, include_text: bool, @@ -183,6 +332,85 @@ pub async fn web_search( } Err(error) => return ToolResult::error(format!("Web search failed: {error}")), } + } else if provider == "brave" { + let mut params = vec![ + ("q", query.to_string()), + ("count", clamped.to_string()), + ]; + if include_text { + params.push(("extra_snippets", "true".to_string())); + } + + match brave_request( + brave_api_key, + brave_base_url, + "/web/search", + ¶ms, + timeout_sec, + ) + .await + { + Ok(body) => { + let rows = body + .get("web") + .and_then(|value| value.get("results")) + .and_then(|value| value.as_array()) + .or_else(|| body.get("results").and_then(|value| value.as_array())); + let mut results: Vec = Vec::new(); + if let Some(rows) = rows { + for row in rows { + let description = row + .get("description") + .and_then(|value| value.as_str()) + .or_else(|| row.get("snippet").and_then(|value| value.as_str())) + .unwrap_or("") + .to_string(); + let extra_texts = row + .get("extra_snippets") + .and_then(|value| value.as_array()) + .map(|items| { + items + .iter() + .filter_map(|value| value.as_str()) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .collect::>() + }) + .unwrap_or_default(); + let snippet = if !description.is_empty() { + description.clone() + } else { + extra_texts.first().cloned().unwrap_or_default() + }; + + let mut item = json!({ + "url": row.get("url").and_then(|value| value.as_str()).unwrap_or(""), + "title": row.get("title").and_then(|value| value.as_str()).unwrap_or(""), + "snippet": snippet, + }); + if include_text { + let mut text_parts = Vec::new(); + if !description.is_empty() { + text_parts.push(description.clone()); + } + text_parts.extend(extra_texts.clone()); + if !text_parts.is_empty() { + item["text"] = json!(clip(&text_parts.join("\n\n"), 4_000)); + } + } + results.push(item); + } + } + + json!({ + "query": query, + "provider": provider, + "results": results, + "total": results.len(), + }) + } + Err(error) => return ToolResult::error(format!("Web search failed: {error}")), + } } else { let mut payload = json!({ "query": query, @@ -300,6 +528,17 @@ pub async fn fetch_url( } } + json!({ + "provider": provider, + "pages": pages, + "total": pages.len(), + }) + } else if provider == "brave" { + let mut pages: Vec = Vec::new(); + for url in &normalized { + pages.push(fetch_direct_page(url, timeout_sec).await); + } + json!({ "provider": provider, "pages": pages, @@ -356,7 +595,7 @@ mod tests { use axum::body::Body; use axum::http::StatusCode; use axum::response::Response; - use axum::routing::post; + use axum::routing::{get, post}; use axum::{Json, Router}; use serde_json::{Value, json}; @@ -381,6 +620,48 @@ mod tests { addr } + async fn start_json_get_server( + path: &'static str, + response_payload: Value, + ) -> std::net::SocketAddr { + let app = Router::new().route( + path, + get(move || { + let response_payload = response_payload.clone(); + async move { Json(response_payload) } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + addr + } + + async fn start_text_get_server( + path: &'static str, + body: &'static str, + content_type: &'static str, + ) -> std::net::SocketAddr { + let app = Router::new().route( + path, + get(move || async move { + Response::builder() + .status(StatusCode::OK) + .header("Content-Type", content_type) + .body(Body::from(body)) + .unwrap() + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + addr + } + async fn start_status_server(path: &'static str, status: StatusCode) -> std::net::SocketAddr { let app = Router::new().route( path, @@ -422,6 +703,8 @@ mod tests { &format!("http://{addr}"), None, "https://api.firecrawl.dev/v1", + None, + "https://api.search.brave.com/res/v1", "example query", 5, true, @@ -461,6 +744,8 @@ mod tests { "https://api.exa.ai", Some("fc-key"), &format!("http://{addr}"), + None, + "https://api.search.brave.com/res/v1", "example query", 5, true, @@ -509,6 +794,76 @@ mod tests { assert_eq!(parsed["pages"][0]["text"], "Article body"); } + #[tokio::test] + async fn test_web_search_brave_output_shape() { + let addr = start_json_get_server( + "/web/search", + json!({ + "web": { + "results": [ + { + "url": "https://example.com/brave", + "title": "Brave Title", + "description": "Brave snippet", + "extra_snippets": ["Extra context"] + } + ] + } + }), + ) + .await; + + let result = web_search( + "brave", + None, + "https://api.exa.ai", + None, + "https://api.firecrawl.dev/v1", + Some("brave-key"), + &format!("http://{addr}"), + "example query", + 5, + true, + 20_000, + 5, + ) + .await; + + assert!(!result.is_error); + let parsed: Value = serde_json::from_str(&result.content).unwrap(); + assert_eq!(parsed["provider"], "brave"); + assert_eq!(parsed["results"][0]["title"], "Brave Title"); + assert!(parsed["results"][0]["text"].as_str().unwrap().contains("Extra context")); + } + + #[tokio::test] + async fn test_fetch_url_brave_output_shape() { + let addr = start_text_get_server( + "/page", + "Brave Page

Hello Brave

Readable text.

", + "text/html; charset=utf-8", + ) + .await; + + let result = fetch_url( + "brave", + None, + "https://api.exa.ai", + None, + "https://api.firecrawl.dev/v1", + &[format!("http://{addr}/page")], + 20_000, + 5, + ) + .await; + + assert!(!result.is_error); + let parsed: Value = serde_json::from_str(&result.content).unwrap(); + assert_eq!(parsed["provider"], "brave"); + assert_eq!(parsed["pages"][0]["title"], "Brave Page"); + assert!(parsed["pages"][0]["text"].as_str().unwrap().contains("Hello Brave")); + } + #[tokio::test] async fn test_missing_firecrawl_key_errors() { let result = web_search( @@ -517,6 +872,8 @@ mod tests { "https://api.exa.ai", None, "https://api.firecrawl.dev/v1", + None, + "https://api.search.brave.com/res/v1", "example query", 5, false, @@ -529,6 +886,28 @@ mod tests { assert!(result.content.contains("FIRECRAWL_API_KEY")); } + #[tokio::test] + async fn test_missing_brave_key_errors() { + let result = web_search( + "brave", + None, + "https://api.exa.ai", + None, + "https://api.firecrawl.dev/v1", + None, + "https://api.search.brave.com/res/v1", + "example query", + 5, + false, + 20_000, + 5, + ) + .await; + + assert!(result.is_error); + assert!(result.content.contains("BRAVE_API_KEY")); + } + #[tokio::test] async fn test_exa_http_error_bubbles_up() { let addr = start_status_server("/search", StatusCode::BAD_GATEWAY).await; @@ -539,6 +918,8 @@ mod tests { &format!("http://{addr}"), None, "https://api.firecrawl.dev/v1", + None, + "https://api.search.brave.com/res/v1", "example query", 5, false, diff --git a/openplanter-desktop/crates/op-tauri/src/commands/config.rs b/openplanter-desktop/crates/op-tauri/src/commands/config.rs index cf3f0edb..7224456c 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/config.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/config.rs @@ -200,6 +200,7 @@ pub fn build_credential_status(cfg: &op_core::config::AgentConfig) -> HashMap { ollama: true, exa: false, firecrawl: true, + brave: false, })); const status = await getCredentialsStatus(); expect(status.openai).toBe(true); expect(status.openrouter).toBe(false); expect(status.zai).toBe(true); expect(status.firecrawl).toBe(true); + expect(status.brave).toBe(false); }); it("listSessions sends limit", async () => { diff --git a/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts b/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts index ef51eed2..e019d03a 100644 --- a/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts +++ b/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts @@ -81,13 +81,13 @@ describe("completionRegistry", () => { expect(childValues).toEqual(["low", "medium", "high", "off"]); }); - it("/web-search has exa and firecrawl children", () => { + it("/web-search has exa, firecrawl, and brave children", () => { const webSearchCmd = COMMAND_COMPLETIONS.find((c) => c.value === "/web-search"); expect(webSearchCmd).toBeDefined(); expect(webSearchCmd!.children).toBeDefined(); const childValues = webSearchCmd!.children!.map((c) => c.value); - expect(childValues).toEqual(["exa", "firecrawl"]); + expect(childValues).toEqual(["exa", "firecrawl", "brave"]); expect(webSearchCmd!.children![0].children?.[0].value).toBe("--save"); }); diff --git a/openplanter-desktop/frontend/src/commands/completionRegistry.ts b/openplanter-desktop/frontend/src/commands/completionRegistry.ts index 2bb2b166..2133f2d3 100644 --- a/openplanter-desktop/frontend/src/commands/completionRegistry.ts +++ b/openplanter-desktop/frontend/src/commands/completionRegistry.ts @@ -39,6 +39,7 @@ const REASONING_LEVELS: CompletionItem[] = [ const WEB_SEARCH_PROVIDERS: CompletionItem[] = [ { value: "exa", description: "Use Exa for web search", children: SAVE_FLAG }, { value: "firecrawl", description: "Use Firecrawl for web search", children: SAVE_FLAG }, + { value: "brave", description: "Use Brave Search for web search", children: SAVE_FLAG }, ]; const ZAI_PLANS: CompletionItem[] = [ diff --git a/openplanter-desktop/frontend/src/commands/slash.ts b/openplanter-desktop/frontend/src/commands/slash.ts index d46675dd..34df61f1 100644 --- a/openplanter-desktop/frontend/src/commands/slash.ts +++ b/openplanter-desktop/frontend/src/commands/slash.ts @@ -34,7 +34,7 @@ export async function dispatchSlashCommand(input: string): Promise Set Z.AI endpoint family (paygo, coding)", " /zai-plan --save Set and persist", " /web-search Show current web search provider", - " /web-search Set web search provider (exa, firecrawl)", + " /web-search Set web search provider (exa, firecrawl, brave)", " /web-search --save Set and persist", " /reasoning Show/set reasoning effort", " /reasoning Set level (low, medium, high, off)", diff --git a/openplanter-desktop/frontend/src/commands/webSearch.test.ts b/openplanter-desktop/frontend/src/commands/webSearch.test.ts index 358cbed6..cb5ed492 100644 --- a/openplanter-desktop/frontend/src/commands/webSearch.test.ts +++ b/openplanter-desktop/frontend/src/commands/webSearch.test.ts @@ -31,7 +31,7 @@ describe("handleWebSearchCommand", () => { it("switches provider for the current session", async () => { __setHandler("update_config", ({ partial }: { partial: Record }) => { - expect(partial.web_search_provider).toBe("firecrawl"); + expect(partial.web_search_provider).toBe("brave"); return { provider: "anthropic", model: "claude-opus-4-6", @@ -42,14 +42,14 @@ describe("handleWebSearchCommand", () => { max_depth: 4, max_steps_per_call: 100, reasoning_effort: "high", - web_search_provider: "firecrawl", + web_search_provider: "brave", demo: false, }; }); - const result = await handleWebSearchCommand("firecrawl"); - expect(result.lines).toContain("Web search provider set to: firecrawl"); - expect(appState.get().webSearchProvider).toBe("firecrawl"); + const result = await handleWebSearchCommand("brave"); + expect(result.lines).toContain("Web search provider set to: brave"); + expect(appState.get().webSearchProvider).toBe("brave"); }); it("save persists the selected provider", async () => { @@ -63,14 +63,14 @@ describe("handleWebSearchCommand", () => { max_depth: 4, max_steps_per_call: 100, reasoning_effort: "high", - web_search_provider: "firecrawl", + web_search_provider: "brave", demo: false, })); __setHandler("save_settings", ({ settings }: { settings: Record }) => { - expect(settings.web_search_provider).toBe("firecrawl"); + expect(settings.web_search_provider).toBe("brave"); }); - const result = await handleWebSearchCommand("firecrawl --save"); + const result = await handleWebSearchCommand("brave --save"); expect(result.lines).toContain("(Settings saved)"); }); }); diff --git a/openplanter-desktop/frontend/src/commands/webSearch.ts b/openplanter-desktop/frontend/src/commands/webSearch.ts index 5a475eb0..c18ed806 100644 --- a/openplanter-desktop/frontend/src/commands/webSearch.ts +++ b/openplanter-desktop/frontend/src/commands/webSearch.ts @@ -3,7 +3,7 @@ import { saveSettings, updateConfig } from "../api/invoke"; import { appState } from "../state/store"; import type { CommandResult } from "./model"; -const VALID_WEB_SEARCH_PROVIDERS = ["exa", "firecrawl"]; +const VALID_WEB_SEARCH_PROVIDERS = ["exa", "firecrawl", "brave"]; /** Handle /web-search [provider] [--save]. */ export async function handleWebSearchCommand(args: string): Promise { diff --git a/openplanter-desktop/frontend/src/components/App.test.ts b/openplanter-desktop/frontend/src/components/App.test.ts index d641b919..f0323542 100644 --- a/openplanter-desktop/frontend/src/components/App.test.ts +++ b/openplanter-desktop/frontend/src/components/App.test.ts @@ -48,7 +48,7 @@ describe("createApp", () => { __setHandler("list_sessions", () => [SESSION_B, SESSION_A]); __setHandler("get_credentials_status", () => ({ openai: true, anthropic: true, openrouter: false, - cerebras: false, zai: true, ollama: true, exa: false, firecrawl: true, + cerebras: false, zai: true, ollama: true, exa: false, firecrawl: true, brave: false, })); __setHandler("open_session", () => ({ id: "20260227-120000-cccc3333", @@ -103,7 +103,7 @@ describe("createApp", () => { await vi.waitFor(() => { const creds = root.querySelector(".cred-status"); - expect(creds!.children.length).toBe(8); + expect(creds!.children.length).toBe(9); expect(creds!.querySelector(".cred-ok")!.textContent).toContain("openai"); expect(creds!.querySelector(".cred-missing")!.textContent).toContain("openrouter"); }); diff --git a/openplanter-desktop/frontend/src/components/App.ts b/openplanter-desktop/frontend/src/components/App.ts index 9f5ef663..9e08f564 100644 --- a/openplanter-desktop/frontend/src/components/App.ts +++ b/openplanter-desktop/frontend/src/components/App.ts @@ -302,7 +302,7 @@ async function loadCredentials(container: HTMLElement): Promise { try { const status = await getCredentialsStatus(); container.innerHTML = ""; - const providers = ["openai", "anthropic", "openrouter", "cerebras", "zai", "ollama", "exa", "firecrawl"]; + const providers = ["openai", "anthropic", "openrouter", "cerebras", "zai", "ollama", "exa", "firecrawl", "brave"]; for (const p of providers) { const row = document.createElement("div"); const hasKey = status[p] ?? false; diff --git a/tests/test_coverage_gaps.py b/tests/test_coverage_gaps.py index 23c49947..15d36b62 100644 --- a/tests/test_coverage_gaps.py +++ b/tests/test_coverage_gaps.py @@ -69,11 +69,12 @@ def test_inner_quotes_preserved(self) -> None: class MergeMissingTests(unittest.TestCase): def test_fills_missing_keys(self) -> None: a = CredentialBundle(openai_api_key="oa") - b = CredentialBundle(anthropic_api_key="an", exa_api_key="exa") + b = CredentialBundle(anthropic_api_key="an", exa_api_key="exa", brave_api_key="brave") a.merge_missing(b) self.assertEqual(a.openai_api_key, "oa") self.assertEqual(a.anthropic_api_key, "an") self.assertEqual(a.exa_api_key, "exa") + self.assertEqual(a.brave_api_key, "brave") def test_does_not_overwrite_existing(self) -> None: a = CredentialBundle(openai_api_key="mine") @@ -95,6 +96,7 @@ def test_merge_all_fields(self) -> None: openrouter_api_key="or", cerebras_api_key="cb", exa_api_key="exa", + brave_api_key="brave", ) a.merge_missing(b) self.assertEqual(a.openai_api_key, "oa") @@ -102,6 +104,7 @@ def test_merge_all_fields(self) -> None: self.assertEqual(a.openrouter_api_key, "or") self.assertEqual(a.cerebras_api_key, "cb") self.assertEqual(a.exa_api_key, "exa") + self.assertEqual(a.brave_api_key, "brave") # --------------------------------------------------------------------------- @@ -116,6 +119,7 @@ def test_reads_standard_env_vars(self) -> None: "ANTHROPIC_API_KEY": "an-key", "OPENROUTER_API_KEY": "or-key", "EXA_API_KEY": "exa-key", + "BRAVE_API_KEY": "brave-key", } with patch.dict(os.environ, env, clear=True): creds = credentials_from_env() @@ -123,6 +127,7 @@ def test_reads_standard_env_vars(self) -> None: self.assertEqual(creds.anthropic_api_key, "an-key") self.assertEqual(creds.openrouter_api_key, "or-key") self.assertEqual(creds.exa_api_key, "exa-key") + self.assertEqual(creds.brave_api_key, "brave-key") def test_rlm_prefix_takes_priority(self) -> None: env = { @@ -241,6 +246,7 @@ def test_api_keys_from_env(self) -> None: "ANTHROPIC_API_KEY": "an", "OPENROUTER_API_KEY": "or", "EXA_API_KEY": "exa", + "BRAVE_API_KEY": "brave", } with patch.dict(os.environ, env, clear=True): cfg = AgentConfig.from_env("/tmp/test-ws") @@ -248,6 +254,7 @@ def test_api_keys_from_env(self) -> None: self.assertEqual(cfg.anthropic_api_key, "an") self.assertEqual(cfg.openrouter_api_key, "or") self.assertEqual(cfg.exa_api_key, "exa") + self.assertEqual(cfg.brave_api_key, "brave") def test_foundry_placeholder_keys_disabled_for_public_endpoints(self) -> None: env = { diff --git a/tests/test_credentials.py b/tests/test_credentials.py index 29538747..005acf47 100644 --- a/tests/test_credentials.py +++ b/tests/test_credentials.py @@ -25,6 +25,7 @@ def test_parse_env_file_extracts_supported_keys(self) -> None: "ZAI_API_KEY=zai-key", "EXA_API_KEY=exa-key", "FIRECRAWL_API_KEY=fc-key", + "BRAVE_API_KEY=brave-key", ] ), encoding="utf-8", @@ -36,6 +37,7 @@ def test_parse_env_file_extracts_supported_keys(self) -> None: self.assertEqual(creds.zai_api_key, "zai-key") self.assertEqual(creds.exa_api_key, "exa-key") self.assertEqual(creds.firecrawl_api_key, "fc-key") + self.assertEqual(creds.brave_api_key, "brave-key") def test_store_roundtrip(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -48,6 +50,7 @@ def test_store_roundtrip(self) -> None: zai_api_key="zai", exa_api_key="exa", firecrawl_api_key="fc", + brave_api_key="brave", ) store.save(creds) loaded = store.load() diff --git a/tests/test_tools.py b/tests/test_tools.py index a5590a56..6a5f9887 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -154,6 +154,55 @@ def test_fetch_url_with_mocked_firecrawl_response(self) -> None: self.assertEqual(parsed["pages"][0]["url"], "https://example.com") self.assertEqual(parsed["pages"][0]["text"], "Page body") + def test_web_search_with_mocked_brave_response(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + tools = WorkspaceTools( + root=root, + web_search_provider="brave", + brave_api_key="brave-key", + ) + mocked = { + "web": { + "results": [ + { + "url": "https://example.com/brave", + "title": "Brave Result", + "description": "Snippet", + "extra_snippets": ["Extra context"], + } + ] + } + } + with patch.object(WorkspaceTools, "_brave_request", return_value=mocked): + raw = tools.web_search("test query", num_results=3, include_text=True) + parsed = json.loads(raw) + self.assertEqual(parsed["provider"], "brave") + self.assertEqual(parsed["query"], "test query") + self.assertEqual(parsed["total"], 1) + self.assertEqual(parsed["results"][0]["url"], "https://example.com/brave") + self.assertIn("Extra context", parsed["results"][0]["text"]) + + def test_fetch_url_with_mocked_brave_response(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + tools = WorkspaceTools( + root=root, + web_search_provider="brave", + ) + mocked = { + "url": "https://example.com/brave", + "title": "Brave Example", + "text": "Page body", + } + with patch.object(WorkspaceTools, "_fetch_url_direct", return_value=mocked): + raw = tools.fetch_url(["https://example.com/brave"]) + parsed = json.loads(raw) + self.assertEqual(parsed["provider"], "brave") + self.assertEqual(parsed["total"], 1) + self.assertEqual(parsed["pages"][0]["title"], "Brave Example") + self.assertEqual(parsed["pages"][0]["text"], "Page body") + def test_web_search_without_exa_key(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -168,6 +217,13 @@ def test_web_search_without_firecrawl_key(self) -> None: out = tools.web_search("test") self.assertIn("FIRECRAWL_API_KEY not configured", out) + def test_web_search_without_brave_key(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + tools = WorkspaceTools(root=root, web_search_provider="brave", brave_api_key=None) + out = tools.web_search("test") + self.assertIn("BRAVE_API_KEY not configured", out) + def test_repo_map_python_symbols(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) diff --git a/tests/test_tools_complex.py b/tests/test_tools_complex.py index edb258a5..bf1352ff 100644 --- a/tests/test_tools_complex.py +++ b/tests/test_tools_complex.py @@ -140,6 +140,20 @@ def test_web_search_clamps_num_results_firecrawl(self) -> None: payload = mock_fc.call_args[0][1] self.assertEqual(payload["limit"], 20) + def test_web_search_clamps_num_results_brave(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + tools = WorkspaceTools( + root=Path(tmpdir), web_search_provider="brave", brave_api_key="test-key" + ) + mock_response = {"web": {"results": []}} + with patch.object( + WorkspaceTools, "_brave_request", return_value=mock_response + ) as mock_brave: + tools.web_search("test query", num_results=50) + mock_brave.assert_called_once() + payload = mock_brave.call_args[0][1] + self.assertEqual(payload["count"], 20) + # 12 def test_fetch_url_non_list_returns_error(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -185,6 +199,13 @@ def test_firecrawl_request_no_key_raises(self) -> None: tools._firecrawl_request("/search", {"query": "test"}) self.assertIn("FIRECRAWL_API_KEY not configured", str(ctx.exception)) + def test_brave_request_no_key_raises(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + tools = WorkspaceTools(root=Path(tmpdir), brave_api_key=None) + with self.assertRaises(ToolError) as ctx: + tools._brave_request("/web/search", {"q": "test"}) + self.assertIn("BRAVE_API_KEY not configured", str(ctx.exception)) + # 16 def test_write_file_creates_nested_dirs(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: From a384928a7c4fe9ac7f892308549664cb1161eb7e Mon Sep 17 00:00:00 2001 From: Drake Thomsen <120344051+ThomsenDrake@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:11:38 -0400 Subject: [PATCH 05/58] Add GPT-5.4 alias and ChatGPT OAuth support for OpenAI auth --- README.md | 4 +++- agent/__main__.py | 16 ++++++++++++++-- agent/credentials.py | 17 +++++++++++++++++ agent/tui.py | 1 + .../crates/op-core/src/config.rs | 6 ++++-- .../crates/op-core/src/credentials.rs | 7 +++++++ .../crates/op-tauri/src/commands/config.rs | 5 ++++- .../crates/op-tauri/src/state.rs | 10 +++++++++- .../frontend/src/commands/model.test.ts | 4 ++++ .../frontend/src/commands/model.ts | 2 ++ tests/test_credentials.py | 3 +++ 11 files changed, 68 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bfede85c..68fbffc3 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ The container mounts `./workspace` as the agent's working directory. | Provider | Default Model | Env Var | |----------|---------------|---------| -| OpenAI | `azure-foundry/gpt-5.3-codex` | `OPENAI_API_KEY` | +| OpenAI | `azure-foundry/gpt-5.3-codex` | `OPENAI_API_KEY` or `OPENAI_OAUTH_TOKEN` | | Anthropic | `anthropic-foundry/claude-opus-4-6` | `ANTHROPIC_API_KEY` | | OpenRouter | `anthropic/claude-sonnet-4-5` | `OPENROUTER_API_KEY` | | Cerebras | `qwen-3-235b-a22b-instruct-2507` | `CEREBRAS_API_KEY` | @@ -96,6 +96,8 @@ OpenAI-compatible requests now default to the Azure Foundry proxy at default to the Anthropic Foundry proxy at `https://foundry-proxy.cheetah-koi.ts.net/anthropic/v1`. +For OpenAI-compatible access, you can authenticate with either a standard API key or a ChatGPT OAuth token (Plus/Pro/Teams): `OPENAI_OAUTH_TOKEN` (or `OPENPLANTER_OPENAI_OAUTH_TOKEN`). + ### Local Models (Ollama) [Ollama](https://ollama.com) runs models locally with no API key. Install Ollama, pull a model (`ollama pull llama3.2`), then: diff --git a/agent/__main__.py b/agent/__main__.py index 728397a5..bf07fd29 100644 --- a/agent/__main__.py +++ b/agent/__main__.py @@ -89,6 +89,10 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--base-url", help="Provider base URL override for this run.") parser.add_argument("--api-key", help="Legacy API key alias (maps to OpenAI).") parser.add_argument("--openai-api-key", help="OpenAI API key override.") + parser.add_argument( + "--openai-oauth-token", + help="ChatGPT OAuth token (Plus/Pro/Teams) override for OpenAI-compatible endpoints.", + ) parser.add_argument("--anthropic-api-key", help="Anthropic API key override.") parser.add_argument("--openrouter-api-key", help="OpenRouter API key override.") parser.add_argument("--cerebras-api-key", help="Cerebras API key override.") @@ -185,7 +189,7 @@ def _resolve_provider(requested: str, creds: CredentialBundle) -> str: return requested if requested == "all": return "all" - if creds.openai_api_key: + if creds.openai_api_key or creds.openai_oauth_token: return "openai" if creds.anthropic_api_key: return "anthropic" @@ -236,6 +240,7 @@ def _load_credentials( creds = CredentialBundle( openai_api_key=user_creds.openai_api_key, + openai_oauth_token=user_creds.openai_oauth_token, anthropic_api_key=user_creds.anthropic_api_key, openrouter_api_key=user_creds.openrouter_api_key, cerebras_api_key=user_creds.cerebras_api_key, @@ -250,6 +255,8 @@ def _load_credentials( stored = store.load() if stored.openai_api_key: creds.openai_api_key = stored.openai_api_key + if stored.openai_oauth_token: + creds.openai_oauth_token = stored.openai_oauth_token if stored.anthropic_api_key: creds.anthropic_api_key = stored.anthropic_api_key if stored.openrouter_api_key: @@ -270,6 +277,8 @@ def _load_credentials( env_creds = credentials_from_env() if env_creds.openai_api_key: creds.openai_api_key = env_creds.openai_api_key + if env_creds.openai_oauth_token: + creds.openai_oauth_token = env_creds.openai_oauth_token if env_creds.anthropic_api_key: creds.anthropic_api_key = env_creds.anthropic_api_key if env_creds.openrouter_api_key: @@ -295,6 +304,8 @@ def _load_credentials( creds.openai_api_key = args.api_key.strip() or creds.openai_api_key if args.openai_api_key: creds.openai_api_key = args.openai_api_key.strip() or creds.openai_api_key + if args.openai_oauth_token: + creds.openai_oauth_token = args.openai_oauth_token.strip() or creds.openai_oauth_token if args.anthropic_api_key: creds.anthropic_api_key = args.anthropic_api_key.strip() or creds.anthropic_api_key if args.openrouter_api_key: @@ -347,7 +358,8 @@ def _apply_runtime_overrides(cfg: AgentConfig, args: argparse.Namespace, creds: cfg.provider = args.provider cfg.provider = _resolve_provider(cfg.provider, creds) - cfg.openai_api_key = resolve_openai_api_key(creds.openai_api_key, cfg.openai_base_url) + effective_openai_key = creds.openai_api_key or creds.openai_oauth_token + cfg.openai_api_key = resolve_openai_api_key(effective_openai_key, cfg.openai_base_url) cfg.anthropic_api_key = resolve_anthropic_api_key( creds.anthropic_api_key, cfg.anthropic_base_url, diff --git a/agent/credentials.py b/agent/credentials.py index a714f82d..2e4d8a40 100644 --- a/agent/credentials.py +++ b/agent/credentials.py @@ -12,6 +12,7 @@ @dataclass(slots=True) class CredentialBundle: openai_api_key: str | None = None + openai_oauth_token: str | None = None anthropic_api_key: str | None = None openrouter_api_key: str | None = None cerebras_api_key: str | None = None @@ -24,6 +25,7 @@ class CredentialBundle: def has_any(self) -> bool: return bool( (self.openai_api_key and self.openai_api_key.strip()) + or (self.openai_oauth_token and self.openai_oauth_token.strip()) or (self.anthropic_api_key and self.anthropic_api_key.strip()) or (self.openrouter_api_key and self.openrouter_api_key.strip()) or (self.cerebras_api_key and self.cerebras_api_key.strip()) @@ -37,6 +39,8 @@ def has_any(self) -> bool: def merge_missing(self, other: "CredentialBundle") -> None: if not self.openai_api_key and other.openai_api_key: self.openai_api_key = other.openai_api_key + if not self.openai_oauth_token and other.openai_oauth_token: + self.openai_oauth_token = other.openai_oauth_token if not self.anthropic_api_key and other.anthropic_api_key: self.anthropic_api_key = other.anthropic_api_key if not self.openrouter_api_key and other.openrouter_api_key: @@ -58,6 +62,8 @@ def to_json(self) -> dict[str, str]: out: dict[str, str] = {} if self.openai_api_key: out["openai_api_key"] = self.openai_api_key + if self.openai_oauth_token: + out["openai_oauth_token"] = self.openai_oauth_token if self.anthropic_api_key: out["anthropic_api_key"] = self.anthropic_api_key if self.openrouter_api_key: @@ -82,6 +88,7 @@ def from_json(cls, payload: dict[str, str] | None) -> "CredentialBundle": return cls() return cls( openai_api_key=(payload.get("openai_api_key") or "").strip() or None, + openai_oauth_token=(payload.get("openai_oauth_token") or "").strip() or None, anthropic_api_key=(payload.get("anthropic_api_key") or "").strip() or None, openrouter_api_key=(payload.get("openrouter_api_key") or "").strip() or None, cerebras_api_key=(payload.get("cerebras_api_key") or "").strip() or None, @@ -124,6 +131,10 @@ def parse_env_file(path: Path) -> CredentialBundle: return CredentialBundle( openai_api_key=(env.get("OPENAI_API_KEY") or env.get("OPENPLANTER_OPENAI_API_KEY") or "").strip() or None, + openai_oauth_token=( + env.get("OPENAI_OAUTH_TOKEN") or env.get("OPENPLANTER_OPENAI_OAUTH_TOKEN") or "" + ).strip() + or None, anthropic_api_key=(env.get("ANTHROPIC_API_KEY") or env.get("OPENPLANTER_ANTHROPIC_API_KEY") or "").strip() or None, openrouter_api_key=(env.get("OPENROUTER_API_KEY") or env.get("OPENPLANTER_OPENROUTER_API_KEY") or "").strip() @@ -147,6 +158,10 @@ def credentials_from_env() -> CredentialBundle: or "" ).strip() or None, + openai_oauth_token=( + os.getenv("OPENPLANTER_OPENAI_OAUTH_TOKEN") or os.getenv("OPENAI_OAUTH_TOKEN") or "" + ).strip() + or None, anthropic_api_key=( os.getenv("OPENPLANTER_ANTHROPIC_API_KEY") or os.getenv("ANTHROPIC_API_KEY") or "" ).strip() @@ -260,6 +275,7 @@ def prompt_for_credentials( """ current = CredentialBundle( openai_api_key=existing.openai_api_key, + openai_oauth_token=existing.openai_oauth_token, anthropic_api_key=existing.anthropic_api_key, openrouter_api_key=existing.openrouter_api_key, cerebras_api_key=existing.cerebras_api_key, @@ -296,6 +312,7 @@ def _ask(label: str, existing_value: str | None) -> str | None: return value current.openai_api_key = _ask("OpenAI", current.openai_api_key) + current.openai_oauth_token = _ask("ChatGPT OAuth (Plus/Pro/Teams)", current.openai_oauth_token) current.anthropic_api_key = _ask("Anthropic", current.anthropic_api_key) current.openrouter_api_key = _ask("OpenRouter", current.openrouter_api_key) current.cerebras_api_key = _ask("Cerebras", current.cerebras_api_key) diff --git a/agent/tui.py b/agent/tui.py index c1a63be2..05a64740 100644 --- a/agent/tui.py +++ b/agent/tui.py @@ -118,6 +118,7 @@ def _build_splash() -> str: "haiku4.5": "anthropic-foundry/claude-haiku-4-5", "gpt5": "azure-foundry/gpt-5.3-codex", "gpt5.3": "azure-foundry/gpt-5.3-codex", + "gpt5.4": "azure-foundry/gpt-5.4", "kimi": "azure-foundry/Kimi-K2.5", "gpt4": "gpt-4.1", "gpt4.1": "gpt-4.1", diff --git a/openplanter-desktop/crates/op-core/src/config.rs b/openplanter-desktop/crates/op-core/src/config.rs index e75b21a1..0d09b42f 100644 --- a/openplanter-desktop/crates/op-core/src/config.rs +++ b/openplanter-desktop/crates/op-core/src/config.rs @@ -272,8 +272,10 @@ impl AgentConfig { pub fn from_env(workspace: impl AsRef) -> Self { let ws = dunce_canonicalize(workspace.as_ref()); - let openai_api_key = - env_opt("OPENPLANTER_OPENAI_API_KEY").or_else(|| env_opt("OPENAI_API_KEY")); + let openai_api_key = env_opt("OPENPLANTER_OPENAI_API_KEY") + .or_else(|| env_opt("OPENAI_API_KEY")) + .or_else(|| env_opt("OPENPLANTER_OPENAI_OAUTH_TOKEN")) + .or_else(|| env_opt("OPENAI_OAUTH_TOKEN")); let anthropic_api_key = env_opt("OPENPLANTER_ANTHROPIC_API_KEY").or_else(|| env_opt("ANTHROPIC_API_KEY")); diff --git a/openplanter-desktop/crates/op-core/src/credentials.rs b/openplanter-desktop/crates/op-core/src/credentials.rs index d7ce3e52..32ef2d53 100644 --- a/openplanter-desktop/crates/op-core/src/credentials.rs +++ b/openplanter-desktop/crates/op-core/src/credentials.rs @@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct CredentialBundle { pub openai_api_key: Option, + pub openai_oauth_token: Option, pub anthropic_api_key: Option, pub openrouter_api_key: Option, pub cerebras_api_key: Option, @@ -28,6 +29,7 @@ impl CredentialBundle { pub fn has_any(&self) -> bool { let keys: [&Option; 9] = [ &self.openai_api_key, + &self.openai_oauth_token, &self.anthropic_api_key, &self.openrouter_api_key, &self.cerebras_api_key, @@ -51,6 +53,7 @@ impl CredentialBundle { }; } fill!(openai_api_key); + fill!(openai_oauth_token); fill!(anthropic_api_key); fill!(openrouter_api_key); fill!(cerebras_api_key); @@ -72,6 +75,7 @@ impl CredentialBundle { }; } add!(openai_api_key, "openai_api_key"); + add!(openai_oauth_token, "openai_oauth_token"); add!(anthropic_api_key, "anthropic_api_key"); add!(openrouter_api_key, "openrouter_api_key"); add!(cerebras_api_key, "cerebras_api_key"); @@ -93,6 +97,7 @@ impl CredentialBundle { } Self { openai_api_key: get_str(payload, "openai_api_key"), + openai_oauth_token: get_str(payload, "openai_oauth_token"), anthropic_api_key: get_str(payload, "anthropic_api_key"), openrouter_api_key: get_str(payload, "openrouter_api_key"), cerebras_api_key: get_str(payload, "cerebras_api_key"), @@ -148,6 +153,7 @@ pub fn parse_env_file(path: &Path) -> CredentialBundle { CredentialBundle { openai_api_key: get_key(&env_map, "OPENAI_API_KEY", "OPENPLANTER_OPENAI_API_KEY"), + openai_oauth_token: get_key(&env_map, "OPENAI_OAUTH_TOKEN", "OPENPLANTER_OPENAI_OAUTH_TOKEN"), anthropic_api_key: get_key( &env_map, "ANTHROPIC_API_KEY", @@ -183,6 +189,7 @@ pub fn credentials_from_env() -> CredentialBundle { CredentialBundle { openai_api_key: env_key("OPENPLANTER_OPENAI_API_KEY", "OPENAI_API_KEY"), + openai_oauth_token: env_key("OPENPLANTER_OPENAI_OAUTH_TOKEN", "OPENAI_OAUTH_TOKEN"), anthropic_api_key: env_key("OPENPLANTER_ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY"), openrouter_api_key: env_key("OPENPLANTER_OPENROUTER_API_KEY", "OPENROUTER_API_KEY"), cerebras_api_key: env_key("OPENPLANTER_CEREBRAS_API_KEY", "CEREBRAS_API_KEY"), diff --git a/openplanter-desktop/crates/op-tauri/src/commands/config.rs b/openplanter-desktop/crates/op-tauri/src/commands/config.rs index 7224456c..93d0a009 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/config.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/config.rs @@ -102,6 +102,7 @@ fn known_models_for_provider(provider: &str) -> Vec { let models: Vec<(&str, &str)> = match provider { "openai" => vec![ ("azure-foundry/gpt-5.3-codex", "GPT-5.3 Codex (Foundry)"), + ("azure-foundry/gpt-5.4", "GPT-5.4 (Foundry)"), ("azure-foundry/Kimi-K2.5", "Kimi K2.5 (Foundry)"), ], "anthropic" => vec![ @@ -215,7 +216,9 @@ pub async fn get_credentials_status( let mut status = HashMap::new(); status.insert( "openai".to_string(), - cfg.openai_api_key.is_some() || env_creds.openai_api_key.is_some(), + cfg.openai_api_key.is_some() + || env_creds.openai_api_key.is_some() + || env_creds.openai_oauth_token.is_some(), ); status.insert( "anthropic".to_string(), diff --git a/openplanter-desktop/crates/op-tauri/src/state.rs b/openplanter-desktop/crates/op-tauri/src/state.rs index 387bb719..f8869196 100644 --- a/openplanter-desktop/crates/op-tauri/src/state.rs +++ b/openplanter-desktop/crates/op-tauri/src/state.rs @@ -43,6 +43,15 @@ pub fn merge_credentials_into_config( env_creds: &CredentialBundle, file_creds: &CredentialBundle, ) { + if cfg.openai_api_key.is_none() { + cfg.openai_api_key = env_creds + .openai_api_key + .clone() + .or_else(|| env_creds.openai_oauth_token.clone()) + .or_else(|| file_creds.openai_api_key.clone()) + .or_else(|| file_creds.openai_oauth_token.clone()); + } + macro_rules! merge { ($field:ident) => { if cfg.$field.is_none() { @@ -53,7 +62,6 @@ pub fn merge_credentials_into_config( } }; } - merge!(openai_api_key); merge!(anthropic_api_key); merge!(openrouter_api_key); merge!(cerebras_api_key); diff --git a/openplanter-desktop/frontend/src/commands/model.test.ts b/openplanter-desktop/frontend/src/commands/model.test.ts index f98dfb86..31eacdd6 100644 --- a/openplanter-desktop/frontend/src/commands/model.test.ts +++ b/openplanter-desktop/frontend/src/commands/model.test.ts @@ -66,6 +66,10 @@ describe("MODEL_ALIASES", () => { expect(MODEL_ALIASES["gpt5"]).toBe("azure-foundry/gpt-5.3-codex"); }); + it("gpt-5.4 alias", () => { + expect(MODEL_ALIASES["gpt-5.4"]).toBe("azure-foundry/gpt-5.4"); + }); + it("zai alias", () => { expect(MODEL_ALIASES["zai"]).toBe("glm-5"); }); diff --git a/openplanter-desktop/frontend/src/commands/model.ts b/openplanter-desktop/frontend/src/commands/model.ts index 45f7016d..68b45bab 100644 --- a/openplanter-desktop/frontend/src/commands/model.ts +++ b/openplanter-desktop/frontend/src/commands/model.ts @@ -14,6 +14,8 @@ export const MODEL_ALIASES: Record = { gpt5: "azure-foundry/gpt-5.3-codex", "gpt-5": "azure-foundry/gpt-5.3-codex", "gpt-5.3": "azure-foundry/gpt-5.3-codex", + gpt54: "azure-foundry/gpt-5.4", + "gpt-5.4": "azure-foundry/gpt-5.4", kimi: "azure-foundry/Kimi-K2.5", gpt4o: "gpt-4o", "gpt-4o": "gpt-4o", diff --git a/tests/test_credentials.py b/tests/test_credentials.py index 005acf47..6d729824 100644 --- a/tests/test_credentials.py +++ b/tests/test_credentials.py @@ -20,6 +20,7 @@ def test_parse_env_file_extracts_supported_keys(self) -> None: "\n".join( [ "OPENAI_API_KEY=oa-key", + "OPENAI_OAUTH_TOKEN=oauth-token", "ANTHROPIC_API_KEY=an-key", "OPENROUTER_API_KEY=or-key", "ZAI_API_KEY=zai-key", @@ -32,6 +33,7 @@ def test_parse_env_file_extracts_supported_keys(self) -> None: ) creds = parse_env_file(env_path) self.assertEqual(creds.openai_api_key, "oa-key") + self.assertEqual(creds.openai_oauth_token, "oauth-token") self.assertEqual(creds.anthropic_api_key, "an-key") self.assertEqual(creds.openrouter_api_key, "or-key") self.assertEqual(creds.zai_api_key, "zai-key") @@ -45,6 +47,7 @@ def test_store_roundtrip(self) -> None: store = CredentialStore(workspace=root, session_root_dir=".openplanter") creds = CredentialBundle( openai_api_key="oa", + openai_oauth_token="oauth", anthropic_api_key="an", openrouter_api_key="or", zai_api_key="zai", From bfd94e2a4ec65d40aea54795376ff9bf948c1cb3 Mon Sep 17 00:00:00 2001 From: Drake Date: Thu, 12 Mar 2026 12:00:58 -0400 Subject: [PATCH 06/58] Add GPT-5.4 defaults and OAuth parity --- README.md | 3 +- agent/__main__.py | 20 +++- agent/builder.py | 24 +++- agent/config.py | 39 +++++-- agent/tui.py | 5 +- .../crates/op-core/src/builder.rs | 28 +++-- .../crates/op-core/src/config.rs | 104 ++++++++++++++---- .../crates/op-core/src/credentials.rs | 6 +- .../crates/op-core/src/model/openai.rs | 6 +- .../op-core/tests/test_model_streaming.rs | 6 +- .../crates/op-tauri/src/commands/config.rs | 47 +++++++- .../crates/op-tauri/src/state.rs | 72 +++++++++++- .../frontend/src/api/invoke.test.ts | 14 +-- .../frontend/src/commands/model.test.ts | 41 ++++++- .../frontend/src/commands/model.ts | 7 +- tests/test_coverage_gaps.py | 23 +++- tests/test_model.py | 4 +- tests/test_settings.py | 2 +- 18 files changed, 367 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 68fbffc3..e2fef280 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ The container mounts `./workspace` as the agent's working directory. | Provider | Default Model | Env Var | |----------|---------------|---------| -| OpenAI | `azure-foundry/gpt-5.3-codex` | `OPENAI_API_KEY` or `OPENAI_OAUTH_TOKEN` | +| OpenAI | `azure-foundry/gpt-5.4` | `OPENAI_API_KEY` or `OPENAI_OAUTH_TOKEN` | | Anthropic | `anthropic-foundry/claude-opus-4-6` | `ANTHROPIC_API_KEY` | | OpenRouter | `anthropic/claude-sonnet-4-5` | `OPENROUTER_API_KEY` | | Cerebras | `qwen-3-235b-a22b-instruct-2507` | `CEREBRAS_API_KEY` | @@ -189,6 +189,7 @@ openplanter-agent [options] |------|-------------| | `--provider NAME` | `auto`, `openai`, `anthropic`, `openrouter`, `cerebras`, `zai`, `ollama` | | `--model NAME` | Model name or `newest` to auto-select | +| `--openai-oauth-token TOKEN` | ChatGPT Plus/Teams/Pro OAuth bearer token for OpenAI-compatible models | | `--zai-plan PLAN` | Z.AI endpoint plan: `paygo` or `coding` | | `--reasoning-effort LEVEL` | `low`, `medium`, `high`, or `none` | | `--list-models` | Fetch available models from the provider API | diff --git a/agent/__main__.py b/agent/__main__.py index bf07fd29..07d5b3d2 100644 --- a/agent/__main__.py +++ b/agent/__main__.py @@ -358,8 +358,12 @@ def _apply_runtime_overrides(cfg: AgentConfig, args: argparse.Namespace, creds: cfg.provider = args.provider cfg.provider = _resolve_provider(cfg.provider, creds) - effective_openai_key = creds.openai_api_key or creds.openai_oauth_token - cfg.openai_api_key = resolve_openai_api_key(effective_openai_key, cfg.openai_base_url) + cfg.openai_oauth_token = (creds.openai_oauth_token or "").strip() or None + cfg.openai_api_key = resolve_openai_api_key( + creds.openai_api_key, + cfg.openai_base_url, + cfg.openai_oauth_token, + ) cfg.anthropic_api_key = resolve_anthropic_api_key( creds.anthropic_api_key, cfg.anthropic_base_url, @@ -396,12 +400,20 @@ def _apply_runtime_overrides(cfg: AgentConfig, args: argparse.Namespace, creds: cfg.ollama_base_url = args.base_url cfg.base_url = args.base_url - cfg.openai_api_key = resolve_openai_api_key(cfg.openai_api_key, cfg.openai_base_url) + cfg.openai_api_key = resolve_openai_api_key( + cfg.openai_api_key, + cfg.openai_base_url, + cfg.openai_oauth_token, + ) cfg.anthropic_api_key = resolve_anthropic_api_key( cfg.anthropic_api_key, cfg.anthropic_base_url, ) - cfg.api_key = resolve_openai_api_key(cfg.api_key, cfg.base_url) + cfg.api_key = resolve_openai_api_key( + cfg.api_key, + cfg.base_url, + cfg.openai_oauth_token, + ) if args.model: cfg.model = args.model diff --git a/agent/builder.py b/agent/builder.py index 89671221..7d7044ac 100644 --- a/agent/builder.py +++ b/agent/builder.py @@ -83,9 +83,17 @@ def _validate_model_provider(model_name: str, provider: str) -> None: def _fetch_models_for_provider(cfg: AgentConfig, provider: str) -> list[dict]: if provider == "openai": - api_key = resolve_openai_api_key(cfg.openai_api_key, cfg.openai_base_url) + api_key = resolve_openai_api_key( + cfg.openai_api_key, + cfg.openai_base_url, + cfg.openai_oauth_token, + ) if not api_key: - raise ModelError("OpenAI key not configured.") + raise ModelError( + "OpenAI auth not configured. Set OPENAI_API_KEY, " + "OPENPLANTER_OPENAI_API_KEY, OPENAI_OAUTH_TOKEN, " + "or OPENPLANTER_OPENAI_OAUTH_TOKEN." + ) models = list_openai_models(api_key=api_key, base_url=cfg.openai_base_url) if is_foundry_openai_base_url(cfg.openai_base_url): return [ @@ -148,7 +156,11 @@ def _resolve_model_name(cfg: AgentConfig) -> str: def build_model_factory(cfg: AgentConfig) -> ModelFactory | None: """Return a factory that creates models by name + optional reasoning effort.""" - openai_api_key = resolve_openai_api_key(cfg.openai_api_key, cfg.openai_base_url) + openai_api_key = resolve_openai_api_key( + cfg.openai_api_key, + cfg.openai_base_url, + cfg.openai_oauth_token, + ) anthropic_api_key = resolve_anthropic_api_key(cfg.anthropic_api_key, cfg.anthropic_base_url) def _factory(model_name: str, reasoning_effort: str | None = None) -> AnthropicModel | OpenAICompatibleModel: @@ -247,7 +259,11 @@ def build_engine(cfg: AgentConfig) -> RLMEngine: _validate_model_provider(model_name, cfg.provider) - openai_api_key = resolve_openai_api_key(cfg.openai_api_key, cfg.openai_base_url) + openai_api_key = resolve_openai_api_key( + cfg.openai_api_key, + cfg.openai_base_url, + cfg.openai_oauth_token, + ) anthropic_api_key = resolve_anthropic_api_key(cfg.anthropic_api_key, cfg.anthropic_base_url) if cfg.provider == "openai" and openai_api_key: diff --git a/agent/config.py b/agent/config.py index 50290176..6a0e0f9c 100644 --- a/agent/config.py +++ b/agent/config.py @@ -14,7 +14,7 @@ ZAI_CODING_BASE_URL = "https://api.z.ai/api/coding/paas/v4" PROVIDER_DEFAULT_MODELS: dict[str, str] = { - "openai": "azure-foundry/gpt-5.3-codex", + "openai": "azure-foundry/gpt-5.4", "anthropic": "anthropic-foundry/claude-opus-4-6", "openrouter": "anthropic/claude-sonnet-4-5", "cerebras": "qwen-3-235b-a22b-instruct-2507", @@ -50,12 +50,19 @@ def is_foundry_anthropic_base_url(url: str) -> bool: return _normalize_base_url(url) == FOUNDRY_ANTHROPIC_BASE_URL -def resolve_openai_api_key(api_key: str | None, base_url: str) -> str | None: +def resolve_openai_api_key( + api_key: str | None, + base_url: str, + openai_oauth_token: str | None = None, +) -> str | None: key = (api_key or "").strip() or None - if key == FOUNDRY_OPENAI_API_KEY_PLACEHOLDER and not is_foundry_openai_base_url(base_url): - return None + if key == FOUNDRY_OPENAI_API_KEY_PLACEHOLDER: + key = None if key: return key + token = (openai_oauth_token or "").strip() or None + if token: + return token if is_foundry_openai_base_url(base_url): return FOUNDRY_OPENAI_API_KEY_PLACEHOLDER return None @@ -106,6 +113,7 @@ class AgentConfig: firecrawl_base_url: str = "https://api.firecrawl.dev/v1" brave_base_url: str = "https://api.search.brave.com/res/v1" openai_api_key: str | None = None + openai_oauth_token: str | None = None anthropic_api_key: str | None = None openrouter_api_key: str | None = None cerebras_api_key: str | None = None @@ -140,11 +148,19 @@ class AgentConfig: demo: bool = False def __post_init__(self) -> None: - self.openai_api_key = resolve_openai_api_key(self.openai_api_key, self.openai_base_url) + self.openai_api_key = resolve_openai_api_key( + self.openai_api_key, + self.openai_base_url, + self.openai_oauth_token, + ) self.anthropic_api_key = resolve_anthropic_api_key( self.anthropic_api_key, self.anthropic_base_url ) - self.api_key = resolve_openai_api_key(self.api_key, self.base_url) + self.api_key = resolve_openai_api_key( + self.api_key, + self.base_url, + self.openai_oauth_token, + ) @classmethod def from_env(cls, workspace: str | Path) -> "AgentConfig": @@ -153,6 +169,10 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": os.getenv("OPENPLANTER_OPENAI_API_KEY") or os.getenv("OPENAI_API_KEY") ) + openai_oauth_token = ( + os.getenv("OPENPLANTER_OPENAI_OAUTH_TOKEN") + or os.getenv("OPENAI_OAUTH_TOKEN") + ) anthropic_api_key = os.getenv("OPENPLANTER_ANTHROPIC_API_KEY") or os.getenv("ANTHROPIC_API_KEY") openrouter_api_key = os.getenv("OPENPLANTER_OPENROUTER_API_KEY") or os.getenv("OPENROUTER_API_KEY") cerebras_api_key = os.getenv("OPENPLANTER_CEREBRAS_API_KEY") or os.getenv("CEREBRAS_API_KEY") @@ -169,7 +189,11 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": "OPENPLANTER_ANTHROPIC_BASE_URL", FOUNDRY_ANTHROPIC_BASE_URL, ) - openai_api_key = resolve_openai_api_key(openai_api_key, openai_base_url) + openai_api_key = resolve_openai_api_key( + openai_api_key, + openai_base_url, + openai_oauth_token, + ) anthropic_api_key = resolve_anthropic_api_key(anthropic_api_key, anthropic_base_url) zai_plan = normalize_zai_plan(os.getenv("OPENPLANTER_ZAI_PLAN", "paygo")) zai_paygo_base_url = os.getenv("OPENPLANTER_ZAI_PAYGO_BASE_URL", ZAI_PAYGO_BASE_URL) @@ -206,6 +230,7 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": firecrawl_base_url=os.getenv("OPENPLANTER_FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v1"), brave_base_url=os.getenv("OPENPLANTER_BRAVE_BASE_URL", "https://api.search.brave.com/res/v1"), openai_api_key=openai_api_key, + openai_oauth_token=(openai_oauth_token or "").strip() or None, anthropic_api_key=anthropic_api_key, openrouter_api_key=openrouter_api_key, cerebras_api_key=cerebras_api_key, diff --git a/agent/tui.py b/agent/tui.py index 05a64740..e5f37152 100644 --- a/agent/tui.py +++ b/agent/tui.py @@ -116,9 +116,12 @@ def _build_splash() -> str: "sonnet4.6": "anthropic-foundry/claude-sonnet-4-6", "haiku": "anthropic-foundry/claude-haiku-4-5", "haiku4.5": "anthropic-foundry/claude-haiku-4-5", - "gpt5": "azure-foundry/gpt-5.3-codex", + "gpt5": "azure-foundry/gpt-5.4", + "gpt-5": "azure-foundry/gpt-5.4", "gpt5.3": "azure-foundry/gpt-5.3-codex", + "gpt-5.3": "azure-foundry/gpt-5.3-codex", "gpt5.4": "azure-foundry/gpt-5.4", + "gpt-5.4": "azure-foundry/gpt-5.4", "kimi": "azure-foundry/Kimi-K2.5", "gpt4": "gpt-4.1", "gpt4.1": "gpt-4.1", diff --git a/openplanter-desktop/crates/op-core/src/builder.rs b/openplanter-desktop/crates/op-core/src/builder.rs index 786e4c10..6bfab8a1 100644 --- a/openplanter-desktop/crates/op-core/src/builder.rs +++ b/openplanter-desktop/crates/op-core/src/builder.rs @@ -170,10 +170,11 @@ pub fn resolve_endpoint(cfg: &AgentConfig, provider: &str) -> Result<(String, St let key = resolve_openai_api_key( cfg.openai_api_key.clone().or_else(|| cfg.api_key.clone()), &cfg.openai_base_url, + cfg.openai_oauth_token.clone(), ) .ok_or_else(|| { ModelError::Message( - "No OpenAI API key. Set OPENAI_API_KEY or OPENPLANTER_OPENAI_API_KEY.".into(), + "No OpenAI auth configured. Set OPENAI_API_KEY, OPENPLANTER_OPENAI_API_KEY, OPENAI_OAUTH_TOKEN, or OPENPLANTER_OPENAI_OAUTH_TOKEN.".into(), ) })?; Ok((cfg.openai_base_url.clone(), key)) @@ -303,7 +304,7 @@ mod tests { fn test_infer_openai() { assert_eq!(infer_provider_for_model("gpt-5.2"), Some("openai")); assert_eq!( - infer_provider_for_model("azure-foundry/gpt-5.3-codex"), + infer_provider_for_model("azure-foundry/gpt-5.4"), Some("openai") ); assert_eq!(infer_provider_for_model("o1-preview"), Some("openai")); @@ -382,14 +383,11 @@ mod tests { #[test] fn test_resolve_model_name_explicit() { let cfg = AgentConfig { - model: "azure-foundry/gpt-5.3-codex".into(), + model: "azure-foundry/gpt-5.4".into(), provider: "openai".into(), ..Default::default() }; - assert_eq!( - resolve_model_name(&cfg).unwrap(), - "azure-foundry/gpt-5.3-codex" - ); + assert_eq!(resolve_model_name(&cfg).unwrap(), "azure-foundry/gpt-5.4"); } #[test] @@ -399,10 +397,7 @@ mod tests { provider: "openai".into(), ..Default::default() }; - assert_eq!( - resolve_model_name(&cfg).unwrap(), - "azure-foundry/gpt-5.3-codex" - ); + assert_eq!(resolve_model_name(&cfg).unwrap(), "azure-foundry/gpt-5.4"); } // ── resolve_provider ── @@ -530,6 +525,17 @@ mod tests { assert_eq!(key, "sk-openai"); } + #[test] + fn test_resolve_endpoint_openai_uses_oauth_token_when_api_key_missing() { + let cfg = AgentConfig { + openai_api_key: Some(crate::config::FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.into()), + openai_oauth_token: Some("oauth-token".into()), + ..Default::default() + }; + let (_, key) = resolve_endpoint(&cfg, "openai").unwrap(); + assert_eq!(key, "oauth-token"); + } + #[test] fn test_resolve_endpoint_zai() { let cfg = AgentConfig { diff --git a/openplanter-desktop/crates/op-core/src/config.rs b/openplanter-desktop/crates/op-core/src/config.rs index 0d09b42f..015acca8 100644 --- a/openplanter-desktop/crates/op-core/src/config.rs +++ b/openplanter-desktop/crates/op-core/src/config.rs @@ -20,7 +20,7 @@ pub const BRAVE_BASE_URL: &str = "https://api.search.brave.com/res/v1"; pub static PROVIDER_DEFAULT_MODELS: LazyLock> = LazyLock::new(|| { HashMap::from([ - ("openai", "azure-foundry/gpt-5.3-codex"), + ("openai", "azure-foundry/gpt-5.4"), ("anthropic", "anthropic-foundry/claude-opus-4-6"), ("openrouter", "anthropic/claude-sonnet-4-5"), ("cerebras", "qwen-3-235b-a22b-instruct-2507"), @@ -93,19 +93,42 @@ pub fn is_foundry_anthropic_base_url(value: &str) -> bool { normalize_base_url(value) == FOUNDRY_ANTHROPIC_BASE_URL } -pub fn resolve_openai_api_key(api_key: Option, base_url: &str) -> Option { +pub fn has_real_openai_api_key(api_key: Option<&str>) -> bool { + api_key + .map(str::trim) + .filter(|value| !value.is_empty()) + .is_some_and(|value| value != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER) +} + +pub fn has_openai_auth(api_key: Option<&str>, openai_oauth_token: Option<&str>) -> bool { + has_real_openai_api_key(api_key) + || openai_oauth_token + .map(str::trim) + .filter(|value| !value.is_empty()) + .is_some() +} + +pub fn resolve_openai_api_key( + api_key: Option, + base_url: &str, + openai_oauth_token: Option, +) -> Option { let normalized = api_key .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) .map(str::to_string); - if normalized.as_deref() == Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER) - && !is_foundry_openai_base_url(base_url) - { - return None; + let real_key = normalized.filter(|value| value != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER); + if real_key.is_some() { + return real_key; } - if normalized.is_some() { - return normalized; + let token = openai_oauth_token + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string); + if token.is_some() { + return token; } if is_foundry_openai_base_url(base_url) { return Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.to_string()); @@ -173,6 +196,7 @@ pub struct AgentConfig { // API keys pub api_key: Option, pub openai_api_key: Option, + pub openai_oauth_token: Option, pub anthropic_api_key: Option, pub openrouter_api_key: Option, pub cerebras_api_key: Option, @@ -231,6 +255,7 @@ impl Default for AgentConfig { brave_base_url: BRAVE_BASE_URL.into(), api_key: Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.into()), openai_api_key: Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.into()), + openai_oauth_token: None, anthropic_api_key: Some(FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER.into()), openrouter_api_key: None, cerebras_api_key: None, @@ -272,10 +297,10 @@ impl AgentConfig { pub fn from_env(workspace: impl AsRef) -> Self { let ws = dunce_canonicalize(workspace.as_ref()); - let openai_api_key = env_opt("OPENPLANTER_OPENAI_API_KEY") - .or_else(|| env_opt("OPENAI_API_KEY")) - .or_else(|| env_opt("OPENPLANTER_OPENAI_OAUTH_TOKEN")) - .or_else(|| env_opt("OPENAI_OAUTH_TOKEN")); + let openai_api_key = + env_opt("OPENPLANTER_OPENAI_API_KEY").or_else(|| env_opt("OPENAI_API_KEY")); + let openai_oauth_token = + env_opt("OPENPLANTER_OPENAI_OAUTH_TOKEN").or_else(|| env_opt("OPENAI_OAUTH_TOKEN")); let anthropic_api_key = env_opt("OPENPLANTER_ANTHROPIC_API_KEY").or_else(|| env_opt("ANTHROPIC_API_KEY")); @@ -303,7 +328,8 @@ impl AgentConfig { .unwrap_or_else(|| FOUNDRY_OPENAI_BASE_URL.into()); let anthropic_base_url = env_or("OPENPLANTER_ANTHROPIC_BASE_URL", FOUNDRY_ANTHROPIC_BASE_URL); - let openai_api_key = resolve_openai_api_key(openai_api_key, &openai_base_url); + let openai_api_key = + resolve_openai_api_key(openai_api_key, &openai_base_url, openai_oauth_token.clone()); let anthropic_api_key = resolve_anthropic_api_key(anthropic_api_key, &anthropic_base_url); let reasoning_effort_raw = env_or("OPENPLANTER_REASONING_EFFORT", "high") @@ -360,6 +386,7 @@ impl AgentConfig { ), brave_base_url: env_or("OPENPLANTER_BRAVE_BASE_URL", BRAVE_BASE_URL), openai_api_key, + openai_oauth_token, anthropic_api_key, openrouter_api_key, cerebras_api_key, @@ -461,7 +488,7 @@ mod tests { fn test_provider_default_models() { assert_eq!( PROVIDER_DEFAULT_MODELS.get("openai"), - Some(&"azure-foundry/gpt-5.3-codex") + Some(&"azure-foundry/gpt-5.4") ); assert_eq!( PROVIDER_DEFAULT_MODELS.get("anthropic"), @@ -489,6 +516,8 @@ mod tests { "OPENPLANTER_REASONING_EFFORT", "OPENPLANTER_OPENAI_API_KEY", "OPENAI_API_KEY", + "OPENPLANTER_OPENAI_OAUTH_TOKEN", + "OPENAI_OAUTH_TOKEN", "OPENPLANTER_OPENAI_BASE_URL", "OPENPLANTER_BASE_URL", "OPENPLANTER_ANTHROPIC_API_KEY", @@ -550,7 +579,7 @@ mod tests { unsafe { // --- Phase 2: test custom values --- env::set_var("OPENPLANTER_PROVIDER", "openai"); - env::set_var("OPENPLANTER_MODEL", "azure-foundry/gpt-5.3-codex"); + env::set_var("OPENPLANTER_MODEL", "azure-foundry/gpt-5.4"); env::set_var("OPENPLANTER_REASONING_EFFORT", "low"); env::set_var("OPENPLANTER_MAX_DEPTH", "8"); env::set_var("OPENPLANTER_RECURSIVE", "false"); @@ -569,7 +598,7 @@ mod tests { let cfg = AgentConfig::from_env("/tmp"); assert_eq!(cfg.provider, "openai"); - assert_eq!(cfg.model, "azure-foundry/gpt-5.3-codex"); + assert_eq!(cfg.model, "azure-foundry/gpt-5.4"); assert_eq!(cfg.reasoning_effort, Some("low".into())); assert_eq!(cfg.max_depth, 8); assert!(!cfg.recursive); @@ -587,6 +616,25 @@ mod tests { assert_eq!(cfg.rate_limit_retry_after_cap_sec, 90.0); // Restore original values + unsafe { + env::remove_var("OPENAI_API_KEY"); + env::set_var("OPENAI_OAUTH_TOKEN", "oauth-token"); + } + + let cfg = AgentConfig::from_env("/tmp"); + assert_eq!(cfg.openai_oauth_token.as_deref(), Some("oauth-token")); + assert_eq!(cfg.openai_api_key.as_deref(), Some("oauth-token")); + assert_eq!(cfg.api_key.as_deref(), Some("oauth-token")); + + unsafe { + env::set_var("OPENAI_API_KEY", "sk-test456"); + } + + let cfg = AgentConfig::from_env("/tmp"); + assert_eq!(cfg.openai_oauth_token.as_deref(), Some("oauth-token")); + assert_eq!(cfg.openai_api_key.as_deref(), Some("sk-test456")); + assert_eq!(cfg.api_key.as_deref(), Some("sk-test456")); + for (k, v) in saved { unsafe { match v { @@ -614,16 +662,34 @@ mod tests { assert!(is_foundry_openai_base_url(FOUNDRY_OPENAI_BASE_URL)); assert!(is_foundry_anthropic_base_url(FOUNDRY_ANTHROPIC_BASE_URL)); assert_eq!( - resolve_openai_api_key(None, FOUNDRY_OPENAI_BASE_URL).as_deref(), + resolve_openai_api_key(None, FOUNDRY_OPENAI_BASE_URL, None).as_deref(), Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER) ); + assert_eq!( + resolve_openai_api_key( + Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.to_string()), + FOUNDRY_OPENAI_BASE_URL, + Some("oauth-token".to_string()), + ) + .as_deref(), + Some("oauth-token") + ); + assert_eq!( + resolve_openai_api_key( + Some("sk-openai".to_string()), + FOUNDRY_OPENAI_BASE_URL, + Some("oauth-token".to_string()), + ) + .as_deref(), + Some("sk-openai") + ); assert_eq!( resolve_anthropic_api_key(None, FOUNDRY_ANTHROPIC_BASE_URL).as_deref(), Some(FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER) ); assert_eq!( - strip_foundry_model_prefix("azure-foundry/gpt-5.3-codex"), - "gpt-5.3-codex" + strip_foundry_model_prefix("azure-foundry/gpt-5.4"), + "gpt-5.4" ); assert_eq!( strip_foundry_model_prefix("anthropic-foundry/claude-opus-4-6"), diff --git a/openplanter-desktop/crates/op-core/src/credentials.rs b/openplanter-desktop/crates/op-core/src/credentials.rs index 32ef2d53..44174c5b 100644 --- a/openplanter-desktop/crates/op-core/src/credentials.rs +++ b/openplanter-desktop/crates/op-core/src/credentials.rs @@ -153,7 +153,11 @@ pub fn parse_env_file(path: &Path) -> CredentialBundle { CredentialBundle { openai_api_key: get_key(&env_map, "OPENAI_API_KEY", "OPENPLANTER_OPENAI_API_KEY"), - openai_oauth_token: get_key(&env_map, "OPENAI_OAUTH_TOKEN", "OPENPLANTER_OPENAI_OAUTH_TOKEN"), + openai_oauth_token: get_key( + &env_map, + "OPENAI_OAUTH_TOKEN", + "OPENPLANTER_OPENAI_OAUTH_TOKEN", + ), anthropic_api_key: get_key( &env_map, "ANTHROPIC_API_KEY", diff --git a/openplanter-desktop/crates/op-core/src/model/openai.rs b/openplanter-desktop/crates/op-core/src/model/openai.rs index b3fb5ad4..b8b5c20e 100644 --- a/openplanter-desktop/crates/op-core/src/model/openai.rs +++ b/openplanter-desktop/crates/op-core/src/model/openai.rs @@ -762,7 +762,7 @@ mod tests { fn test_reasoning_model_gpt5() { assert!(make_model("gpt-5.2", None).is_reasoning_model()); assert!(make_model("gpt-5", None).is_reasoning_model()); - assert!(make_model("azure-foundry/gpt-5.3-codex", None).is_reasoning_model()); + assert!(make_model("azure-foundry/gpt-5.4", None).is_reasoning_model()); } #[test] @@ -848,12 +848,12 @@ mod tests { #[test] fn test_payload_strips_foundry_prefix() { - let model = make_model("azure-foundry/gpt-5.3-codex", Some("high")); + let model = make_model("azure-foundry/gpt-5.4", Some("high")); let msgs = vec![Message::User { content: "Hi".to_string(), }]; let payload = model.build_payload(&msgs, &[], true); - assert_eq!(payload["model"], "gpt-5.3-codex"); + assert_eq!(payload["model"], "gpt-5.4"); } #[test] diff --git a/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs b/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs index c2ce34c6..ae880264 100644 --- a/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs +++ b/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs @@ -1007,7 +1007,7 @@ async fn test_solve_missing_key_emits_error() { api_key: None, openai_api_key: None, demo: false, - // No API key set + // No OpenAI auth set ..Default::default() }; @@ -1016,8 +1016,8 @@ async fn test_solve_missing_key_emits_error() { let recorded = errors.lock().unwrap().clone(); assert!( - recorded.iter().any(|e| e.contains("API key")), - "should emit error about missing API key, got: {:?}", + recorded.iter().any(|e| e.contains("OpenAI auth")), + "should emit error about missing OpenAI auth, got: {:?}", recorded ); } diff --git a/openplanter-desktop/crates/op-tauri/src/commands/config.rs b/openplanter-desktop/crates/op-tauri/src/commands/config.rs index 93d0a009..31980a2c 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/config.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/config.rs @@ -1,5 +1,7 @@ use crate::state::AppState; -use op_core::config::{normalize_web_search_provider, normalize_zai_plan, resolve_zai_base_url}; +use op_core::config::{ + has_openai_auth, normalize_web_search_provider, normalize_zai_plan, resolve_zai_base_url, +}; use op_core::credentials::credentials_from_env; use op_core::events::{ConfigView, ModelInfo, PartialConfig}; use op_core::settings::{PersistentSettings, SettingsStore}; @@ -101,8 +103,8 @@ pub async fn update_config( fn known_models_for_provider(provider: &str) -> Vec { let models: Vec<(&str, &str)> = match provider { "openai" => vec![ - ("azure-foundry/gpt-5.3-codex", "GPT-5.3 Codex (Foundry)"), ("azure-foundry/gpt-5.4", "GPT-5.4 (Foundry)"), + ("azure-foundry/gpt-5.3-codex", "GPT-5.3 Codex (Foundry)"), ("azure-foundry/Kimi-K2.5", "Kimi K2.5 (Foundry)"), ], "anthropic" => vec![ @@ -193,7 +195,13 @@ pub async fn save_settings( /// Build credential status from config: which providers/services have API keys configured. pub fn build_credential_status(cfg: &op_core::config::AgentConfig) -> HashMap { let mut status = HashMap::new(); - status.insert("openai".to_string(), cfg.openai_api_key.is_some()); + status.insert( + "openai".to_string(), + has_openai_auth( + cfg.openai_api_key.as_deref(), + cfg.openai_oauth_token.as_deref(), + ), + ); status.insert("anthropic".to_string(), cfg.anthropic_api_key.is_some()); status.insert("openrouter".to_string(), cfg.openrouter_api_key.is_some()); status.insert("cerebras".to_string(), cfg.cerebras_api_key.is_some()); @@ -216,9 +224,13 @@ pub async fn get_credentials_status( let mut status = HashMap::new(); status.insert( "openai".to_string(), - cfg.openai_api_key.is_some() - || env_creds.openai_api_key.is_some() - || env_creds.openai_oauth_token.is_some(), + has_openai_auth( + cfg.openai_api_key.as_deref(), + cfg.openai_oauth_token.as_deref(), + ) || has_openai_auth( + env_creds.openai_api_key.as_deref(), + env_creds.openai_oauth_token.as_deref(), + ), ); status.insert( "anthropic".to_string(), @@ -347,6 +359,7 @@ mod tests { // Force all keys to None let mut cfg = cfg; cfg.openai_api_key = None; + cfg.openai_oauth_token = None; cfg.anthropic_api_key = None; cfg.openrouter_api_key = None; cfg.cerebras_api_key = None; @@ -368,6 +381,7 @@ mod tests { fn test_cred_status_openai_set() { let mut cfg = op_core::config::AgentConfig::from_env("/nonexistent"); cfg.openai_api_key = Some("sk-test".to_string()); + cfg.openai_oauth_token = None; cfg.anthropic_api_key = None; cfg.openrouter_api_key = None; cfg.cerebras_api_key = None; @@ -381,6 +395,7 @@ mod tests { fn test_cred_status_anthropic_set() { let mut cfg = op_core::config::AgentConfig::from_env("/nonexistent"); cfg.openai_api_key = None; + cfg.openai_oauth_token = None; cfg.anthropic_api_key = Some("sk-ant-test".to_string()); cfg.openrouter_api_key = None; cfg.cerebras_api_key = None; @@ -393,6 +408,7 @@ mod tests { fn test_cred_status_ollama_always_true() { let mut cfg = op_core::config::AgentConfig::from_env("/nonexistent"); cfg.openai_api_key = None; + cfg.openai_oauth_token = None; cfg.anthropic_api_key = None; cfg.openrouter_api_key = None; cfg.cerebras_api_key = None; @@ -405,6 +421,7 @@ mod tests { fn test_cred_status_all_set() { let mut cfg = op_core::config::AgentConfig::from_env("/nonexistent"); cfg.openai_api_key = Some("k1".to_string()); + cfg.openai_oauth_token = Some("oauth-token".to_string()); cfg.anthropic_api_key = Some("k2".to_string()); cfg.openrouter_api_key = Some("k3".to_string()); cfg.cerebras_api_key = Some("k4".to_string()); @@ -428,4 +445,22 @@ mod tests { "should have 9 entries (6 providers + 3 web services)" ); } + + #[test] + fn test_cred_status_openai_oauth_counts_as_configured() { + let mut cfg = op_core::config::AgentConfig::from_env("/nonexistent"); + cfg.openai_api_key = None; + cfg.openai_oauth_token = Some("oauth-token".to_string()); + let status = build_credential_status(&cfg); + assert_eq!(status["openai"], true); + } + + #[test] + fn test_cred_status_openai_placeholder_does_not_count() { + let mut cfg = op_core::config::AgentConfig::from_env("/nonexistent"); + cfg.openai_api_key = Some(op_core::config::FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.to_string()); + cfg.openai_oauth_token = None; + let status = build_credential_status(&cfg); + assert_eq!(status["openai"], false); + } } diff --git a/openplanter-desktop/crates/op-tauri/src/state.rs b/openplanter-desktop/crates/op-tauri/src/state.rs index f8869196..7038e7b3 100644 --- a/openplanter-desktop/crates/op-tauri/src/state.rs +++ b/openplanter-desktop/crates/op-tauri/src/state.rs @@ -1,5 +1,6 @@ use op_core::config::{ - AgentConfig, normalize_web_search_provider, normalize_zai_plan, resolve_zai_base_url, + AgentConfig, FOUNDRY_OPENAI_API_KEY_PLACEHOLDER, normalize_web_search_provider, + normalize_zai_plan, resolve_openai_api_key, resolve_zai_base_url, }; use op_core::credentials::{ CredentialBundle, credentials_from_env, discover_env_candidates, parse_env_file, @@ -43,14 +44,45 @@ pub fn merge_credentials_into_config( env_creds: &CredentialBundle, file_creds: &CredentialBundle, ) { - if cfg.openai_api_key.is_none() { - cfg.openai_api_key = env_creds - .openai_api_key + if cfg.openai_oauth_token.is_none() { + cfg.openai_oauth_token = env_creds + .openai_oauth_token .clone() - .or_else(|| env_creds.openai_oauth_token.clone()) - .or_else(|| file_creds.openai_api_key.clone()) .or_else(|| file_creds.openai_oauth_token.clone()); } + cfg.openai_api_key = cfg + .openai_api_key + .clone() + .filter(|value| { + let trimmed = value.trim(); + !trimmed.is_empty() && trimmed != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER + }) + .or_else(|| env_creds.openai_api_key.clone()) + .or_else(|| file_creds.openai_api_key.clone()) + .or_else(|| cfg.openai_api_key.clone()); + cfg.openai_api_key = resolve_openai_api_key( + cfg.openai_api_key.clone(), + &cfg.openai_base_url, + cfg.openai_oauth_token.clone(), + ); + cfg.api_key = resolve_openai_api_key( + cfg.openai_api_key + .clone() + .filter(|value| { + let trimmed = value.trim(); + !trimmed.is_empty() && trimmed != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER + }) + .or_else(|| { + cfg.api_key.clone().filter(|value| { + let trimmed = value.trim(); + !trimmed.is_empty() && trimmed != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER + }) + }) + .or_else(|| cfg.openai_api_key.clone()) + .or_else(|| cfg.api_key.clone()), + &cfg.base_url, + cfg.openai_oauth_token.clone(), + ); macro_rules! merge { ($field:ident) => { @@ -388,6 +420,7 @@ mod tests { fn empty_cfg() -> AgentConfig { let mut cfg = AgentConfig::from_env("/nonexistent"); cfg.openai_api_key = None; + cfg.openai_oauth_token = None; cfg.anthropic_api_key = None; cfg.openrouter_api_key = None; cfg.cerebras_api_key = None; @@ -424,6 +457,33 @@ mod tests { assert_eq!(cfg.openai_api_key, Some("existing".to_string())); } + #[test] + fn test_merge_prefers_real_openai_key_over_oauth() { + let mut cfg = empty_cfg(); + let env_creds = CredentialBundle { + openai_api_key: Some("env-key".to_string()), + openai_oauth_token: Some("oauth-token".to_string()), + ..Default::default() + }; + merge_credentials_into_config(&mut cfg, &env_creds, &CredentialBundle::default()); + assert_eq!(cfg.openai_oauth_token, Some("oauth-token".to_string())); + assert_eq!(cfg.openai_api_key, Some("env-key".to_string())); + assert_eq!(cfg.api_key, Some("env-key".to_string())); + } + + #[test] + fn test_merge_uses_oauth_when_only_placeholder_exists() { + let mut cfg = AgentConfig::default(); + let env_creds = CredentialBundle { + openai_oauth_token: Some("oauth-token".to_string()), + ..Default::default() + }; + merge_credentials_into_config(&mut cfg, &env_creds, &CredentialBundle::default()); + assert_eq!(cfg.openai_oauth_token, Some("oauth-token".to_string())); + assert_eq!(cfg.openai_api_key, Some("oauth-token".to_string())); + assert_eq!(cfg.api_key, Some("oauth-token".to_string())); + } + #[test] fn test_merge_env_over_file() { let mut cfg = empty_cfg(); diff --git a/openplanter-desktop/frontend/src/api/invoke.test.ts b/openplanter-desktop/frontend/src/api/invoke.test.ts index 35fecc66..02da105a 100644 --- a/openplanter-desktop/frontend/src/api/invoke.test.ts +++ b/openplanter-desktop/frontend/src/api/invoke.test.ts @@ -66,10 +66,10 @@ describe("invoke wrappers", () => { it("updateConfig sends partial and returns config", async () => { __setHandler("update_config", ({ partial }: any) => { - expect(partial.model).toBe("azure-foundry/gpt-5.3-codex"); + expect(partial.model).toBe("azure-foundry/gpt-5.4"); return { provider: "openai", - model: "azure-foundry/gpt-5.3-codex", + model: "azure-foundry/gpt-5.4", zai_plan: "coding", workspace: ".", session_id: null, @@ -81,8 +81,8 @@ describe("invoke wrappers", () => { demo: false, }; }); - const config = await updateConfig({ model: "azure-foundry/gpt-5.3-codex" }); - expect(config.model).toBe("azure-foundry/gpt-5.3-codex"); + const config = await updateConfig({ model: "azure-foundry/gpt-5.4" }); + expect(config.model).toBe("azure-foundry/gpt-5.4"); expect(config.zai_plan).toBe("coding"); expect(config.web_search_provider).toBe("firecrawl"); }); @@ -92,15 +92,15 @@ describe("invoke wrappers", () => { expect(provider).toBe("openai"); return [ { - id: "azure-foundry/gpt-5.3-codex", - name: "GPT-5.3 Codex (Foundry)", + id: "azure-foundry/gpt-5.4", + name: "GPT-5.4 (Foundry)", provider: "openai", }, ]; }); const models = await listModels("openai"); expect(models).toHaveLength(1); - expect(models[0].id).toBe("azure-foundry/gpt-5.3-codex"); + expect(models[0].id).toBe("azure-foundry/gpt-5.4"); }); it("saveSettings sends settings object", async () => { diff --git a/openplanter-desktop/frontend/src/commands/model.test.ts b/openplanter-desktop/frontend/src/commands/model.test.ts index 31eacdd6..c8aa27c9 100644 --- a/openplanter-desktop/frontend/src/commands/model.test.ts +++ b/openplanter-desktop/frontend/src/commands/model.test.ts @@ -17,7 +17,7 @@ describe("inferProvider", () => { it("gpt returns openai", () => { expect(inferProvider("gpt-5.2")).toBe("openai"); - expect(inferProvider("azure-foundry/gpt-5.3-codex")).toBe("openai"); + expect(inferProvider("azure-foundry/gpt-5.4")).toBe("openai"); }); it("o1 returns openai", () => { @@ -63,13 +63,25 @@ describe("MODEL_ALIASES", () => { }); it("gpt5 alias", () => { - expect(MODEL_ALIASES["gpt5"]).toBe("azure-foundry/gpt-5.3-codex"); + expect(MODEL_ALIASES["gpt5"]).toBe("azure-foundry/gpt-5.4"); + }); + + it("gpt-5 alias", () => { + expect(MODEL_ALIASES["gpt-5"]).toBe("azure-foundry/gpt-5.4"); + }); + + it("gpt-5.3 alias", () => { + expect(MODEL_ALIASES["gpt-5.3"]).toBe("azure-foundry/gpt-5.3-codex"); }); it("gpt-5.4 alias", () => { expect(MODEL_ALIASES["gpt-5.4"]).toBe("azure-foundry/gpt-5.4"); }); + it("gpt5.4 alias", () => { + expect(MODEL_ALIASES["gpt5.4"]).toBe("azure-foundry/gpt-5.4"); + }); + it("zai alias", () => { expect(MODEL_ALIASES["zai"]).toBe("glm-5"); }); @@ -141,4 +153,29 @@ describe("handleModelCommand", () => { expect(appState.get().model).toBe("glm-5"); expect(appState.get().zaiPlan).toBe("coding"); }); + + it("gpt5 alias switches to gpt-5.4", async () => { + __setHandler("update_config", ({ partial }: { partial: Record }) => { + expect(partial.model).toBe("azure-foundry/gpt-5.4"); + expect(partial.provider).toBe("openai"); + return { + provider: "openai", + model: "azure-foundry/gpt-5.4", + zai_plan: "paygo", + workspace: ".", + session_id: null, + recursive: true, + max_depth: 4, + max_steps_per_call: 100, + reasoning_effort: "high", + web_search_provider: "exa", + demo: false, + }; + }); + + const result = await handleModelCommand("gpt5"); + expect(result.lines).toContain("Switched to openai/azure-foundry/gpt-5.4"); + expect(appState.get().provider).toBe("openai"); + expect(appState.get().model).toBe("azure-foundry/gpt-5.4"); + }); }); diff --git a/openplanter-desktop/frontend/src/commands/model.ts b/openplanter-desktop/frontend/src/commands/model.ts index 68b45bab..91900662 100644 --- a/openplanter-desktop/frontend/src/commands/model.ts +++ b/openplanter-desktop/frontend/src/commands/model.ts @@ -11,10 +11,11 @@ export const MODEL_ALIASES: Record = { "sonnet-4": "anthropic-foundry/claude-sonnet-4-6", "haiku-4": "anthropic-foundry/claude-haiku-4-5", "opus-4": "anthropic-foundry/claude-opus-4-6", - gpt5: "azure-foundry/gpt-5.3-codex", - "gpt-5": "azure-foundry/gpt-5.3-codex", + gpt5: "azure-foundry/gpt-5.4", + "gpt-5": "azure-foundry/gpt-5.4", + "gpt5.3": "azure-foundry/gpt-5.3-codex", "gpt-5.3": "azure-foundry/gpt-5.3-codex", - gpt54: "azure-foundry/gpt-5.4", + "gpt5.4": "azure-foundry/gpt-5.4", "gpt-5.4": "azure-foundry/gpt-5.4", kimi: "azure-foundry/Kimi-K2.5", gpt4o: "gpt-4o", diff --git a/tests/test_coverage_gaps.py b/tests/test_coverage_gaps.py index 15d36b62..067e255e 100644 --- a/tests/test_coverage_gaps.py +++ b/tests/test_coverage_gaps.py @@ -256,6 +256,23 @@ def test_api_keys_from_env(self) -> None: self.assertEqual(cfg.exa_api_key, "exa") self.assertEqual(cfg.brave_api_key, "brave") + def test_openai_oauth_token_from_env_without_api_key(self) -> None: + env = {"OPENAI_OAUTH_TOKEN": "oauth-token"} + with patch.dict(os.environ, env, clear=True): + cfg = AgentConfig.from_env("/tmp/test-ws") + self.assertEqual(cfg.openai_oauth_token, "oauth-token") + self.assertEqual(cfg.openai_api_key, "oauth-token") + + def test_openai_api_key_beats_oauth_token(self) -> None: + env = { + "OPENAI_API_KEY": "oa", + "OPENAI_OAUTH_TOKEN": "oauth-token", + } + with patch.dict(os.environ, env, clear=True): + cfg = AgentConfig.from_env("/tmp/test-ws") + self.assertEqual(cfg.openai_oauth_token, "oauth-token") + self.assertEqual(cfg.openai_api_key, "oa") + def test_foundry_placeholder_keys_disabled_for_public_endpoints(self) -> None: env = { "OPENPLANTER_OPENAI_BASE_URL": "https://api.openai.com/v1", @@ -340,7 +357,7 @@ def test_explicit_model_returned(self) -> None: def test_empty_model_uses_provider_default(self) -> None: cfg = AgentConfig(workspace=Path("/tmp"), provider="openai", model="") - self.assertEqual(_resolve_model_name(cfg), "azure-foundry/gpt-5.3-codex") + self.assertEqual(_resolve_model_name(cfg), "azure-foundry/gpt-5.4") def test_empty_model_anthropic_default(self) -> None: cfg = AgentConfig(workspace=Path("/tmp"), provider="anthropic", model="") @@ -374,7 +391,7 @@ def test_openai_provider_with_key(self) -> None: cfg = AgentConfig( workspace=Path(tmpdir), provider="openai", - model="azure-foundry/gpt-5.3-codex", + model="azure-foundry/gpt-5.4", openai_api_key="test-key", ) engine = build_engine(cfg) @@ -396,7 +413,7 @@ def test_no_key_fallback_to_echo(self) -> None: cfg = AgentConfig( workspace=Path(tmpdir), provider="openai", - model="azure-foundry/gpt-5.3-codex", + model="azure-foundry/gpt-5.4", openai_base_url="https://api.openai.com/v1", openai_api_key=None, ) diff --git a/tests/test_model.py b/tests/test_model.py index 0631eb19..db5c97cb 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -55,14 +55,14 @@ def fake_http_json(url, method, headers, payload=None, timeout_sec=90): # type: with patch("agent.model._http_stream_sse", mock_openai_stream(fake_http_json)): model = OpenAICompatibleModel( - model="azure-foundry/gpt-5.3-codex", + model="azure-foundry/gpt-5.4", api_key="k", reasoning_effort="high", ) conv = model.create_conversation("system", "user msg") turn = model.complete(conv) self.assertEqual(turn.text, "ok") - self.assertEqual(captured["payload"]["model"], "gpt-5.3-codex") + self.assertEqual(captured["payload"]["model"], "gpt-5.4") def test_openai_payload_includes_thinking_type(self) -> None: captured: dict = {} diff --git a/tests/test_settings.py b/tests/test_settings.py index d39c08ea..c01e1a41 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -176,7 +176,7 @@ def test_gpt_is_openai(self) -> None: self.assertEqual(infer_provider_for_model("gpt-4.1-mini"), "openai") self.assertEqual(infer_provider_for_model("GPT-4o"), "openai") self.assertEqual( - infer_provider_for_model("azure-foundry/gpt-5.3-codex"), + infer_provider_for_model("azure-foundry/gpt-5.4"), "openai", ) From ba663dfdb6b003161a3b56ee29133739873aeb91 Mon Sep 17 00:00:00 2001 From: Drake Date: Thu, 12 Mar 2026 13:02:48 -0400 Subject: [PATCH 07/58] Add voyage credential status to desktop --- .../crates/op-core/src/credentials.rs | 20 ++++++++++++++++++- .../crates/op-tauri/src/commands/config.rs | 14 ++++++++++--- .../crates/op-tauri/src/state.rs | 4 +++- .../frontend/src/api/invoke.test.ts | 2 ++ .../frontend/src/components/App.test.ts | 4 ++-- .../frontend/src/components/App.ts | 2 +- 6 files changed, 38 insertions(+), 8 deletions(-) diff --git a/openplanter-desktop/crates/op-core/src/credentials.rs b/openplanter-desktop/crates/op-core/src/credentials.rs index 44174c5b..44817768 100644 --- a/openplanter-desktop/crates/op-core/src/credentials.rs +++ b/openplanter-desktop/crates/op-core/src/credentials.rs @@ -27,7 +27,7 @@ pub struct CredentialBundle { impl CredentialBundle { /// Returns `true` if any key has a non-empty value. pub fn has_any(&self) -> bool { - let keys: [&Option; 9] = [ + let keys = [ &self.openai_api_key, &self.openai_oauth_token, &self.anthropic_api_key, @@ -339,6 +339,24 @@ mod tests { assert!(bundle.has_any()); } + #[test] + fn test_credential_bundle_has_any_with_voyage_key() { + let bundle = CredentialBundle { + voyage_api_key: Some("voyage-test".into()), + ..Default::default() + }; + assert!(bundle.has_any()); + } + + #[test] + fn test_credential_bundle_whitespace_only_values_do_not_count() { + let bundle = CredentialBundle { + voyage_api_key: Some(" ".into()), + ..Default::default() + }; + assert!(!bundle.has_any()); + } + #[test] fn test_credential_bundle_merge_missing() { let mut a = CredentialBundle { diff --git a/openplanter-desktop/crates/op-tauri/src/commands/config.rs b/openplanter-desktop/crates/op-tauri/src/commands/config.rs index 31980a2c..9e624b3e 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/config.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/config.rs @@ -210,6 +210,7 @@ pub fn build_credential_status(cfg: &op_core::config::AgentConfig) -> HashMap { exa: false, firecrawl: true, brave: false, + voyage: true, })); const status = await getCredentialsStatus(); expect(status.openai).toBe(true); @@ -134,6 +135,7 @@ describe("invoke wrappers", () => { expect(status.zai).toBe(true); expect(status.firecrawl).toBe(true); expect(status.brave).toBe(false); + expect(status.voyage).toBe(true); }); it("listSessions sends limit", async () => { diff --git a/openplanter-desktop/frontend/src/components/App.test.ts b/openplanter-desktop/frontend/src/components/App.test.ts index f0323542..30037232 100644 --- a/openplanter-desktop/frontend/src/components/App.test.ts +++ b/openplanter-desktop/frontend/src/components/App.test.ts @@ -48,7 +48,7 @@ describe("createApp", () => { __setHandler("list_sessions", () => [SESSION_B, SESSION_A]); __setHandler("get_credentials_status", () => ({ openai: true, anthropic: true, openrouter: false, - cerebras: false, zai: true, ollama: true, exa: false, firecrawl: true, brave: false, + cerebras: false, zai: true, ollama: true, exa: false, firecrawl: true, brave: false, voyage: true, })); __setHandler("open_session", () => ({ id: "20260227-120000-cccc3333", @@ -103,7 +103,7 @@ describe("createApp", () => { await vi.waitFor(() => { const creds = root.querySelector(".cred-status"); - expect(creds!.children.length).toBe(9); + expect(creds!.children.length).toBe(10); expect(creds!.querySelector(".cred-ok")!.textContent).toContain("openai"); expect(creds!.querySelector(".cred-missing")!.textContent).toContain("openrouter"); }); diff --git a/openplanter-desktop/frontend/src/components/App.ts b/openplanter-desktop/frontend/src/components/App.ts index 9e08f564..9b162fdf 100644 --- a/openplanter-desktop/frontend/src/components/App.ts +++ b/openplanter-desktop/frontend/src/components/App.ts @@ -302,7 +302,7 @@ async function loadCredentials(container: HTMLElement): Promise { try { const status = await getCredentialsStatus(); container.innerHTML = ""; - const providers = ["openai", "anthropic", "openrouter", "cerebras", "zai", "ollama", "exa", "firecrawl", "brave"]; + const providers = ["openai", "anthropic", "openrouter", "cerebras", "zai", "ollama", "exa", "firecrawl", "brave", "voyage"]; for (const p of providers) { const row = document.createElement("div"); const hasKey = status[p] ?? false; From 05f5c56f72d1e6460628e33cfb036075879fdcb7 Mon Sep 17 00:00:00 2001 From: Drake Date: Thu, 12 Mar 2026 13:47:29 -0400 Subject: [PATCH 08/58] Add Tavily web search provider parity --- README.md | 5 +- agent/__main__.py | 13 +- agent/builder.py | 2 + agent/config.py | 7 +- agent/credentials.py | 11 + agent/tool_defs.py | 4 +- agent/tools.py | 106 ++++++- .../crates/op-core/src/config.rs | 25 +- .../crates/op-core/src/credentials.rs | 13 + .../crates/op-core/src/tools/defs.rs | 4 +- .../crates/op-core/src/tools/mod.rs | 12 + .../crates/op-core/src/tools/web.rs | 289 +++++++++++++++++- .../crates/op-tauri/src/commands/config.rs | 16 +- .../crates/op-tauri/src/state.rs | 6 +- .../frontend/src/api/invoke.test.ts | 2 + .../src/commands/completionRegistry.test.ts | 4 +- .../src/commands/completionRegistry.ts | 1 + .../frontend/src/commands/slash.ts | 2 +- .../frontend/src/commands/webSearch.test.ts | 16 +- .../frontend/src/commands/webSearch.ts | 2 +- .../frontend/src/components/App.test.ts | 4 +- .../frontend/src/components/App.ts | 2 +- tests/test_coverage_gaps.py | 18 +- tests/test_credentials.py | 3 + tests/test_tools.py | 60 ++++ 25 files changed, 585 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index e2fef280..618f6b43 100644 --- a/README.md +++ b/README.md @@ -150,9 +150,10 @@ export OPENPLANTER_RATE_LIMIT_RETRY_AFTER_CAP_SEC=120.0 export OPENPLANTER_ZAI_STREAM_MAX_RETRIES=10 ``` -Additional service keys: `EXA_API_KEY`, `FIRECRAWL_API_KEY`, `BRAVE_API_KEY` (web search), `VOYAGE_API_KEY` (embeddings). +Additional service keys: `EXA_API_KEY`, `FIRECRAWL_API_KEY`, `BRAVE_API_KEY`, `TAVILY_API_KEY` (web search), `VOYAGE_API_KEY` (embeddings). All keys can also be set with an `OPENPLANTER_` prefix (e.g. `OPENPLANTER_OPENAI_API_KEY`), via `.env` files in the workspace, or via CLI flags. +Provider base URLs can also be overridden with `OPENPLANTER_*_BASE_URL`, including `OPENPLANTER_TAVILY_BASE_URL`. ## Agent Tools @@ -162,7 +163,7 @@ The agent has access to 19 tools, organized around its investigation workflow: **Shell execution** — `run_shell`, `run_shell_bg`, `check_shell_bg`, `kill_shell_bg` — run analysis scripts, data pipelines, and validation checks. -**Web** — `web_search` (Exa, Firecrawl, or Brave), `fetch_url` — pull public records, verify entities, and retrieve supplementary data. +**Web** — `web_search` (Exa, Firecrawl, Brave, or Tavily), `fetch_url` — pull public records, verify entities, and retrieve supplementary data. **Planning & delegation** — `think`, `subtask`, `execute`, `list_artifacts`, `read_artifact` — decompose investigations into focused sub-tasks, each with acceptance criteria and independent verification. diff --git a/agent/__main__.py b/agent/__main__.py index 07d5b3d2..41678c0b 100644 --- a/agent/__main__.py +++ b/agent/__main__.py @@ -105,9 +105,10 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--exa-api-key", help="Exa API key override.") parser.add_argument("--firecrawl-api-key", help="Firecrawl API key override.") parser.add_argument("--brave-api-key", help="Brave Search API key override.") + parser.add_argument("--tavily-api-key", help="Tavily API key override.") parser.add_argument( "--web-search-provider", - choices=["exa", "firecrawl", "brave"], + choices=["exa", "firecrawl", "brave", "tavily"], help="Web search backend provider.", ) parser.add_argument("--voyage-api-key", help="Voyage API key override.") @@ -248,6 +249,7 @@ def _load_credentials( exa_api_key=user_creds.exa_api_key, firecrawl_api_key=user_creds.firecrawl_api_key, brave_api_key=user_creds.brave_api_key, + tavily_api_key=user_creds.tavily_api_key, voyage_api_key=user_creds.voyage_api_key, ) @@ -271,6 +273,8 @@ def _load_credentials( creds.firecrawl_api_key = stored.firecrawl_api_key if stored.brave_api_key: creds.brave_api_key = stored.brave_api_key + if stored.tavily_api_key: + creds.tavily_api_key = stored.tavily_api_key if stored.voyage_api_key: creds.voyage_api_key = stored.voyage_api_key @@ -293,6 +297,8 @@ def _load_credentials( creds.firecrawl_api_key = env_creds.firecrawl_api_key if env_creds.brave_api_key: creds.brave_api_key = env_creds.brave_api_key + if env_creds.tavily_api_key: + creds.tavily_api_key = env_creds.tavily_api_key if env_creds.voyage_api_key: creds.voyage_api_key = env_creds.voyage_api_key @@ -320,6 +326,8 @@ def _load_credentials( creds.firecrawl_api_key = args.firecrawl_api_key.strip() or creds.firecrawl_api_key if args.brave_api_key: creds.brave_api_key = args.brave_api_key.strip() or creds.brave_api_key + if args.tavily_api_key: + creds.tavily_api_key = args.tavily_api_key.strip() or creds.tavily_api_key if args.voyage_api_key: creds.voyage_api_key = args.voyage_api_key.strip() or creds.voyage_api_key @@ -374,6 +382,7 @@ def _apply_runtime_overrides(cfg: AgentConfig, args: argparse.Namespace, creds: cfg.exa_api_key = creds.exa_api_key cfg.firecrawl_api_key = creds.firecrawl_api_key cfg.brave_api_key = creds.brave_api_key + cfg.tavily_api_key = creds.tavily_api_key cfg.voyage_api_key = creds.voyage_api_key cfg.api_key = cfg.openai_api_key @@ -419,7 +428,7 @@ def _apply_runtime_overrides(cfg: AgentConfig, args: argparse.Namespace, creds: cfg.model = args.model if args.web_search_provider: cfg.web_search_provider = args.web_search_provider - if cfg.web_search_provider not in {"exa", "firecrawl", "brave"}: + if cfg.web_search_provider not in {"exa", "firecrawl", "brave", "tavily"}: cfg.web_search_provider = "exa" if args.reasoning_effort: cfg.reasoning_effort = None if args.reasoning_effort == "none" else args.reasoning_effort diff --git a/agent/builder.py b/agent/builder.py index 7d7044ac..146d53f9 100644 --- a/agent/builder.py +++ b/agent/builder.py @@ -249,6 +249,8 @@ def build_engine(cfg: AgentConfig) -> RLMEngine: firecrawl_base_url=cfg.firecrawl_base_url, brave_api_key=cfg.brave_api_key, brave_base_url=cfg.brave_base_url, + tavily_api_key=cfg.tavily_api_key, + tavily_base_url=cfg.tavily_base_url, ) try: diff --git a/agent/config.py b/agent/config.py index 6a0e0f9c..fc00d408 100644 --- a/agent/config.py +++ b/agent/config.py @@ -112,6 +112,7 @@ class AgentConfig: exa_base_url: str = "https://api.exa.ai" firecrawl_base_url: str = "https://api.firecrawl.dev/v1" brave_base_url: str = "https://api.search.brave.com/res/v1" + tavily_base_url: str = "https://api.tavily.com" openai_api_key: str | None = None openai_oauth_token: str | None = None anthropic_api_key: str | None = None @@ -121,6 +122,7 @@ class AgentConfig: exa_api_key: str | None = None firecrawl_api_key: str | None = None brave_api_key: str | None = None + tavily_api_key: str | None = None web_search_provider: str = "exa" voyage_api_key: str | None = None max_depth: int = 4 @@ -180,6 +182,7 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": exa_api_key = os.getenv("OPENPLANTER_EXA_API_KEY") or os.getenv("EXA_API_KEY") firecrawl_api_key = os.getenv("OPENPLANTER_FIRECRAWL_API_KEY") or os.getenv("FIRECRAWL_API_KEY") brave_api_key = os.getenv("OPENPLANTER_BRAVE_API_KEY") or os.getenv("BRAVE_API_KEY") + tavily_api_key = os.getenv("OPENPLANTER_TAVILY_API_KEY") or os.getenv("TAVILY_API_KEY") voyage_api_key = os.getenv("OPENPLANTER_VOYAGE_API_KEY") or os.getenv("VOYAGE_API_KEY") openai_base_url = os.getenv("OPENPLANTER_OPENAI_BASE_URL") or os.getenv( "OPENPLANTER_BASE_URL", @@ -208,7 +211,7 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": ) ) web_search_provider = (os.getenv("OPENPLANTER_WEB_SEARCH_PROVIDER", "exa").strip().lower() or "exa") - if web_search_provider not in {"exa", "firecrawl", "brave"}: + if web_search_provider not in {"exa", "firecrawl", "brave", "tavily"}: web_search_provider = "exa" return cls( workspace=ws, @@ -229,6 +232,7 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": exa_base_url=os.getenv("OPENPLANTER_EXA_BASE_URL", "https://api.exa.ai"), firecrawl_base_url=os.getenv("OPENPLANTER_FIRECRAWL_BASE_URL", "https://api.firecrawl.dev/v1"), brave_base_url=os.getenv("OPENPLANTER_BRAVE_BASE_URL", "https://api.search.brave.com/res/v1"), + tavily_base_url=os.getenv("OPENPLANTER_TAVILY_BASE_URL", "https://api.tavily.com"), openai_api_key=openai_api_key, openai_oauth_token=(openai_oauth_token or "").strip() or None, anthropic_api_key=anthropic_api_key, @@ -238,6 +242,7 @@ def from_env(cls, workspace: str | Path) -> "AgentConfig": exa_api_key=exa_api_key, firecrawl_api_key=firecrawl_api_key, brave_api_key=brave_api_key, + tavily_api_key=tavily_api_key, web_search_provider=web_search_provider, voyage_api_key=voyage_api_key, max_depth=int(os.getenv("OPENPLANTER_MAX_DEPTH", "4")), diff --git a/agent/credentials.py b/agent/credentials.py index 2e4d8a40..a79e428e 100644 --- a/agent/credentials.py +++ b/agent/credentials.py @@ -20,6 +20,7 @@ class CredentialBundle: exa_api_key: str | None = None firecrawl_api_key: str | None = None brave_api_key: str | None = None + tavily_api_key: str | None = None voyage_api_key: str | None = None def has_any(self) -> bool: @@ -33,6 +34,7 @@ def has_any(self) -> bool: or (self.exa_api_key and self.exa_api_key.strip()) or (self.firecrawl_api_key and self.firecrawl_api_key.strip()) or (self.brave_api_key and self.brave_api_key.strip()) + or (self.tavily_api_key and self.tavily_api_key.strip()) or (self.voyage_api_key and self.voyage_api_key.strip()) ) @@ -55,6 +57,8 @@ def merge_missing(self, other: "CredentialBundle") -> None: self.firecrawl_api_key = other.firecrawl_api_key if not self.brave_api_key and other.brave_api_key: self.brave_api_key = other.brave_api_key + if not self.tavily_api_key and other.tavily_api_key: + self.tavily_api_key = other.tavily_api_key if not self.voyage_api_key and other.voyage_api_key: self.voyage_api_key = other.voyage_api_key @@ -78,6 +82,8 @@ def to_json(self) -> dict[str, str]: out["firecrawl_api_key"] = self.firecrawl_api_key if self.brave_api_key: out["brave_api_key"] = self.brave_api_key + if self.tavily_api_key: + out["tavily_api_key"] = self.tavily_api_key if self.voyage_api_key: out["voyage_api_key"] = self.voyage_api_key return out @@ -96,6 +102,7 @@ def from_json(cls, payload: dict[str, str] | None) -> "CredentialBundle": exa_api_key=(payload.get("exa_api_key") or "").strip() or None, firecrawl_api_key=(payload.get("firecrawl_api_key") or "").strip() or None, brave_api_key=(payload.get("brave_api_key") or "").strip() or None, + tavily_api_key=(payload.get("tavily_api_key") or "").strip() or None, voyage_api_key=(payload.get("voyage_api_key") or "").strip() or None, ) @@ -146,6 +153,7 @@ def parse_env_file(path: Path) -> CredentialBundle: firecrawl_api_key=(env.get("FIRECRAWL_API_KEY") or env.get("OPENPLANTER_FIRECRAWL_API_KEY") or "").strip() or None, brave_api_key=(env.get("BRAVE_API_KEY") or env.get("OPENPLANTER_BRAVE_API_KEY") or "").strip() or None, + tavily_api_key=(env.get("TAVILY_API_KEY") or env.get("OPENPLANTER_TAVILY_API_KEY") or "").strip() or None, voyage_api_key=(env.get("VOYAGE_API_KEY") or env.get("OPENPLANTER_VOYAGE_API_KEY") or "").strip() or None, ) @@ -184,6 +192,7 @@ def credentials_from_env() -> CredentialBundle: ).strip() or None, brave_api_key=(os.getenv("OPENPLANTER_BRAVE_API_KEY") or os.getenv("BRAVE_API_KEY") or "").strip() or None, + tavily_api_key=(os.getenv("OPENPLANTER_TAVILY_API_KEY") or os.getenv("TAVILY_API_KEY") or "").strip() or None, voyage_api_key=(os.getenv("OPENPLANTER_VOYAGE_API_KEY") or os.getenv("VOYAGE_API_KEY") or "").strip() or None, ) @@ -283,6 +292,7 @@ def prompt_for_credentials( exa_api_key=existing.exa_api_key, firecrawl_api_key=existing.firecrawl_api_key, brave_api_key=existing.brave_api_key, + tavily_api_key=existing.tavily_api_key, voyage_api_key=existing.voyage_api_key, ) @@ -320,6 +330,7 @@ def _ask(label: str, existing_value: str | None) -> str | None: current.exa_api_key = _ask("Exa", current.exa_api_key) current.firecrawl_api_key = _ask("Firecrawl", current.firecrawl_api_key) current.brave_api_key = _ask("Brave", current.brave_api_key) + current.tavily_api_key = _ask("Tavily", current.tavily_api_key) current.voyage_api_key = _ask("Voyage", current.voyage_api_key) if not force and current.has_any() and not existing.has_any(): changed = True diff --git a/agent/tool_defs.py b/agent/tool_defs.py index 63d4765f..73ef01ed 100644 --- a/agent/tool_defs.py +++ b/agent/tool_defs.py @@ -63,7 +63,7 @@ }, { "name": "web_search", - "description": "Search the web using the configured provider (Exa, Firecrawl, or Brave). Returns URLs, titles, and optional page text.", + "description": "Search the web using the configured provider (Exa, Firecrawl, Brave, or Tavily). Returns URLs, titles, and optional page text.", "parameters": { "type": "object", "properties": { @@ -86,7 +86,7 @@ }, { "name": "fetch_url", - "description": "Fetch and return the text content of one or more URLs.", + "description": "Fetch and return the text content of one or more URLs using the configured provider backend (Exa, Firecrawl, Brave, or Tavily).", "parameters": { "type": "object", "properties": { diff --git a/agent/tools.py b/agent/tools.py index 102d4863..e626d140 100644 --- a/agent/tools.py +++ b/agent/tools.py @@ -124,6 +124,8 @@ class WorkspaceTools: firecrawl_base_url: str = "https://api.firecrawl.dev/v1" brave_api_key: str | None = None brave_base_url: str = "https://api.search.brave.com/res/v1" + tavily_api_key: str | None = None + tavily_base_url: str = "https://api.tavily.com" def __post_init__(self) -> None: self.root = self.root.expanduser().resolve() @@ -938,6 +940,38 @@ def _brave_request(self, endpoint: str, params: dict[str, Any]) -> dict[str, Any raise ToolError(f"Brave API returned non-object response: {type(parsed)!r}") return parsed + def _tavily_request(self, endpoint: str, payload: dict[str, Any]) -> dict[str, Any]: + if not (self.tavily_api_key and self.tavily_api_key.strip()): + raise ToolError("TAVILY_API_KEY not configured") + url = self.tavily_base_url.rstrip("/") + endpoint + req = urllib.request.Request( + url=url, + data=json.dumps(payload).encode("utf-8"), + headers={ + "Authorization": f"Bearer {self.tavily_api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=self.command_timeout_sec) as resp: + raw = resp.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise ToolError(f"Tavily API HTTP {exc.code}: {body}") from exc + except urllib.error.URLError as exc: + raise ToolError(f"Tavily API connection error: {exc}") from exc + except OSError as exc: + raise ToolError(f"Tavily API network error: {exc}") from exc + + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise ToolError(f"Tavily API returned non-JSON payload: {raw[:500]}") from exc + if not isinstance(parsed, dict): + raise ToolError(f"Tavily API returned non-object response: {type(parsed)!r}") + return parsed + def _fetch_url_direct(self, url: str) -> dict[str, str]: req = urllib.request.Request( url=url, @@ -993,7 +1027,7 @@ def web_search( return "web_search requires non-empty query" clamped_results = max(1, min(int(num_results), 20)) provider = (self.web_search_provider or "exa").strip().lower() - if provider not in {"exa", "firecrawl", "brave"}: + if provider not in {"exa", "firecrawl", "brave", "tavily"}: provider = "exa" if provider == "firecrawl": @@ -1097,6 +1131,43 @@ def web_search( } return self._clip(json.dumps(output, indent=2, ensure_ascii=True), self.max_file_chars) + if provider == "tavily": + payload = { + "query": query, + "max_results": clamped_results, + } + if include_text: + payload["include_raw_content"] = "markdown" + + try: + parsed = self._tavily_request("/search", payload) + except Exception as exc: + return f"Web search failed: {exc}" + + rows = parsed.get("results") + out_results: list[dict[str, Any]] = [] + for row in rows if isinstance(rows, list) else []: + if not isinstance(row, dict): + continue + snippet = str(row.get("content", "") or row.get("snippet", "")) + text_value = row.get("raw_content") or row.get("content") or "" + item: dict[str, Any] = { + "url": str(row.get("url", "")), + "title": str(row.get("title", "")), + "snippet": snippet, + } + if include_text and isinstance(text_value, str) and text_value: + item["text"] = self._clip(text_value, 4000) + out_results.append(item) + + output = { + "query": query, + "provider": provider, + "results": out_results, + "total": len(out_results), + } + return self._clip(json.dumps(output, indent=2, ensure_ascii=True), self.max_file_chars) + payload: dict[str, Any] = { "query": query, "numResults": clamped_results, @@ -1144,7 +1215,7 @@ def fetch_url(self, urls: list[str]) -> str: return "fetch_url requires at least one valid URL" normalized = normalized[:10] provider = (self.web_search_provider or "exa").strip().lower() - if provider not in {"exa", "firecrawl", "brave"}: + if provider not in {"exa", "firecrawl", "brave", "tavily"}: provider = "exa" if provider == "firecrawl": @@ -1189,6 +1260,37 @@ def fetch_url(self, urls: list[str]) -> str: } return self._clip(json.dumps(output, indent=2, ensure_ascii=True), self.max_file_chars) + if provider == "tavily": + payload = { + "urls": normalized, + "extract_depth": "basic", + "include_images": False, + } + try: + parsed = self._tavily_request("/extract", payload) + except Exception as exc: + return f"Fetch URL failed: {exc}" + + pages: list[dict[str, Any]] = [] + rows = parsed.get("results") + for row in rows if isinstance(rows, list) else []: + if not isinstance(row, dict): + continue + text = row.get("raw_content") or row.get("content") or "" + pages.append( + { + "url": str(row.get("url", "")), + "title": str(row.get("title", "") or ""), + "text": self._clip(str(text), 8000), + } + ) + output = { + "provider": provider, + "pages": pages, + "total": len(pages), + } + return self._clip(json.dumps(output, indent=2, ensure_ascii=True), self.max_file_chars) + payload: dict[str, Any] = { "ids": normalized, "text": {"maxCharacters": 8000}, diff --git a/openplanter-desktop/crates/op-core/src/config.rs b/openplanter-desktop/crates/op-core/src/config.rs index 015acca8..b2a0a847 100644 --- a/openplanter-desktop/crates/op-core/src/config.rs +++ b/openplanter-desktop/crates/op-core/src/config.rs @@ -15,6 +15,7 @@ pub const FOUNDRY_ANTHROPIC_API_KEY_PLACEHOLDER: &str = "dont-worry-it-will-be-i pub const ZAI_PAYGO_BASE_URL: &str = "https://api.z.ai/api/paas/v4"; pub const ZAI_CODING_BASE_URL: &str = "https://api.z.ai/api/coding/paas/v4"; pub const BRAVE_BASE_URL: &str = "https://api.search.brave.com/res/v1"; +pub const TAVILY_BASE_URL: &str = "https://api.tavily.com"; /// Default model for each supported provider. pub static PROVIDER_DEFAULT_MODELS: LazyLock> = @@ -77,6 +78,7 @@ pub fn normalize_web_search_provider(value: Option<&str>) -> String { match value.unwrap_or_default().trim().to_lowercase().as_str() { "firecrawl" => "firecrawl".to_string(), "brave" => "brave".to_string(), + "tavily" => "tavily".to_string(), _ => "exa".to_string(), } } @@ -192,6 +194,7 @@ pub struct AgentConfig { pub exa_base_url: String, pub firecrawl_base_url: String, pub brave_base_url: String, + pub tavily_base_url: String, // API keys pub api_key: Option, @@ -204,6 +207,7 @@ pub struct AgentConfig { pub exa_api_key: Option, pub firecrawl_api_key: Option, pub brave_api_key: Option, + pub tavily_api_key: Option, pub web_search_provider: String, pub voyage_api_key: Option, @@ -253,6 +257,7 @@ impl Default for AgentConfig { exa_base_url: "https://api.exa.ai".into(), firecrawl_base_url: "https://api.firecrawl.dev/v1".into(), brave_base_url: BRAVE_BASE_URL.into(), + tavily_base_url: TAVILY_BASE_URL.into(), api_key: Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.into()), openai_api_key: Some(FOUNDRY_OPENAI_API_KEY_PLACEHOLDER.into()), openai_oauth_token: None, @@ -263,6 +268,7 @@ impl Default for AgentConfig { exa_api_key: None, firecrawl_api_key: None, brave_api_key: None, + tavily_api_key: None, web_search_provider: "exa".into(), voyage_api_key: None, max_depth: 4, @@ -319,6 +325,8 @@ impl AgentConfig { env_opt("OPENPLANTER_FIRECRAWL_API_KEY").or_else(|| env_opt("FIRECRAWL_API_KEY")); let brave_api_key = env_opt("OPENPLANTER_BRAVE_API_KEY").or_else(|| env_opt("BRAVE_API_KEY")); + let tavily_api_key = + env_opt("OPENPLANTER_TAVILY_API_KEY").or_else(|| env_opt("TAVILY_API_KEY")); let voyage_api_key = env_opt("OPENPLANTER_VOYAGE_API_KEY").or_else(|| env_opt("VOYAGE_API_KEY")); @@ -385,6 +393,7 @@ impl AgentConfig { "https://api.firecrawl.dev/v1", ), brave_base_url: env_or("OPENPLANTER_BRAVE_BASE_URL", BRAVE_BASE_URL), + tavily_base_url: env_or("OPENPLANTER_TAVILY_BASE_URL", TAVILY_BASE_URL), openai_api_key, openai_oauth_token, anthropic_api_key, @@ -394,6 +403,7 @@ impl AgentConfig { exa_api_key, firecrawl_api_key, brave_api_key, + tavily_api_key, web_search_provider, voyage_api_key, max_depth: env_int("OPENPLANTER_MAX_DEPTH", 4), @@ -475,6 +485,8 @@ mod tests { assert_eq!(cfg.web_search_provider, "exa"); assert_eq!(cfg.brave_base_url, BRAVE_BASE_URL); assert!(cfg.brave_api_key.is_none()); + assert_eq!(cfg.tavily_base_url, TAVILY_BASE_URL); + assert!(cfg.tavily_api_key.is_none()); assert_eq!(cfg.rate_limit_max_retries, 12); assert_eq!(cfg.rate_limit_backoff_base_sec, 1.0); assert_eq!(cfg.rate_limit_backoff_max_sec, 60.0); @@ -532,6 +544,9 @@ mod tests { "OPENPLANTER_BRAVE_API_KEY", "BRAVE_API_KEY", "OPENPLANTER_BRAVE_BASE_URL", + "OPENPLANTER_TAVILY_API_KEY", + "TAVILY_API_KEY", + "OPENPLANTER_TAVILY_BASE_URL", "OPENPLANTER_ZAI_PLAN", "OPENPLANTER_ZAI_BASE_URL", "OPENPLANTER_RATE_LIMIT_MAX_RETRIES", @@ -568,6 +583,7 @@ mod tests { ); assert!(cfg.zai_api_key.is_none()); assert!(cfg.brave_api_key.is_none()); + assert!(cfg.tavily_api_key.is_none()); assert_eq!(cfg.openai_base_url, FOUNDRY_OPENAI_BASE_URL); assert_eq!(cfg.anthropic_base_url, FOUNDRY_ANTHROPIC_BASE_URL); assert_eq!(cfg.web_search_provider, "exa"); @@ -587,13 +603,15 @@ mod tests { env::set_var("OPENAI_API_KEY", "sk-test123"); env::set_var("ZAI_API_KEY", "zai-test123"); env::set_var("BRAVE_API_KEY", "brave-test123"); - env::set_var("OPENPLANTER_WEB_SEARCH_PROVIDER", "brave"); + env::set_var("TAVILY_API_KEY", "tavily-test123"); + env::set_var("OPENPLANTER_WEB_SEARCH_PROVIDER", "tavily"); env::set_var("OPENPLANTER_RATE_LIMIT_MAX_RETRIES", "5"); env::set_var("OPENPLANTER_RATE_LIMIT_BACKOFF_BASE_SEC", "2.5"); env::set_var("OPENPLANTER_RATE_LIMIT_BACKOFF_MAX_SEC", "30.0"); env::set_var("OPENPLANTER_RATE_LIMIT_RETRY_AFTER_CAP_SEC", "90.0"); env::set_var("OPENPLANTER_ZAI_PLAN", "coding"); env::set_var("OPENPLANTER_ZAI_STREAM_MAX_RETRIES", "7"); + env::set_var("OPENPLANTER_TAVILY_BASE_URL", "https://tavily.example"); } let cfg = AgentConfig::from_env("/tmp"); @@ -606,10 +624,12 @@ mod tests { assert_eq!(cfg.openai_api_key, Some("sk-test123".into())); assert_eq!(cfg.zai_api_key, Some("zai-test123".into())); assert_eq!(cfg.brave_api_key, Some("brave-test123".into())); + assert_eq!(cfg.tavily_api_key, Some("tavily-test123".into())); assert_eq!(cfg.zai_plan, "coding"); assert_eq!(cfg.zai_base_url, ZAI_CODING_BASE_URL); assert_eq!(cfg.zai_stream_max_retries, 7); - assert_eq!(cfg.web_search_provider, "brave"); + assert_eq!(cfg.web_search_provider, "tavily"); + assert_eq!(cfg.tavily_base_url, "https://tavily.example"); assert_eq!(cfg.rate_limit_max_retries, 5); assert_eq!(cfg.rate_limit_backoff_base_sec, 2.5); assert_eq!(cfg.rate_limit_backoff_max_sec, 30.0); @@ -658,6 +678,7 @@ mod tests { "firecrawl" ); assert_eq!(normalize_web_search_provider(Some("brave")), "brave"); + assert_eq!(normalize_web_search_provider(Some("tavily")), "tavily"); assert_eq!(normalize_web_search_provider(Some("other")), "exa"); assert!(is_foundry_openai_base_url(FOUNDRY_OPENAI_BASE_URL)); assert!(is_foundry_anthropic_base_url(FOUNDRY_ANTHROPIC_BASE_URL)); diff --git a/openplanter-desktop/crates/op-core/src/credentials.rs b/openplanter-desktop/crates/op-core/src/credentials.rs index 44817768..5ec9aa49 100644 --- a/openplanter-desktop/crates/op-core/src/credentials.rs +++ b/openplanter-desktop/crates/op-core/src/credentials.rs @@ -21,6 +21,7 @@ pub struct CredentialBundle { pub exa_api_key: Option, pub firecrawl_api_key: Option, pub brave_api_key: Option, + pub tavily_api_key: Option, pub voyage_api_key: Option, } @@ -37,6 +38,7 @@ impl CredentialBundle { &self.exa_api_key, &self.firecrawl_api_key, &self.brave_api_key, + &self.tavily_api_key, &self.voyage_api_key, ]; keys.iter() @@ -61,6 +63,7 @@ impl CredentialBundle { fill!(exa_api_key); fill!(firecrawl_api_key); fill!(brave_api_key); + fill!(tavily_api_key); fill!(voyage_api_key); } @@ -83,6 +86,7 @@ impl CredentialBundle { add!(exa_api_key, "exa_api_key"); add!(firecrawl_api_key, "firecrawl_api_key"); add!(brave_api_key, "brave_api_key"); + add!(tavily_api_key, "tavily_api_key"); add!(voyage_api_key, "voyage_api_key"); out } @@ -105,6 +109,7 @@ impl CredentialBundle { exa_api_key: get_str(payload, "exa_api_key"), firecrawl_api_key: get_str(payload, "firecrawl_api_key"), brave_api_key: get_str(payload, "brave_api_key"), + tavily_api_key: get_str(payload, "tavily_api_key"), voyage_api_key: get_str(payload, "voyage_api_key"), } } @@ -177,6 +182,7 @@ pub fn parse_env_file(path: &Path) -> CredentialBundle { "OPENPLANTER_FIRECRAWL_API_KEY", ), brave_api_key: get_key(&env_map, "BRAVE_API_KEY", "OPENPLANTER_BRAVE_API_KEY"), + tavily_api_key: get_key(&env_map, "TAVILY_API_KEY", "OPENPLANTER_TAVILY_API_KEY"), voyage_api_key: get_key(&env_map, "VOYAGE_API_KEY", "OPENPLANTER_VOYAGE_API_KEY"), } } @@ -201,6 +207,7 @@ pub fn credentials_from_env() -> CredentialBundle { exa_api_key: env_key("OPENPLANTER_EXA_API_KEY", "EXA_API_KEY"), firecrawl_api_key: env_key("OPENPLANTER_FIRECRAWL_API_KEY", "FIRECRAWL_API_KEY"), brave_api_key: env_key("OPENPLANTER_BRAVE_API_KEY", "BRAVE_API_KEY"), + tavily_api_key: env_key("OPENPLANTER_TAVILY_API_KEY", "TAVILY_API_KEY"), voyage_api_key: env_key("OPENPLANTER_VOYAGE_API_KEY", "VOYAGE_API_KEY"), } } @@ -383,6 +390,7 @@ mod tests { openrouter_api_key: Some("or-456".into()), firecrawl_api_key: Some("fc-789".into()), brave_api_key: Some("brave-101".into()), + tavily_api_key: Some("tavily-202".into()), ..Default::default() }; let json = bundle.to_json(); @@ -391,6 +399,7 @@ mod tests { assert_eq!(json.get("openrouter_api_key").unwrap(), "or-456"); assert_eq!(json.get("firecrawl_api_key").unwrap(), "fc-789"); assert_eq!(json.get("brave_api_key").unwrap(), "brave-101"); + assert_eq!(json.get("tavily_api_key").unwrap(), "tavily-202"); } #[test] @@ -407,6 +416,7 @@ EXA_API_KEY="exa-quoted" ZAI_API_KEY=zai-from-env OPENPLANTER_FIRECRAWL_API_KEY="firecrawl-quoted" BRAVE_API_KEY=brave-from-env +OPENPLANTER_TAVILY_API_KEY=tavily-from-env UNRELATED_VAR=foo "#, ) @@ -419,6 +429,7 @@ UNRELATED_VAR=foo assert_eq!(bundle.zai_api_key, Some("zai-from-env".into())); assert_eq!(bundle.firecrawl_api_key, Some("firecrawl-quoted".into())); assert_eq!(bundle.brave_api_key, Some("brave-from-env".into())); + assert_eq!(bundle.tavily_api_key, Some("tavily-from-env".into())); assert!(bundle.cerebras_api_key.is_none()); } @@ -431,6 +442,7 @@ UNRELATED_VAR=foo anthropic_api_key: Some("ant-test".into()), zai_api_key: Some("zai-test".into()), brave_api_key: Some("brave-test".into()), + tavily_api_key: Some("tavily-test".into()), ..Default::default() }; store.save(&bundle).unwrap(); @@ -439,6 +451,7 @@ UNRELATED_VAR=foo assert_eq!(loaded.anthropic_api_key, Some("ant-test".into())); assert_eq!(loaded.zai_api_key, Some("zai-test".into())); assert_eq!(loaded.brave_api_key, Some("brave-test".into())); + assert_eq!(loaded.tavily_api_key, Some("tavily-test".into())); } #[test] diff --git a/openplanter-desktop/crates/op-core/src/tools/defs.rs b/openplanter-desktop/crates/op-core/src/tools/defs.rs index 7b1d5835..88b268e5 100644 --- a/openplanter-desktop/crates/op-core/src/tools/defs.rs +++ b/openplanter-desktop/crates/op-core/src/tools/defs.rs @@ -176,7 +176,7 @@ fn mvp_tool_defs() -> Vec { // ── Web ── ToolDef { name: "web_search", - description: "Search the web using the configured Exa, Firecrawl, or Brave backend. Returns URLs, titles, snippets, and optional page text.", + description: "Search the web using the configured Exa, Firecrawl, Brave, or Tavily backend. Returns URLs, titles, snippets, and optional page text.", parameters: json!({ "type": "object", "properties": { @@ -199,7 +199,7 @@ fn mvp_tool_defs() -> Vec { }, ToolDef { name: "fetch_url", - description: "Fetch and return the text content of one or more URLs using the configured Exa, Firecrawl, or Brave backend.", + description: "Fetch and return the text content of one or more URLs using the configured Exa, Firecrawl, Brave, or Tavily backend.", parameters: json!({ "type": "object", "properties": { diff --git a/openplanter-desktop/crates/op-core/src/tools/mod.rs b/openplanter-desktop/crates/op-core/src/tools/mod.rs index f6220a92..693eb00e 100644 --- a/openplanter-desktop/crates/op-core/src/tools/mod.rs +++ b/openplanter-desktop/crates/op-core/src/tools/mod.rs @@ -60,6 +60,8 @@ pub struct WorkspaceTools { firecrawl_base_url: String, brave_api_key: Option, brave_base_url: String, + tavily_api_key: Option, + tavily_base_url: String, files_read: HashSet, bg_jobs: shell::BgJobs, } @@ -92,6 +94,8 @@ impl WorkspaceTools { firecrawl_base_url: config.firecrawl_base_url.clone(), brave_api_key: config.brave_api_key.clone(), brave_base_url: config.brave_base_url.clone(), + tavily_api_key: config.tavily_api_key.clone(), + tavily_base_url: config.tavily_base_url.clone(), files_read: HashSet::new(), bg_jobs: shell::BgJobs::new(), } @@ -120,6 +124,8 @@ impl WorkspaceTools { firecrawl_base_url: config.firecrawl_base_url.clone(), brave_api_key: config.brave_api_key.clone(), brave_base_url: config.brave_base_url.clone(), + tavily_api_key: config.tavily_api_key.clone(), + tavily_base_url: config.tavily_base_url.clone(), files_read: HashSet::new(), bg_jobs: shell::BgJobs::new(), } @@ -249,6 +255,8 @@ impl WorkspaceTools { &self.firecrawl_base_url, self.brave_api_key.as_deref(), &self.brave_base_url, + self.tavily_api_key.as_deref(), + &self.tavily_base_url, query, num_results, include_text, @@ -273,6 +281,10 @@ impl WorkspaceTools { &self.exa_base_url, self.firecrawl_api_key.as_deref(), &self.firecrawl_base_url, + self.brave_api_key.as_deref(), + &self.brave_base_url, + self.tavily_api_key.as_deref(), + &self.tavily_base_url, &urls, self.max_file_chars, self.command_timeout_sec, diff --git a/openplanter-desktop/crates/op-core/src/tools/web.rs b/openplanter-desktop/crates/op-core/src/tools/web.rs index 2b36060e..eda00835 100644 --- a/openplanter-desktop/crates/op-core/src/tools/web.rs +++ b/openplanter-desktop/crates/op-core/src/tools/web.rs @@ -1,6 +1,6 @@ -/// Web tools: Exa / Firecrawl / Brave search and fetch_url. -use std::time::Duration; use std::sync::LazyLock; +/// Web tools: Exa / Firecrawl / Brave / Tavily search and fetch_url. +use std::time::Duration; use regex::Regex; use serde_json::json; @@ -161,6 +161,40 @@ async fn brave_request( .map_err(|e| format!("Brave API returned non-JSON payload: {e}")) } +async fn tavily_request( + api_key: Option<&str>, + tavily_base_url: &str, + endpoint: &str, + payload: &serde_json::Value, + timeout_sec: u64, +) -> Result { + let api_key = match api_key { + Some(value) if !value.trim().is_empty() => value, + _ => return Err("TAVILY_API_KEY not configured".into()), + }; + + let url = format!("{}{}", tavily_base_url.trim_end_matches('/'), endpoint); + let client = reqwest::Client::new(); + let response = client + .post(&url) + .header("Authorization", format!("Bearer {api_key}")) + .header("Content-Type", "application/json") + .timeout(Duration::from_secs(timeout_sec)) + .json(payload) + .send() + .await + .map_err(|e| format!("Tavily API request failed: {e}"))?; + + let response = response + .error_for_status() + .map_err(|e| format!("Tavily API request failed: {e}"))?; + + response + .json::() + .await + .map_err(|e| format!("Tavily API returned non-JSON payload: {e}")) +} + async fn fetch_direct_page(url: &str, timeout_sec: u64) -> serde_json::Value { let client = reqwest::Client::new(); let response = match client @@ -240,6 +274,8 @@ pub async fn web_search( firecrawl_base_url: &str, brave_api_key: Option<&str>, brave_base_url: &str, + tavily_api_key: Option<&str>, + tavily_base_url: &str, query: &str, num_results: i64, include_text: bool, @@ -333,10 +369,7 @@ pub async fn web_search( Err(error) => return ToolResult::error(format!("Web search failed: {error}")), } } else if provider == "brave" { - let mut params = vec![ - ("q", query.to_string()), - ("count", clamped.to_string()), - ]; + let mut params = vec![("q", query.to_string()), ("count", clamped.to_string())]; if include_text { params.push(("extra_snippets", "true".to_string())); } @@ -411,6 +444,62 @@ pub async fn web_search( } Err(error) => return ToolResult::error(format!("Web search failed: {error}")), } + } else if provider == "tavily" { + let mut payload = json!({ + "query": query, + "max_results": clamped, + }); + if include_text { + payload["include_raw_content"] = json!("markdown"); + } + + match tavily_request( + tavily_api_key, + tavily_base_url, + "/search", + &payload, + timeout_sec, + ) + .await + { + Ok(body) => { + let mut results: Vec = Vec::new(); + if let Some(rows) = body.get("results").and_then(|value| value.as_array()) { + for row in rows { + let snippet = row + .get("content") + .and_then(|value| value.as_str()) + .or_else(|| row.get("snippet").and_then(|value| value.as_str())) + .unwrap_or(""); + let mut item = json!({ + "url": row.get("url").and_then(|value| value.as_str()).unwrap_or(""), + "title": row.get("title").and_then(|value| value.as_str()).unwrap_or(""), + "snippet": snippet, + }); + if include_text { + if let Some(text) = row + .get("raw_content") + .and_then(|value| value.as_str()) + .or_else(|| row.get("content").and_then(|value| value.as_str())) + { + if !text.is_empty() { + item["text"] = json!(clip(text, 4_000)); + } + } + } + results.push(item); + } + } + + json!({ + "query": query, + "provider": provider, + "results": results, + "total": results.len(), + }) + } + Err(error) => return ToolResult::error(format!("Web search failed: {error}")), + } } else { let mut payload = json!({ "query": query, @@ -468,6 +557,10 @@ pub async fn fetch_url( exa_base_url: &str, firecrawl_api_key: Option<&str>, firecrawl_base_url: &str, + brave_api_key: Option<&str>, + brave_base_url: &str, + tavily_api_key: Option<&str>, + tavily_base_url: &str, urls: &[String], max_file_chars: usize, timeout_sec: u64, @@ -534,6 +627,8 @@ pub async fn fetch_url( "total": pages.len(), }) } else if provider == "brave" { + let _ = brave_api_key; + let _ = brave_base_url; let mut pages: Vec = Vec::new(); for url in &normalized { pages.push(fetch_direct_page(url, timeout_sec).await); @@ -544,6 +639,48 @@ pub async fn fetch_url( "pages": pages, "total": pages.len(), }) + } else if provider == "tavily" { + let payload = json!({ + "urls": normalized, + "extract_depth": "basic", + "include_images": false, + }); + + match tavily_request( + tavily_api_key, + tavily_base_url, + "/extract", + &payload, + timeout_sec, + ) + .await + { + Ok(body) => { + let mut pages: Vec = Vec::new(); + if let Some(rows) = body.get("results").and_then(|value| value.as_array()) { + for row in rows { + pages.push(json!({ + "url": row.get("url").and_then(|value| value.as_str()).unwrap_or(""), + "title": row.get("title").and_then(|value| value.as_str()).unwrap_or(""), + "text": clip( + row.get("raw_content") + .and_then(|value| value.as_str()) + .or_else(|| row.get("content").and_then(|value| value.as_str())) + .unwrap_or(""), + 8_000, + ), + })); + } + } + + json!({ + "provider": provider, + "pages": pages, + "total": pages.len(), + }) + } + Err(error) => return ToolResult::error(format!("Fetch URL failed: {error}")), + } } else { let payload = json!({ "ids": normalized, @@ -705,6 +842,8 @@ mod tests { "https://api.firecrawl.dev/v1", None, "https://api.search.brave.com/res/v1", + None, + "https://api.tavily.com", "example query", 5, true, @@ -746,6 +885,8 @@ mod tests { &format!("http://{addr}"), None, "https://api.search.brave.com/res/v1", + None, + "https://api.tavily.com", "example query", 5, true, @@ -781,6 +922,10 @@ mod tests { "https://api.exa.ai", Some("fc-key"), &format!("http://{addr}"), + None, + "https://api.search.brave.com/res/v1", + None, + "https://api.tavily.com", &[String::from("https://example.com/article")], 20_000, 5, @@ -821,6 +966,8 @@ mod tests { "https://api.firecrawl.dev/v1", Some("brave-key"), &format!("http://{addr}"), + None, + "https://api.tavily.com", "example query", 5, true, @@ -833,7 +980,12 @@ mod tests { let parsed: Value = serde_json::from_str(&result.content).unwrap(); assert_eq!(parsed["provider"], "brave"); assert_eq!(parsed["results"][0]["title"], "Brave Title"); - assert!(parsed["results"][0]["text"].as_str().unwrap().contains("Extra context")); + assert!( + parsed["results"][0]["text"] + .as_str() + .unwrap() + .contains("Extra context") + ); } #[tokio::test] @@ -851,6 +1003,10 @@ mod tests { "https://api.exa.ai", None, "https://api.firecrawl.dev/v1", + None, + "https://api.search.brave.com/res/v1", + None, + "https://api.tavily.com", &[format!("http://{addr}/page")], 20_000, 5, @@ -861,7 +1017,12 @@ mod tests { let parsed: Value = serde_json::from_str(&result.content).unwrap(); assert_eq!(parsed["provider"], "brave"); assert_eq!(parsed["pages"][0]["title"], "Brave Page"); - assert!(parsed["pages"][0]["text"].as_str().unwrap().contains("Hello Brave")); + assert!( + parsed["pages"][0]["text"] + .as_str() + .unwrap() + .contains("Hello Brave") + ); } #[tokio::test] @@ -874,6 +1035,8 @@ mod tests { "https://api.firecrawl.dev/v1", None, "https://api.search.brave.com/res/v1", + None, + "https://api.tavily.com", "example query", 5, false, @@ -896,6 +1059,8 @@ mod tests { "https://api.firecrawl.dev/v1", None, "https://api.search.brave.com/res/v1", + None, + "https://api.tavily.com", "example query", 5, false, @@ -908,6 +1073,112 @@ mod tests { assert!(result.content.contains("BRAVE_API_KEY")); } + #[tokio::test] + async fn test_web_search_tavily_output_shape() { + let addr = start_json_server( + "/search", + json!({ + "results": [ + { + "url": "https://example.com/tavily", + "title": "Tavily Title", + "content": "Tavily snippet", + "raw_content": "Tavily raw content" + } + ] + }), + ) + .await; + + let result = web_search( + "tavily", + None, + "https://api.exa.ai", + None, + "https://api.firecrawl.dev/v1", + None, + "https://api.search.brave.com/res/v1", + Some("tavily-key"), + &format!("http://{addr}"), + "example query", + 5, + true, + 20_000, + 5, + ) + .await; + + assert!(!result.is_error); + let parsed: Value = serde_json::from_str(&result.content).unwrap(); + assert_eq!(parsed["provider"], "tavily"); + assert_eq!(parsed["results"][0]["title"], "Tavily Title"); + assert_eq!(parsed["results"][0]["snippet"], "Tavily snippet"); + assert_eq!(parsed["results"][0]["text"], "Tavily raw content"); + } + + #[tokio::test] + async fn test_fetch_url_tavily_output_shape() { + let addr = start_json_server( + "/extract", + json!({ + "results": [ + { + "url": "https://example.com/article", + "title": "Tavily Article", + "raw_content": "Article body" + } + ] + }), + ) + .await; + + let result = fetch_url( + "tavily", + None, + "https://api.exa.ai", + None, + "https://api.firecrawl.dev/v1", + None, + "https://api.search.brave.com/res/v1", + Some("tavily-key"), + &format!("http://{addr}"), + &[String::from("https://example.com/article")], + 20_000, + 5, + ) + .await; + + assert!(!result.is_error); + let parsed: Value = serde_json::from_str(&result.content).unwrap(); + assert_eq!(parsed["provider"], "tavily"); + assert_eq!(parsed["pages"][0]["title"], "Tavily Article"); + assert_eq!(parsed["pages"][0]["text"], "Article body"); + } + + #[tokio::test] + async fn test_missing_tavily_key_errors() { + let result = web_search( + "tavily", + None, + "https://api.exa.ai", + None, + "https://api.firecrawl.dev/v1", + None, + "https://api.search.brave.com/res/v1", + None, + "https://api.tavily.com", + "example query", + 5, + false, + 20_000, + 5, + ) + .await; + + assert!(result.is_error); + assert!(result.content.contains("TAVILY_API_KEY")); + } + #[tokio::test] async fn test_exa_http_error_bubbles_up() { let addr = start_status_server("/search", StatusCode::BAD_GATEWAY).await; @@ -920,6 +1191,8 @@ mod tests { "https://api.firecrawl.dev/v1", None, "https://api.search.brave.com/res/v1", + None, + "https://api.tavily.com", "example query", 5, false, diff --git a/openplanter-desktop/crates/op-tauri/src/commands/config.rs b/openplanter-desktop/crates/op-tauri/src/commands/config.rs index 9e624b3e..4d331489 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/config.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/config.rs @@ -210,6 +210,7 @@ pub fn build_credential_status(cfg: &op_core::config::AgentConfig) -> HashMap { exa: false, firecrawl: true, brave: false, + tavily: true, voyage: true, })); const status = await getCredentialsStatus(); @@ -135,6 +136,7 @@ describe("invoke wrappers", () => { expect(status.zai).toBe(true); expect(status.firecrawl).toBe(true); expect(status.brave).toBe(false); + expect(status.tavily).toBe(true); expect(status.voyage).toBe(true); }); diff --git a/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts b/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts index e019d03a..4ef78cf7 100644 --- a/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts +++ b/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts @@ -81,13 +81,13 @@ describe("completionRegistry", () => { expect(childValues).toEqual(["low", "medium", "high", "off"]); }); - it("/web-search has exa, firecrawl, and brave children", () => { + it("/web-search has exa, firecrawl, brave, and tavily children", () => { const webSearchCmd = COMMAND_COMPLETIONS.find((c) => c.value === "/web-search"); expect(webSearchCmd).toBeDefined(); expect(webSearchCmd!.children).toBeDefined(); const childValues = webSearchCmd!.children!.map((c) => c.value); - expect(childValues).toEqual(["exa", "firecrawl", "brave"]); + expect(childValues).toEqual(["exa", "firecrawl", "brave", "tavily"]); expect(webSearchCmd!.children![0].children?.[0].value).toBe("--save"); }); diff --git a/openplanter-desktop/frontend/src/commands/completionRegistry.ts b/openplanter-desktop/frontend/src/commands/completionRegistry.ts index 2133f2d3..973dc00e 100644 --- a/openplanter-desktop/frontend/src/commands/completionRegistry.ts +++ b/openplanter-desktop/frontend/src/commands/completionRegistry.ts @@ -40,6 +40,7 @@ const WEB_SEARCH_PROVIDERS: CompletionItem[] = [ { value: "exa", description: "Use Exa for web search", children: SAVE_FLAG }, { value: "firecrawl", description: "Use Firecrawl for web search", children: SAVE_FLAG }, { value: "brave", description: "Use Brave Search for web search", children: SAVE_FLAG }, + { value: "tavily", description: "Use Tavily for web search", children: SAVE_FLAG }, ]; const ZAI_PLANS: CompletionItem[] = [ diff --git a/openplanter-desktop/frontend/src/commands/slash.ts b/openplanter-desktop/frontend/src/commands/slash.ts index 34df61f1..748b312d 100644 --- a/openplanter-desktop/frontend/src/commands/slash.ts +++ b/openplanter-desktop/frontend/src/commands/slash.ts @@ -34,7 +34,7 @@ export async function dispatchSlashCommand(input: string): Promise Set Z.AI endpoint family (paygo, coding)", " /zai-plan --save Set and persist", " /web-search Show current web search provider", - " /web-search Set web search provider (exa, firecrawl, brave)", + " /web-search Set web search provider (exa, firecrawl, brave, tavily)", " /web-search --save Set and persist", " /reasoning Show/set reasoning effort", " /reasoning Set level (low, medium, high, off)", diff --git a/openplanter-desktop/frontend/src/commands/webSearch.test.ts b/openplanter-desktop/frontend/src/commands/webSearch.test.ts index cb5ed492..70d08e6d 100644 --- a/openplanter-desktop/frontend/src/commands/webSearch.test.ts +++ b/openplanter-desktop/frontend/src/commands/webSearch.test.ts @@ -31,7 +31,7 @@ describe("handleWebSearchCommand", () => { it("switches provider for the current session", async () => { __setHandler("update_config", ({ partial }: { partial: Record }) => { - expect(partial.web_search_provider).toBe("brave"); + expect(partial.web_search_provider).toBe("tavily"); return { provider: "anthropic", model: "claude-opus-4-6", @@ -42,14 +42,14 @@ describe("handleWebSearchCommand", () => { max_depth: 4, max_steps_per_call: 100, reasoning_effort: "high", - web_search_provider: "brave", + web_search_provider: "tavily", demo: false, }; }); - const result = await handleWebSearchCommand("brave"); - expect(result.lines).toContain("Web search provider set to: brave"); - expect(appState.get().webSearchProvider).toBe("brave"); + const result = await handleWebSearchCommand("tavily"); + expect(result.lines).toContain("Web search provider set to: tavily"); + expect(appState.get().webSearchProvider).toBe("tavily"); }); it("save persists the selected provider", async () => { @@ -63,14 +63,14 @@ describe("handleWebSearchCommand", () => { max_depth: 4, max_steps_per_call: 100, reasoning_effort: "high", - web_search_provider: "brave", + web_search_provider: "tavily", demo: false, })); __setHandler("save_settings", ({ settings }: { settings: Record }) => { - expect(settings.web_search_provider).toBe("brave"); + expect(settings.web_search_provider).toBe("tavily"); }); - const result = await handleWebSearchCommand("brave --save"); + const result = await handleWebSearchCommand("tavily --save"); expect(result.lines).toContain("(Settings saved)"); }); }); diff --git a/openplanter-desktop/frontend/src/commands/webSearch.ts b/openplanter-desktop/frontend/src/commands/webSearch.ts index c18ed806..9db4fdf4 100644 --- a/openplanter-desktop/frontend/src/commands/webSearch.ts +++ b/openplanter-desktop/frontend/src/commands/webSearch.ts @@ -3,7 +3,7 @@ import { saveSettings, updateConfig } from "../api/invoke"; import { appState } from "../state/store"; import type { CommandResult } from "./model"; -const VALID_WEB_SEARCH_PROVIDERS = ["exa", "firecrawl", "brave"]; +const VALID_WEB_SEARCH_PROVIDERS = ["exa", "firecrawl", "brave", "tavily"]; /** Handle /web-search [provider] [--save]. */ export async function handleWebSearchCommand(args: string): Promise { diff --git a/openplanter-desktop/frontend/src/components/App.test.ts b/openplanter-desktop/frontend/src/components/App.test.ts index 30037232..1a3d0bd6 100644 --- a/openplanter-desktop/frontend/src/components/App.test.ts +++ b/openplanter-desktop/frontend/src/components/App.test.ts @@ -48,7 +48,7 @@ describe("createApp", () => { __setHandler("list_sessions", () => [SESSION_B, SESSION_A]); __setHandler("get_credentials_status", () => ({ openai: true, anthropic: true, openrouter: false, - cerebras: false, zai: true, ollama: true, exa: false, firecrawl: true, brave: false, voyage: true, + cerebras: false, zai: true, ollama: true, exa: false, firecrawl: true, brave: false, tavily: true, voyage: true, })); __setHandler("open_session", () => ({ id: "20260227-120000-cccc3333", @@ -103,7 +103,7 @@ describe("createApp", () => { await vi.waitFor(() => { const creds = root.querySelector(".cred-status"); - expect(creds!.children.length).toBe(10); + expect(creds!.children.length).toBe(11); expect(creds!.querySelector(".cred-ok")!.textContent).toContain("openai"); expect(creds!.querySelector(".cred-missing")!.textContent).toContain("openrouter"); }); diff --git a/openplanter-desktop/frontend/src/components/App.ts b/openplanter-desktop/frontend/src/components/App.ts index 9b162fdf..715c0f38 100644 --- a/openplanter-desktop/frontend/src/components/App.ts +++ b/openplanter-desktop/frontend/src/components/App.ts @@ -302,7 +302,7 @@ async function loadCredentials(container: HTMLElement): Promise { try { const status = await getCredentialsStatus(); container.innerHTML = ""; - const providers = ["openai", "anthropic", "openrouter", "cerebras", "zai", "ollama", "exa", "firecrawl", "brave", "voyage"]; + const providers = ["openai", "anthropic", "openrouter", "cerebras", "zai", "ollama", "exa", "firecrawl", "brave", "tavily", "voyage"]; for (const p of providers) { const row = document.createElement("div"); const hasKey = status[p] ?? false; diff --git a/tests/test_coverage_gaps.py b/tests/test_coverage_gaps.py index 067e255e..25675a90 100644 --- a/tests/test_coverage_gaps.py +++ b/tests/test_coverage_gaps.py @@ -69,12 +69,18 @@ def test_inner_quotes_preserved(self) -> None: class MergeMissingTests(unittest.TestCase): def test_fills_missing_keys(self) -> None: a = CredentialBundle(openai_api_key="oa") - b = CredentialBundle(anthropic_api_key="an", exa_api_key="exa", brave_api_key="brave") + b = CredentialBundle( + anthropic_api_key="an", + exa_api_key="exa", + brave_api_key="brave", + tavily_api_key="tavily", + ) a.merge_missing(b) self.assertEqual(a.openai_api_key, "oa") self.assertEqual(a.anthropic_api_key, "an") self.assertEqual(a.exa_api_key, "exa") self.assertEqual(a.brave_api_key, "brave") + self.assertEqual(a.tavily_api_key, "tavily") def test_does_not_overwrite_existing(self) -> None: a = CredentialBundle(openai_api_key="mine") @@ -97,6 +103,7 @@ def test_merge_all_fields(self) -> None: cerebras_api_key="cb", exa_api_key="exa", brave_api_key="brave", + tavily_api_key="tavily", ) a.merge_missing(b) self.assertEqual(a.openai_api_key, "oa") @@ -105,6 +112,7 @@ def test_merge_all_fields(self) -> None: self.assertEqual(a.cerebras_api_key, "cb") self.assertEqual(a.exa_api_key, "exa") self.assertEqual(a.brave_api_key, "brave") + self.assertEqual(a.tavily_api_key, "tavily") # --------------------------------------------------------------------------- @@ -120,6 +128,7 @@ def test_reads_standard_env_vars(self) -> None: "OPENROUTER_API_KEY": "or-key", "EXA_API_KEY": "exa-key", "BRAVE_API_KEY": "brave-key", + "TAVILY_API_KEY": "tavily-key", } with patch.dict(os.environ, env, clear=True): creds = credentials_from_env() @@ -128,6 +137,7 @@ def test_reads_standard_env_vars(self) -> None: self.assertEqual(creds.openrouter_api_key, "or-key") self.assertEqual(creds.exa_api_key, "exa-key") self.assertEqual(creds.brave_api_key, "brave-key") + self.assertEqual(creds.tavily_api_key, "tavily-key") def test_rlm_prefix_takes_priority(self) -> None: env = { @@ -191,6 +201,8 @@ def test_custom_env_overrides(self) -> None: "OPENPLANTER_MAX_DEPTH": "5", "OPENPLANTER_MAX_STEPS": "20", "OPENPLANTER_SHELL": "/bin/bash", + "OPENPLANTER_WEB_SEARCH_PROVIDER": "tavily", + "OPENPLANTER_TAVILY_BASE_URL": "https://tavily.example", } with patch.dict(os.environ, env, clear=True): cfg = AgentConfig.from_env("/tmp/test-ws") @@ -200,6 +212,8 @@ def test_custom_env_overrides(self) -> None: self.assertEqual(cfg.max_depth, 5) self.assertEqual(cfg.max_steps_per_call, 20) self.assertEqual(cfg.shell, "/bin/bash") + self.assertEqual(cfg.web_search_provider, "tavily") + self.assertEqual(cfg.tavily_base_url, "https://tavily.example") def test_rate_limit_and_zai_stream_retries_from_env(self) -> None: env = { @@ -247,6 +261,7 @@ def test_api_keys_from_env(self) -> None: "OPENROUTER_API_KEY": "or", "EXA_API_KEY": "exa", "BRAVE_API_KEY": "brave", + "TAVILY_API_KEY": "tavily", } with patch.dict(os.environ, env, clear=True): cfg = AgentConfig.from_env("/tmp/test-ws") @@ -255,6 +270,7 @@ def test_api_keys_from_env(self) -> None: self.assertEqual(cfg.openrouter_api_key, "or") self.assertEqual(cfg.exa_api_key, "exa") self.assertEqual(cfg.brave_api_key, "brave") + self.assertEqual(cfg.tavily_api_key, "tavily") def test_openai_oauth_token_from_env_without_api_key(self) -> None: env = {"OPENAI_OAUTH_TOKEN": "oauth-token"} diff --git a/tests/test_credentials.py b/tests/test_credentials.py index 6d729824..161b66cb 100644 --- a/tests/test_credentials.py +++ b/tests/test_credentials.py @@ -27,6 +27,7 @@ def test_parse_env_file_extracts_supported_keys(self) -> None: "EXA_API_KEY=exa-key", "FIRECRAWL_API_KEY=fc-key", "BRAVE_API_KEY=brave-key", + "OPENPLANTER_TAVILY_API_KEY=tavily-key", ] ), encoding="utf-8", @@ -40,6 +41,7 @@ def test_parse_env_file_extracts_supported_keys(self) -> None: self.assertEqual(creds.exa_api_key, "exa-key") self.assertEqual(creds.firecrawl_api_key, "fc-key") self.assertEqual(creds.brave_api_key, "brave-key") + self.assertEqual(creds.tavily_api_key, "tavily-key") def test_store_roundtrip(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: @@ -54,6 +56,7 @@ def test_store_roundtrip(self) -> None: exa_api_key="exa", firecrawl_api_key="fc", brave_api_key="brave", + tavily_api_key="tavily", ) store.save(creds) loaded = store.load() diff --git a/tests/test_tools.py b/tests/test_tools.py index 6a5f9887..c1fd374d 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -203,6 +203,59 @@ def test_fetch_url_with_mocked_brave_response(self) -> None: self.assertEqual(parsed["pages"][0]["title"], "Brave Example") self.assertEqual(parsed["pages"][0]["text"], "Page body") + def test_web_search_with_mocked_tavily_response(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + tools = WorkspaceTools( + root=root, + web_search_provider="tavily", + tavily_api_key="tavily-key", + ) + mocked = { + "results": [ + { + "url": "https://example.com/tavily", + "title": "Tavily Result", + "content": "Snippet", + "raw_content": "Long markdown body", + } + ] + } + with patch.object(WorkspaceTools, "_tavily_request", return_value=mocked): + raw = tools.web_search("test query", num_results=3, include_text=True) + parsed = json.loads(raw) + self.assertEqual(parsed["provider"], "tavily") + self.assertEqual(parsed["query"], "test query") + self.assertEqual(parsed["total"], 1) + self.assertEqual(parsed["results"][0]["url"], "https://example.com/tavily") + self.assertEqual(parsed["results"][0]["snippet"], "Snippet") + self.assertEqual(parsed["results"][0]["text"], "Long markdown body") + + def test_fetch_url_with_mocked_tavily_response(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + tools = WorkspaceTools( + root=root, + web_search_provider="tavily", + tavily_api_key="tavily-key", + ) + mocked = { + "results": [ + { + "url": "https://example.com/tavily", + "title": "Tavily Example", + "raw_content": "Page body", + } + ] + } + with patch.object(WorkspaceTools, "_tavily_request", return_value=mocked): + raw = tools.fetch_url(["https://example.com/tavily"]) + parsed = json.loads(raw) + self.assertEqual(parsed["provider"], "tavily") + self.assertEqual(parsed["total"], 1) + self.assertEqual(parsed["pages"][0]["title"], "Tavily Example") + self.assertEqual(parsed["pages"][0]["text"], "Page body") + def test_web_search_without_exa_key(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) @@ -224,6 +277,13 @@ def test_web_search_without_brave_key(self) -> None: out = tools.web_search("test") self.assertIn("BRAVE_API_KEY not configured", out) + def test_web_search_without_tavily_key(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + tools = WorkspaceTools(root=root, web_search_provider="tavily", tavily_api_key=None) + out = tools.web_search("test") + self.assertIn("TAVILY_API_KEY not configured", out) + def test_repo_map_python_symbols(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) From f104f0c2ee087bb39fce37414bd10f5371bde881 Mon Sep 17 00:00:00 2001 From: Drake Date: Thu, 12 Mar 2026 15:58:58 -0400 Subject: [PATCH 09/58] Add desktop init and migration workflows --- .../crates/op-core/src/config_hydration.rs | 126 ++ .../crates/op-core/src/events.rs | 126 ++ openplanter-desktop/crates/op-core/src/lib.rs | 2 + .../crates/op-core/src/workspace_init.rs | 1176 +++++++++++++++++ .../crates/op-tauri/src/commands/agent.rs | 24 +- .../crates/op-tauri/src/commands/init.rs | 82 ++ .../crates/op-tauri/src/commands/mod.rs | 1 + .../crates/op-tauri/src/main.rs | 5 + .../crates/op-tauri/src/state.rs | 143 +- .../frontend/src/api/events.test.ts | 17 + .../frontend/src/api/events.ts | 15 +- .../frontend/src/api/invoke.test.ts | 82 ++ .../frontend/src/api/invoke.ts | 29 + openplanter-desktop/frontend/src/api/types.ts | 76 ++ .../src/commands/completionRegistry.test.ts | 13 + .../src/commands/completionRegistry.ts | 11 + .../frontend/src/commands/init.ts | 133 ++ .../frontend/src/commands/slash.test.ts | 43 + .../frontend/src/commands/slash.ts | 7 + .../frontend/src/components/App.test.ts | 40 +- .../frontend/src/components/App.ts | 5 + .../frontend/src/components/InputBar.test.ts | 47 +- .../frontend/src/components/InputBar.ts | 20 +- .../src/components/WorkspaceInitGate.ts | 402 ++++++ openplanter-desktop/frontend/src/main.ts | 26 +- .../frontend/src/state/store.test.ts | 3 + .../frontend/src/state/store.ts | 20 + 27 files changed, 2541 insertions(+), 133 deletions(-) create mode 100644 openplanter-desktop/crates/op-core/src/config_hydration.rs create mode 100644 openplanter-desktop/crates/op-core/src/workspace_init.rs create mode 100644 openplanter-desktop/crates/op-tauri/src/commands/init.rs create mode 100644 openplanter-desktop/frontend/src/commands/init.ts create mode 100644 openplanter-desktop/frontend/src/components/WorkspaceInitGate.ts diff --git a/openplanter-desktop/crates/op-core/src/config_hydration.rs b/openplanter-desktop/crates/op-core/src/config_hydration.rs new file mode 100644 index 00000000..90177523 --- /dev/null +++ b/openplanter-desktop/crates/op-core/src/config_hydration.rs @@ -0,0 +1,126 @@ +use std::env; + +use crate::config::{ + AgentConfig, FOUNDRY_OPENAI_API_KEY_PLACEHOLDER, normalize_web_search_provider, + normalize_zai_plan, resolve_openai_api_key, resolve_zai_base_url, +}; +use crate::credentials::CredentialBundle; +use crate::settings::PersistentSettings; + +/// Merge credentials into an AgentConfig. +/// Priority: existing config value > env_creds > file_creds. +pub fn merge_credentials_into_config( + cfg: &mut AgentConfig, + env_creds: &CredentialBundle, + file_creds: &CredentialBundle, +) { + if cfg.openai_oauth_token.is_none() { + cfg.openai_oauth_token = env_creds + .openai_oauth_token + .clone() + .or_else(|| file_creds.openai_oauth_token.clone()); + } + cfg.openai_api_key = cfg + .openai_api_key + .clone() + .filter(|value| { + let trimmed = value.trim(); + !trimmed.is_empty() && trimmed != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER + }) + .or_else(|| env_creds.openai_api_key.clone()) + .or_else(|| file_creds.openai_api_key.clone()) + .or_else(|| cfg.openai_api_key.clone()); + cfg.openai_api_key = resolve_openai_api_key( + cfg.openai_api_key.clone(), + &cfg.openai_base_url, + cfg.openai_oauth_token.clone(), + ); + cfg.api_key = resolve_openai_api_key( + cfg.openai_api_key + .clone() + .filter(|value| { + let trimmed = value.trim(); + !trimmed.is_empty() && trimmed != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER + }) + .or_else(|| { + cfg.api_key.clone().filter(|value| { + let trimmed = value.trim(); + !trimmed.is_empty() && trimmed != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER + }) + }) + .or_else(|| cfg.openai_api_key.clone()) + .or_else(|| cfg.api_key.clone()), + &cfg.base_url, + cfg.openai_oauth_token.clone(), + ); + + macro_rules! merge { + ($field:ident) => { + if cfg.$field.is_none() { + cfg.$field = env_creds + .$field + .clone() + .or_else(|| file_creds.$field.clone()); + } + }; + } + merge!(anthropic_api_key); + merge!(openrouter_api_key); + merge!(cerebras_api_key); + merge!(zai_api_key); + merge!(exa_api_key); + merge!(firecrawl_api_key); + merge!(brave_api_key); + merge!(tavily_api_key); + merge!(voyage_api_key); +} + +pub fn apply_settings_to_config(cfg: &mut AgentConfig, settings: &PersistentSettings) { + if !has_env_value(&["OPENPLANTER_REASONING_EFFORT"]) { + if let Some(reasoning_effort) = settings.default_reasoning_effort.clone() { + cfg.reasoning_effort = Some(reasoning_effort); + } + } + + if !has_env_value(&["OPENPLANTER_ZAI_PLAN"]) { + if let Some(plan) = settings.zai_plan.as_deref() { + cfg.zai_plan = normalize_zai_plan(Some(plan)); + } + } + + if !has_env_value(&["OPENPLANTER_ZAI_BASE_URL"]) { + cfg.zai_base_url = resolve_zai_base_url( + &cfg.zai_plan, + &cfg.zai_paygo_base_url, + &cfg.zai_coding_base_url, + ); + } + + if !has_env_value(&["OPENPLANTER_WEB_SEARCH_PROVIDER"]) { + if let Some(provider) = settings.web_search_provider.as_deref() { + cfg.web_search_provider = normalize_web_search_provider(Some(provider)); + } + } + + if !has_env_value(&["OPENPLANTER_MODEL"]) { + let saved_model = if cfg.provider == "auto" { + settings.default_model.as_deref() + } else { + settings + .default_model_for_provider(cfg.provider.as_str()) + .or(settings.default_model.as_deref()) + }; + if let Some(model) = saved_model { + cfg.model = model.to_string(); + } + } +} + +fn has_env_value(keys: &[&str]) -> bool { + keys.iter().any(|key| { + env::var(key) + .ok() + .map(|value| !value.trim().is_empty()) + .unwrap_or(false) + }) +} diff --git a/openplanter-desktop/crates/op-core/src/events.rs b/openplanter-desktop/crates/op-core/src/events.rs index 156cfce4..22c111d8 100644 --- a/openplanter-desktop/crates/op-core/src/events.rs +++ b/openplanter-desktop/crates/op-core/src/events.rs @@ -164,6 +164,116 @@ pub struct SlashResult { pub success: bool, } +/// Frontend gate state for workspace initialization. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum InitGateState { + Ready, + RequiresAction, + Blocked, +} + +/// Report returned by standard workspace initialization. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct StandardInitReportView { + pub workspace: String, + pub created_paths: Vec, + pub copied_paths: Vec, + pub skipped_existing: u64, + pub errors: Vec, + pub onboarding_required: bool, +} + +/// Current initialization state for the runtime workspace. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct InitStatusView { + pub runtime_workspace: String, + pub gate_state: String, + pub onboarding_completed: bool, + pub has_openplanter_root: bool, + pub has_runtime_wiki: bool, + pub has_runtime_index: bool, + pub init_state_path: String, + pub last_migration_target: Option, + pub warnings: Vec, +} + +/// Migration source classification. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum MigrationSourceKind { + OpenPlanterWorkspace, + ManualResearch, + Unknown, +} + +/// Inspection data for a migration source. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct MigrationSourceInspection { + pub path: String, + pub kind: String, + pub has_sessions: bool, + pub has_settings: bool, + pub has_credentials: bool, + pub has_runtime_wiki: bool, + pub has_baseline_wiki: bool, + pub markdown_files: u64, + pub warnings: Vec, +} + +/// A user-selected migration source. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct MigrationSourceInput { + pub path: String, +} + +/// Request payload for migration init. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct MigrationInitRequest { + pub target_workspace: String, + pub sources: Vec, +} + +/// Progress stages emitted during migration. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum MigrationProgressStage { + Inspect, + Copy, + MergeSessions, + MergeSettings, + MergeCredentials, + Synthesize, + Rewrite, + Done, +} + +/// Progress event emitted while migration runs. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct MigrationProgressEvent { + pub stage: String, + pub message: String, + pub current: u32, + pub total: u32, +} + +/// Result payload returned after migration init completes. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct MigrationInitResultView { + pub target_workspace: String, + pub sources: Vec, + pub sessions_copied: u64, + pub sessions_renamed: u64, + pub settings_merged_fields: Vec, + pub credentials_merged_fields: Vec, + pub wiki_files_synthesized: u64, + pub raw_preservation_root: String, + pub rewrite_summary: String, + pub restart_required: bool, + pub restart_message: String, + pub warnings: Vec, +} + #[cfg(test)] mod tests { use super::*; @@ -262,4 +372,20 @@ mod tests { assert_eq!(parsed["tool_name"], "read_file"); assert_eq!(parsed["tokens"]["input_tokens"], 1234); } + + #[test] + fn test_init_gate_state_serialization() { + assert_eq!( + serde_json::to_string(&InitGateState::RequiresAction).unwrap(), + "\"requires_action\"" + ); + } + + #[test] + fn test_migration_progress_stage_serialization() { + assert_eq!( + serde_json::to_string(&MigrationProgressStage::MergeSessions).unwrap(), + "\"merge_sessions\"" + ); + } } diff --git a/openplanter-desktop/crates/op-core/src/lib.rs b/openplanter-desktop/crates/op-core/src/lib.rs index 62efa5cf..aeb3a3ef 100644 --- a/openplanter-desktop/crates/op-core/src/lib.rs +++ b/openplanter-desktop/crates/op-core/src/lib.rs @@ -1,5 +1,6 @@ pub mod builder; pub mod config; +pub mod config_hydration; pub mod credentials; pub mod engine; pub mod events; @@ -9,3 +10,4 @@ pub mod session; pub mod settings; pub mod tools; pub mod wiki; +pub mod workspace_init; diff --git a/openplanter-desktop/crates/op-core/src/workspace_init.rs b/openplanter-desktop/crates/op-core/src/workspace_init.rs new file mode 100644 index 00000000..255fd9b1 --- /dev/null +++ b/openplanter-desktop/crates/op-core/src/workspace_init.rs @@ -0,0 +1,1176 @@ +use std::collections::{HashMap, HashSet}; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; + +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::runtime::Builder as TokioRuntimeBuilder; +use tokio_util::sync::CancellationToken; +use walkdir::WalkDir; + +use crate::config::AgentConfig; +use crate::config_hydration::{apply_settings_to_config, merge_credentials_into_config}; +use crate::credentials::{CredentialBundle, CredentialStore}; +use crate::engine::curator::{CuratorResult, run_curator}; +use crate::events::{ + InitGateState, InitStatusView, MigrationInitRequest, MigrationInitResultView, + MigrationProgressEvent, MigrationProgressStage, MigrationSourceInspection, MigrationSourceKind, + SessionInfo, StandardInitReportView, +}; +use crate::settings::{PersistentSettings, SettingsStore}; + +const INIT_STATE_FILE: &str = "init-state.json"; +const BASELINE_INDEX: &str = include_str!("../../../../wiki/index.md"); +const BASELINE_TEMPLATE: &str = include_str!("../../../../wiki/template.md"); + +#[derive(Debug, Error)] +pub enum WorkspaceInitError { + #[error("{0}")] + InvalidRequest(String), + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + #[error("Serialization error: {0}")] + Serde(#[from] serde_json::Error), + #[error("Curator rewrite failed: {0}")] + Curator(String), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct InitStateFile { + version: u32, + initialized_at: String, + last_standard_init_at: Option, + onboarding_completed: bool, + last_migration_target: Option, +} + +impl Default for InitStateFile { + fn default() -> Self { + Self { + version: 1, + initialized_at: now_rfc3339(), + last_standard_init_at: None, + onboarding_completed: false, + last_migration_target: None, + } + } +} + +#[derive(Debug, Clone)] +struct SourceSpec { + original: String, + canonical: PathBuf, + inspection: MigrationSourceInspection, +} + +pub fn run_standard_init( + workspace: &Path, + session_root_dir: &str, + mark_onboarding_complete: bool, +) -> Result { + let workspace = workspace.to_path_buf(); + let root = workspace.join(session_root_dir); + let wiki_dir = root.join("wiki"); + let index_path = wiki_dir.join("index.md"); + let init_path = root.join(INIT_STATE_FILE); + + let root_preexisting = root.exists(); + let index_preexisting = index_path.exists(); + let mut report = StandardInitReportView { + workspace: workspace.display().to_string(), + ..Default::default() + }; + + ensure_dir(&workspace, &mut report.created_paths)?; + ensure_dir(&root, &mut report.created_paths)?; + ensure_dir(&root.join("sessions"), &mut report.created_paths)?; + ensure_dir(&root.join("migration"), &mut report.created_paths)?; + ensure_dir( + &root.join("migration").join("raw"), + &mut report.created_paths, + )?; + ensure_dir(&wiki_dir, &mut report.created_paths)?; + + write_text_if_missing(&root.join("settings.json"), "{}", &mut report)?; + write_text_if_missing(&root.join("credentials.json"), "{}", &mut report)?; + write_text_if_missing(&index_path, BASELINE_INDEX, &mut report)?; + write_text_if_missing( + &wiki_dir.join("template.md"), + BASELINE_TEMPLATE, + &mut report, + )?; + + let mut state = read_init_state(&init_path).unwrap_or_else(|| InitStateFile { + onboarding_completed: root_preexisting || index_preexisting, + ..InitStateFile::default() + }); + if mark_onboarding_complete { + state.onboarding_completed = true; + } + state.last_standard_init_at = Some(now_rfc3339()); + write_init_state(&init_path, &state)?; + report.onboarding_required = !state.onboarding_completed; + + Ok(report) +} + +pub fn complete_first_run_gate( + workspace: &Path, + session_root_dir: &str, +) -> Result { + let _ = run_standard_init(workspace, session_root_dir, true)?; + get_init_status(workspace, session_root_dir) +} + +pub fn get_init_status( + workspace: &Path, + session_root_dir: &str, +) -> Result { + let root = workspace.join(session_root_dir); + let wiki_dir = root.join("wiki"); + let index_path = wiki_dir.join("index.md"); + let init_path = root.join(INIT_STATE_FILE); + let mut warnings = Vec::new(); + let init_state = match fs::read_to_string(&init_path) { + Ok(content) => match serde_json::from_str::(&content) { + Ok(state) => Some(state), + Err(err) => { + warnings.push(format!("Failed to parse init state: {err}")); + None + } + }, + Err(_) => None, + }; + let onboarding_completed = init_state + .as_ref() + .map(|state| state.onboarding_completed) + .unwrap_or_else(|| root.exists() && index_path.exists()); + let gate_state = + if root.exists() && wiki_dir.exists() && index_path.exists() && onboarding_completed { + InitGateState::Ready + } else { + InitGateState::RequiresAction + }; + + Ok(InitStatusView { + runtime_workspace: workspace.display().to_string(), + gate_state: gate_state_name(gate_state).to_string(), + onboarding_completed, + has_openplanter_root: root.exists(), + has_runtime_wiki: wiki_dir.exists(), + has_runtime_index: index_path.exists(), + init_state_path: init_path.display().to_string(), + last_migration_target: init_state.and_then(|state| state.last_migration_target), + warnings, + }) +} + +pub fn inspect_migration_source(path: &Path) -> MigrationSourceInspection { + let canonical = canonicalize_or_self(path); + let openplanter_root = canonical.join(".openplanter"); + let runtime_wiki = openplanter_root.join("wiki"); + let baseline_wiki = canonical.join("wiki"); + let markdown_files = count_markdown_files(&canonical); + let kind = if openplanter_root.exists() { + MigrationSourceKind::OpenPlanterWorkspace + } else if markdown_files > 0 { + MigrationSourceKind::ManualResearch + } else { + MigrationSourceKind::Unknown + }; + + MigrationSourceInspection { + path: canonical.display().to_string(), + kind: source_kind_name(kind).to_string(), + has_sessions: openplanter_root.join("sessions").exists(), + has_settings: openplanter_root.join("settings.json").exists(), + has_credentials: openplanter_root.join("credentials.json").exists(), + has_runtime_wiki: runtime_wiki.exists(), + has_baseline_wiki: baseline_wiki.exists(), + markdown_files, + warnings: Vec::new(), + } +} + +pub fn run_migration_init( + request: &MigrationInitRequest, + runtime_config: &AgentConfig, + emit_progress: F, +) -> Result +where + F: FnMut(MigrationProgressEvent), +{ + run_migration_init_with_runner(request, runtime_config, emit_progress, run_curator_blocking) +} + +fn run_migration_init_with_runner( + request: &MigrationInitRequest, + runtime_config: &AgentConfig, + mut emit_progress: F, + mut curator_runner: R, +) -> Result +where + F: FnMut(MigrationProgressEvent), + R: FnMut(&str, &AgentConfig) -> Result, +{ + if request.target_workspace.trim().is_empty() { + return Err(WorkspaceInitError::InvalidRequest( + "Target workspace is required".to_string(), + )); + } + if request.sources.is_empty() { + return Err(WorkspaceInitError::InvalidRequest( + "At least one migration source is required".to_string(), + )); + } + + let session_root_dir = runtime_config.session_root_dir.as_str(); + let target = canonicalize_target_path(&expand_home(&request.target_workspace))?; + let total = request.sources.len() as u32; + let mut source_specs = Vec::new(); + let mut seen_sources = HashSet::new(); + + for (index, source) in request.sources.iter().enumerate() { + let source_path = expand_home(&source.path); + if !source_path.exists() { + return Err(WorkspaceInitError::InvalidRequest(format!( + "Source does not exist: {}", + source.path + ))); + } + let canonical = canonicalize_or_self(&source_path); + if canonical == target { + return Err(WorkspaceInitError::InvalidRequest( + "Target workspace cannot also be a source".to_string(), + )); + } + if !seen_sources.insert(canonical.clone()) { + return Err(WorkspaceInitError::InvalidRequest(format!( + "Duplicate source: {}", + canonical.display() + ))); + } + emit_progress(progress_event( + MigrationProgressStage::Inspect, + format!("Inspecting {}", canonical.display()), + (index + 1) as u32, + total, + )); + source_specs.push(SourceSpec { + original: source.path.clone(), + canonical: canonical.clone(), + inspection: inspect_migration_source(&canonical), + }); + } + + let _ = run_standard_init(&target, session_root_dir, false)?; + let root = target.join(session_root_dir); + let raw_root = root.join("migration").join("raw"); + let target_sessions_dir = root.join("sessions"); + let target_wiki_dir = root.join("wiki"); + let mut warnings = Vec::new(); + let mut raw_specs = Vec::new(); + + for (index, spec) in source_specs.iter().enumerate() { + let slug = format!( + "{:02}-{}", + index + 1, + slugify_component(&display_name(&spec.canonical)) + ); + let raw_dest = raw_root.join(slug); + emit_progress(progress_event( + MigrationProgressStage::Copy, + format!("Copying raw content from {}", spec.canonical.display()), + (index + 1) as u32, + total, + )); + copy_source_snapshot(&spec.canonical, &raw_dest, &spec.inspection, &mut warnings)?; + raw_specs.push((spec.clone(), raw_dest)); + } + + emit_progress(progress_event( + MigrationProgressStage::MergeSessions, + "Merging sessions".to_string(), + 0, + total, + )); + let mut sessions_copied = 0u64; + let mut sessions_renamed = 0u64; + for (_, raw_dest) in &raw_specs { + let sessions_dir = raw_dest.join(".openplanter").join("sessions"); + if !sessions_dir.exists() { + continue; + } + for entry in fs::read_dir(&sessions_dir)? { + let entry = entry?; + if !entry.path().is_dir() { + continue; + } + let original_id = entry.file_name().to_string_lossy().to_string(); + let resolved_id = unique_session_id(&target_sessions_dir, &original_id); + if resolved_id != original_id { + sessions_renamed += 1; + } + let target_session_dir = target_sessions_dir.join(&resolved_id); + copy_dir_all(&entry.path(), &target_session_dir)?; + rewrite_session_metadata_id(&target_session_dir, &resolved_id)?; + sessions_copied += 1; + } + } + + emit_progress(progress_event( + MigrationProgressStage::MergeSettings, + "Merging settings".to_string(), + 0, + total, + )); + let settings_store = SettingsStore::new(&target, session_root_dir); + let mut merged_settings = settings_store.load(); + let mut settings_fields = Vec::new(); + for (_, raw_dest) in &raw_specs { + let settings_path = raw_dest.join(".openplanter").join("settings.json"); + if settings_path.exists() { + let incoming = read_settings_from_path(&settings_path)?; + merge_settings_missing(&mut merged_settings, &incoming, &mut settings_fields); + } + } + settings_store.save(&merged_settings)?; + settings_fields.sort(); + settings_fields.dedup(); + + emit_progress(progress_event( + MigrationProgressStage::MergeCredentials, + "Merging credentials".to_string(), + 0, + total, + )); + let credential_store = CredentialStore::new(&target, session_root_dir); + let mut merged_credentials = credential_store.load(); + let mut credential_fields = Vec::new(); + for (_, raw_dest) in &raw_specs { + let credentials_path = raw_dest.join(".openplanter").join("credentials.json"); + if credentials_path.exists() { + let incoming = read_credentials_from_path(&credentials_path)?; + merge_credentials_missing(&mut merged_credentials, &incoming, &mut credential_fields); + } + } + credential_store.save(&merged_credentials)?; + credential_fields.sort(); + credential_fields.dedup(); + + emit_progress(progress_event( + MigrationProgressStage::Synthesize, + "Preparing the target wiki for a one-time curator rewrite".to_string(), + 0, + 1, + )); + clear_runtime_wiki_documents(&target_wiki_dir)?; + let curator_context = build_migration_curator_context(&target, &raw_root, &raw_specs); + let curator_config = build_target_curator_config( + runtime_config, + &target, + &merged_settings, + &merged_credentials, + ); + + emit_progress(progress_event( + MigrationProgressStage::Rewrite, + "Running a one-time curator rewrite over imported sources".to_string(), + 0, + 1, + )); + let curator_result = curator_runner(&curator_context, &curator_config)?; + let rewrite_summary = normalize_rewrite_summary(&curator_result); + let wiki_files_synthesized = count_runtime_wiki_pages(&target_wiki_dir); + emit_progress(progress_event( + MigrationProgressStage::Rewrite, + rewrite_summary.clone(), + 1, + 1, + )); + + let init_path = root.join(INIT_STATE_FILE); + let mut state = read_init_state(&init_path).unwrap_or_default(); + state.onboarding_completed = true; + state.last_migration_target = Some(target.display().to_string()); + state.last_standard_init_at = Some(now_rfc3339()); + write_init_state(&init_path, &state)?; + + let result = MigrationInitResultView { + target_workspace: target.display().to_string(), + sources: raw_specs + .iter() + .map(|(spec, _)| spec.canonical.display().to_string()) + .collect(), + sessions_copied, + sessions_renamed, + settings_merged_fields: settings_fields, + credentials_merged_fields: credential_fields, + wiki_files_synthesized, + raw_preservation_root: raw_root.display().to_string(), + rewrite_summary, + restart_required: true, + restart_message: format!( + "Migration completed. Restart OpenPlanter with OPENPLANTER_WORKSPACE={} to use the new Desktop workspace.", + target.display() + ), + warnings, + }; + + emit_progress(progress_event( + MigrationProgressStage::Done, + "Migration complete".to_string(), + total, + total, + )); + Ok(result) +} + +fn now_rfc3339() -> String { + Utc::now().to_rfc3339() +} + +fn gate_state_name(state: InitGateState) -> &'static str { + match state { + InitGateState::Ready => "ready", + InitGateState::RequiresAction => "requires_action", + InitGateState::Blocked => "blocked", + } +} + +fn source_kind_name(kind: MigrationSourceKind) -> &'static str { + match kind { + MigrationSourceKind::OpenPlanterWorkspace => "openplanter_workspace", + MigrationSourceKind::ManualResearch => "manual_research", + MigrationSourceKind::Unknown => "unknown", + } +} + +fn progress_stage_name(stage: MigrationProgressStage) -> &'static str { + match stage { + MigrationProgressStage::Inspect => "inspect", + MigrationProgressStage::Copy => "copy", + MigrationProgressStage::MergeSessions => "merge_sessions", + MigrationProgressStage::MergeSettings => "merge_settings", + MigrationProgressStage::MergeCredentials => "merge_credentials", + MigrationProgressStage::Synthesize => "synthesize", + MigrationProgressStage::Rewrite => "rewrite", + MigrationProgressStage::Done => "done", + } +} + +fn progress_event( + stage: MigrationProgressStage, + message: String, + current: u32, + total: u32, +) -> MigrationProgressEvent { + MigrationProgressEvent { + stage: progress_stage_name(stage).to_string(), + message, + current, + total, + } +} + +fn read_init_state(path: &Path) -> Option { + let content = fs::read_to_string(path).ok()?; + serde_json::from_str(&content).ok() +} + +fn write_init_state(path: &Path, state: &InitStateFile) -> Result<(), WorkspaceInitError> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, serde_json::to_string_pretty(state)?)?; + Ok(()) +} + +fn ensure_dir(path: &Path, created_paths: &mut Vec) -> Result<(), WorkspaceInitError> { + if !path.exists() { + fs::create_dir_all(path)?; + created_paths.push(path.display().to_string()); + } + Ok(()) +} + +fn write_text_if_missing( + path: &Path, + contents: &str, + report: &mut StandardInitReportView, +) -> Result<(), WorkspaceInitError> { + if path.exists() { + report.skipped_existing += 1; + return Ok(()); + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, contents)?; + report.copied_paths.push(path.display().to_string()); + Ok(()) +} + +fn expand_home(raw: &str) -> PathBuf { + if raw == "~" { + return home_dir().unwrap_or_else(|| PathBuf::from(raw)); + } + if let Some(rest) = raw.strip_prefix("~/") { + if let Some(home) = home_dir() { + return home.join(rest); + } + } + PathBuf::from(raw) +} + +fn home_dir() -> Option { + #[cfg(windows)] + { + env::var_os("USERPROFILE").map(PathBuf::from) + } + #[cfg(not(windows))] + { + env::var_os("HOME").map(PathBuf::from) + } +} + +fn canonicalize_or_self(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) +} + +fn canonicalize_target_path(path: &Path) -> Result { + if path.exists() { + return Ok(canonicalize_or_self(path)); + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + Ok(path.to_path_buf()) +} + +fn count_markdown_files(path: &Path) -> u64 { + WalkDir::new(path) + .into_iter() + .filter_entry(|entry| !should_skip_walk_entry(entry.path())) + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .filter(|entry| is_markdown(entry.path())) + .count() as u64 +} + +fn should_skip_walk_entry(path: &Path) -> bool { + path.file_name() + .and_then(|value| value.to_str()) + .map(|name| { + matches!( + name, + ".git" | "node_modules" | "target" | "dist" | "__pycache__" + ) + }) + .unwrap_or(false) +} + +fn is_markdown(path: &Path) -> bool { + matches!( + path.extension().and_then(|value| value.to_str()), + Some("md") | Some("markdown") + ) +} + +fn display_name(path: &Path) -> String { + path.file_name() + .and_then(|value| value.to_str()) + .map(ToString::to_string) + .unwrap_or_else(|| path.display().to_string()) +} + +fn slugify_component(text: &str) -> String { + let slug = text + .to_lowercase() + .chars() + .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '-' }) + .collect::() + .split('-') + .filter(|part| !part.is_empty()) + .collect::>() + .join("-"); + if slug.is_empty() { + "workspace".to_string() + } else { + slug + } +} + +fn copy_source_snapshot( + source: &Path, + raw_dest: &Path, + inspection: &MigrationSourceInspection, + warnings: &mut Vec, +) -> Result<(), WorkspaceInitError> { + fs::create_dir_all(raw_dest)?; + let openplanter_root = source.join(".openplanter"); + + if inspection.has_settings { + copy_file( + &openplanter_root.join("settings.json"), + &raw_dest.join(".openplanter").join("settings.json"), + )?; + } + if inspection.has_credentials { + copy_file( + &openplanter_root.join("credentials.json"), + &raw_dest.join(".openplanter").join("credentials.json"), + )?; + } + if inspection.has_sessions { + copy_dir_all( + &openplanter_root.join("sessions"), + &raw_dest.join(".openplanter").join("sessions"), + )?; + } + if inspection.has_runtime_wiki { + copy_dir_all( + &openplanter_root.join("wiki"), + &raw_dest.join(".openplanter").join("wiki"), + )?; + } else if inspection.has_baseline_wiki { + copy_dir_all(&source.join("wiki"), &raw_dest.join("wiki"))?; + } + + if inspection.kind == source_kind_name(MigrationSourceKind::ManualResearch) { + let docs_root = raw_dest.join("documents"); + let mut copied_any = false; + for entry in WalkDir::new(source) + .into_iter() + .filter_entry(|entry| !should_skip_walk_entry(entry.path())) + .filter_map(Result::ok) + { + if !entry.file_type().is_file() || !is_markdown(entry.path()) { + continue; + } + let rel = match entry.path().strip_prefix(source) { + Ok(rel) => rel, + Err(_) => continue, + }; + copy_file(entry.path(), &docs_root.join(rel))?; + copied_any = true; + } + if !copied_any { + warnings.push(format!( + "No markdown documents found in manual source {}", + source.display() + )); + } + } + + Ok(()) +} + +fn copy_file(src: &Path, dst: &Path) -> Result<(), WorkspaceInitError> { + if let Some(parent) = dst.parent() { + fs::create_dir_all(parent)?; + } + fs::copy(src, dst)?; + Ok(()) +} + +fn copy_dir_all(src: &Path, dst: &Path) -> Result<(), WorkspaceInitError> { + if !src.exists() { + return Ok(()); + } + for entry in WalkDir::new(src).into_iter().filter_map(Result::ok) { + let rel = match entry.path().strip_prefix(src) { + Ok(rel) => rel, + Err(_) => continue, + }; + let target = dst.join(rel); + if entry.file_type().is_dir() { + fs::create_dir_all(&target)?; + } else if entry.file_type().is_file() { + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + fs::copy(entry.path(), &target)?; + } + } + Ok(()) +} + +fn unique_session_id(target_sessions_dir: &Path, original_id: &str) -> String { + let mut candidate = original_id.to_string(); + let mut suffix = 1u32; + while target_sessions_dir.join(&candidate).exists() { + suffix += 1; + candidate = format!("{original_id}-m{suffix}"); + } + candidate +} + +fn rewrite_session_metadata_id(session_dir: &Path, new_id: &str) -> Result<(), WorkspaceInitError> { + let metadata_path = session_dir.join("metadata.json"); + if !metadata_path.exists() { + return Ok(()); + } + let content = fs::read_to_string(&metadata_path)?; + let mut info: SessionInfo = serde_json::from_str(&content)?; + info.id = new_id.to_string(); + fs::write(&metadata_path, serde_json::to_string_pretty(&info)?)?; + Ok(()) +} + +fn read_settings_from_path(path: &Path) -> Result { + let content = fs::read_to_string(path)?; + let parsed: serde_json::Value = serde_json::from_str(&content)?; + Ok(PersistentSettings::from_json(&parsed).unwrap_or_default()) +} + +fn merge_settings_missing( + target: &mut PersistentSettings, + incoming: &PersistentSettings, + filled_fields: &mut Vec, +) { + macro_rules! fill { + ($field:ident) => { + if target.$field.is_none() && incoming.$field.is_some() { + target.$field = incoming.$field.clone(); + filled_fields.push(stringify!($field).to_string()); + } + }; + } + fill!(default_model); + fill!(default_reasoning_effort); + fill!(default_model_openai); + fill!(default_model_anthropic); + fill!(default_model_openrouter); + fill!(default_model_cerebras); + fill!(default_model_zai); + fill!(default_model_ollama); + fill!(zai_plan); + fill!(web_search_provider); +} + +fn read_credentials_from_path(path: &Path) -> Result { + let content = fs::read_to_string(path)?; + let parsed: HashMap = serde_json::from_str(&content)?; + Ok(CredentialBundle::from_json(&parsed)) +} + +fn merge_credentials_missing( + target: &mut CredentialBundle, + incoming: &CredentialBundle, + filled_fields: &mut Vec, +) { + macro_rules! fill { + ($field:ident) => { + if target.$field.is_none() && incoming.$field.is_some() { + target.$field = incoming.$field.clone(); + filled_fields.push(stringify!($field).to_string()); + } + }; + } + fill!(openai_api_key); + fill!(openai_oauth_token); + fill!(anthropic_api_key); + fill!(openrouter_api_key); + fill!(cerebras_api_key); + fill!(zai_api_key); + fill!(exa_api_key); + fill!(firecrawl_api_key); + fill!(brave_api_key); + fill!(tavily_api_key); + fill!(voyage_api_key); +} + +fn clear_runtime_wiki_documents(wiki_dir: &Path) -> Result<(), WorkspaceInitError> { + if !wiki_dir.exists() { + return Ok(()); + } + for entry in fs::read_dir(wiki_dir)? { + let entry = entry?; + let path = entry.path(); + let name = entry.file_name(); + let keep = name == "index.md" || name == "template.md"; + if keep { + continue; + } + if path.is_dir() { + fs::remove_dir_all(path)?; + } else { + fs::remove_file(path)?; + } + } + Ok(()) +} + +fn build_target_curator_config( + runtime_config: &AgentConfig, + target: &Path, + merged_settings: &PersistentSettings, + merged_credentials: &CredentialBundle, +) -> AgentConfig { + let mut config = runtime_config.clone(); + config.workspace = target.to_path_buf(); + apply_settings_to_config(&mut config, merged_settings); + merge_credentials_into_config( + &mut config, + merged_credentials, + &CredentialBundle::default(), + ); + config +} + +fn build_migration_curator_context( + target: &Path, + raw_root: &Path, + raw_specs: &[(SourceSpec, PathBuf)], +) -> String { + let raw_root_display = raw_root + .strip_prefix(target) + .unwrap_or(raw_root) + .display() + .to_string(); + let mut lines = vec![ + "You are performing a one-time workspace migration rewrite for the Desktop app." + .to_string(), + format!("Target workspace: {}", target.display()), + "Rewrite the canonical Desktop wiki inside `.openplanter/wiki/`.".to_string(), + format!( + "Read imported raw material from `{raw_root_display}` and treat it as the source of truth." + ), + "Merge duplicate information across sources, keep the result factual and legible, preserve provenance, and update `.openplanter/wiki/index.md` to match the final page set.".to_string(), + "Do not write outside `.openplanter/wiki/`, and do not modify raw snapshots under `.openplanter/migration/raw/`.".to_string(), + String::new(), + "Ordered import sources:".to_string(), + ]; + for (index, (spec, raw_dest)) in raw_specs.iter().enumerate() { + let raw_display = raw_dest + .strip_prefix(target) + .unwrap_or(raw_dest) + .display() + .to_string(); + lines.push(format!( + "{}. kind={} | source={} | original_input={} | raw_snapshot={}", + index + 1, + spec.inspection.kind, + spec.canonical.display(), + spec.original, + raw_display + )); + } + lines.join("\n") +} + +fn normalize_rewrite_summary(result: &CuratorResult) -> String { + let summary = result.summary.trim(); + if summary.is_empty() { + format!( + "Curator rewrite completed with {} wiki file(s) changed.", + result.files_changed + ) + } else { + summary.to_string() + } +} + +fn count_runtime_wiki_pages(wiki_dir: &Path) -> u64 { + WalkDir::new(wiki_dir) + .into_iter() + .filter_entry(|entry| !should_skip_walk_entry(entry.path())) + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .filter(|entry| is_markdown(entry.path())) + .filter(|entry| { + entry + .path() + .file_name() + .and_then(|value| value.to_str()) + .map(|name| { + !name.eq_ignore_ascii_case("index.md") + && !name.eq_ignore_ascii_case("template.md") + }) + .unwrap_or(true) + }) + .count() as u64 +} + +fn run_curator_blocking( + context: &str, + config: &AgentConfig, +) -> Result { + let runtime = TokioRuntimeBuilder::new_current_thread() + .enable_all() + .build() + .map_err(|err| WorkspaceInitError::Curator(err.to_string()))?; + runtime + .block_on(run_curator(context, config, CancellationToken::new())) + .map_err(WorkspaceInitError::Curator) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::MigrationSourceInput; + use tempfile::tempdir; + + fn runtime_config(workspace: &Path) -> AgentConfig { + let mut cfg = AgentConfig::from_env(workspace); + cfg.workspace = workspace.to_path_buf(); + cfg.provider = "auto".to_string(); + cfg.model = "seed-model".to_string(); + cfg.api_key = None; + cfg.openai_api_key = None; + cfg.openai_oauth_token = None; + cfg + } + + #[test] + fn standard_init_is_idempotent() { + let temp = tempdir().unwrap(); + let first = run_standard_init(temp.path(), ".openplanter", false).unwrap(); + assert!( + temp.path() + .join(".openplanter") + .join("wiki") + .join("index.md") + .exists() + ); + assert!(first.onboarding_required); + + let second = run_standard_init(temp.path(), ".openplanter", true).unwrap(); + assert!(!second.onboarding_required); + + let status = get_init_status(temp.path(), ".openplanter").unwrap(); + assert_eq!(status.gate_state, "ready"); + } + + #[test] + fn inspect_source_detects_openplanter_workspace() { + let temp = tempdir().unwrap(); + let root = temp.path().join(".openplanter"); + fs::create_dir_all(root.join("sessions")).unwrap(); + fs::write(root.join("settings.json"), "{}").unwrap(); + fs::write(root.join("credentials.json"), "{}").unwrap(); + fs::create_dir_all(root.join("wiki")).unwrap(); + fs::write(root.join("wiki").join("index.md"), BASELINE_INDEX).unwrap(); + + let inspection = inspect_migration_source(temp.path()); + assert_eq!(inspection.kind, "openplanter_workspace"); + assert!(inspection.has_sessions); + assert!(inspection.has_settings); + } + + #[test] + fn migration_preserves_sources_and_merges_sessions() { + let temp = tempdir().unwrap(); + let source_a = temp.path().join("source-a"); + let source_b = temp.path().join("source-b"); + let target = temp.path().join("target"); + + for source in [&source_a, &source_b] { + fs::create_dir_all(source.join(".openplanter").join("sessions").join("same-id")) + .unwrap(); + fs::create_dir_all( + source + .join(".openplanter") + .join("wiki") + .join("campaign-finance"), + ) + .unwrap(); + fs::write( + source + .join(".openplanter") + .join("sessions") + .join("same-id") + .join("metadata.json"), + serde_json::to_string_pretty(&SessionInfo { + id: "same-id".to_string(), + created_at: "2026-01-01T00:00:00Z".to_string(), + turn_count: 1, + last_objective: Some("Investigate".to_string()), + }) + .unwrap(), + ) + .unwrap(); + fs::write( + source + .join(".openplanter") + .join("wiki") + .join("campaign-finance") + .join(format!("{}.md", display_name(source))), + format!( + "# {}\n\n## Summary\n\nImported from {}\n", + display_name(source), + source.display() + ), + ) + .unwrap(); + } + + fs::write( + source_a.join(".openplanter").join("settings.json"), + "{\"default_model\":\"alpha\"}", + ) + .unwrap(); + fs::write( + source_b.join(".openplanter").join("credentials.json"), + "{\"openai_api_key\":\"secret\"}", + ) + .unwrap(); + + let request = MigrationInitRequest { + target_workspace: target.display().to_string(), + sources: vec![ + MigrationSourceInput { + path: source_a.display().to_string(), + }, + MigrationSourceInput { + path: source_b.display().to_string(), + }, + ], + }; + + let mut progress = Vec::new(); + let mut run_count = 0u32; + let source_a_display = source_a.display().to_string(); + let source_b_display = source_b.display().to_string(); + let result = run_migration_init_with_runner( + &request, + &runtime_config(temp.path()), + |event| progress.push(event.stage), + |context, cfg| { + run_count += 1; + assert!(context.contains(".openplanter/migration/raw")); + assert!(context.contains(&source_a_display)); + assert!(context.contains(&source_b_display)); + assert_eq!(cfg.workspace, target); + assert_eq!(cfg.model, "alpha"); + assert_eq!(cfg.openai_api_key.as_deref(), Some("secret")); + + let wiki_dir = cfg.workspace.join(&cfg.session_root_dir).join("wiki"); + fs::create_dir_all(wiki_dir.join("campaign-finance")).unwrap(); + fs::write( + wiki_dir.join("campaign-finance").join("merged.md"), + "# Merged Source\n\n## Overview\n\nCurated output.\n", + ) + .unwrap(); + fs::write(wiki_dir.join("index.md"), BASELINE_INDEX).unwrap(); + + Ok(CuratorResult { + summary: "Curator rewrote 1 wiki file from imported sources.".to_string(), + files_changed: 1, + }) + }, + ) + .unwrap(); + + assert_eq!(result.sessions_copied, 2); + assert_eq!(result.sessions_renamed, 1); + assert_eq!(result.wiki_files_synthesized, 1); + assert_eq!( + result.rewrite_summary, + "Curator rewrote 1 wiki file from imported sources." + ); + assert_eq!(run_count, 1); + assert!( + target + .join(".openplanter") + .join("migration") + .join("raw") + .exists() + ); + assert!( + source_a + .join(".openplanter") + .join("sessions") + .join("same-id") + .exists() + ); + assert!( + target + .join(".openplanter") + .join("wiki") + .join("campaign-finance") + .exists() + || target + .join(".openplanter") + .join("wiki") + .join("imported") + .exists() + ); + let settings = SettingsStore::new(&target, ".openplanter").load(); + assert_eq!(settings.default_model.as_deref(), Some("alpha")); + let creds = CredentialStore::new(&target, ".openplanter").load(); + assert_eq!(creds.openai_api_key.as_deref(), Some("secret")); + let synth_index = progress + .iter() + .position(|stage| stage == "synthesize") + .unwrap(); + let rewrite_index = progress + .iter() + .position(|stage| stage == "rewrite") + .unwrap(); + assert!(synth_index < rewrite_index); + assert_eq!( + progress + .iter() + .filter(|stage| stage.as_str() == "rewrite") + .count(), + 2 + ); + assert_eq!(progress.last().map(String::as_str), Some("done")); + } + + #[test] + fn migration_surfaces_curator_errors_after_preserving_raw_sources() { + let temp = tempdir().unwrap(); + let source = temp.path().join("source-a"); + let target = temp.path().join("target"); + + fs::create_dir_all(source.join(".openplanter").join("sessions").join("same-id")).unwrap(); + fs::create_dir_all(source.join(".openplanter").join("wiki")).unwrap(); + fs::write( + source.join(".openplanter").join("wiki").join("source-a.md"), + "# Source A\n", + ) + .unwrap(); + + let request = MigrationInitRequest { + target_workspace: target.display().to_string(), + sources: vec![MigrationSourceInput { + path: source.display().to_string(), + }], + }; + + let result = run_migration_init_with_runner( + &request, + &runtime_config(temp.path()), + |_| {}, + |_context, _cfg| { + Err(WorkspaceInitError::Curator( + "missing credentials".to_string(), + )) + }, + ); + + assert!(matches!( + result, + Err(WorkspaceInitError::Curator(message)) if message == "missing credentials" + )); + assert!( + target + .join(".openplanter") + .join("migration") + .join("raw") + .exists() + ); + assert!( + source + .join(".openplanter") + .join("wiki") + .join("source-a.md") + .exists() + ); + } +} diff --git a/openplanter-desktop/crates/op-tauri/src/commands/agent.rs b/openplanter-desktop/crates/op-tauri/src/commands/agent.rs index 201ab9df..d251af51 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/agent.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/agent.rs @@ -6,6 +6,7 @@ use crate::commands::session::sessions_dir; use crate::state::AppState; use op_core::engine::SolveEmitter; use op_core::session::replay::{ReplayEntry, ReplayLogger}; +use op_core::workspace_init; /// Start solving an objective. Result streamed via events. #[tauri::command] @@ -15,15 +16,29 @@ pub async fn solve( app: AppHandle, state: State<'_, AppState>, ) -> Result<(), String> { + let cfg = state.config.lock().await.clone(); + let init_status = workspace_init::get_init_status(&cfg.workspace, &cfg.session_root_dir) + .map_err(|e| e.to_string())?; + if init_status.gate_state != "ready" { + return Err("Workspace initialization is not complete. Run /init first.".to_string()); + } + + { + let mut running = state.agent_running.lock().await; + if *running { + return Err("An agent task is already running.".to_string()); + } + *running = true; + } + // Create a fresh cancellation token for this solve run let token = CancellationToken::new(); { let mut current = state.cancel_token.lock().await; *current = token.clone(); } - - let cfg = state.config.lock().await.clone(); let error_handle = app.clone(); + let running_flag = state.agent_running.clone(); // Set up replay logging for this session let session_dir = sessions_dir(&state).await.join(&session_id); @@ -74,6 +89,11 @@ pub async fn solve( }) .await; + { + let mut running = running_flag.lock().await; + *running = false; + } + // If the inner task panicked, emit an error so the frontend // doesn't get stuck in "running" state forever. if let Err(e) = result { diff --git a/openplanter-desktop/crates/op-tauri/src/commands/init.rs b/openplanter-desktop/crates/op-tauri/src/commands/init.rs new file mode 100644 index 00000000..8e96e7f6 --- /dev/null +++ b/openplanter-desktop/crates/op-tauri/src/commands/init.rs @@ -0,0 +1,82 @@ +use std::path::PathBuf; + +use crate::state::AppState; +use op_core::events::{ + InitStatusView, MigrationInitRequest, MigrationInitResultView, MigrationSourceInspection, + StandardInitReportView, +}; +use op_core::workspace_init; +use tauri::{AppHandle, Emitter, State}; + +async fn current_workspace_config(state: &State<'_, AppState>) -> op_core::config::AgentConfig { + state.config.lock().await.clone() +} + +async fn ensure_idle(state: &State<'_, AppState>) -> Result<(), String> { + if *state.agent_running.lock().await { + return Err("Cannot run init while the agent is active".to_string()); + } + Ok(()) +} + +#[tauri::command] +pub async fn get_init_status(state: State<'_, AppState>) -> Result { + let cfg = current_workspace_config(&state).await; + workspace_init::get_init_status(&cfg.workspace, &cfg.session_root_dir) + .map_err(|e| e.to_string()) +} + +#[tauri::command] +pub async fn run_standard_init( + state: State<'_, AppState>, +) -> Result { + ensure_idle(&state).await?; + let _guard = state.init_lock.lock().await; + let cfg = current_workspace_config(&state).await; + tokio::task::spawn_blocking(move || { + workspace_init::run_standard_init(&cfg.workspace, &cfg.session_root_dir, true) + }) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string()) +} + +#[tauri::command] +pub async fn complete_first_run_gate(state: State<'_, AppState>) -> Result { + ensure_idle(&state).await?; + let _guard = state.init_lock.lock().await; + let cfg = current_workspace_config(&state).await; + tokio::task::spawn_blocking(move || { + workspace_init::complete_first_run_gate(&cfg.workspace, &cfg.session_root_dir) + }) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string()) +} + +#[tauri::command] +pub async fn inspect_migration_source(path: String) -> Result { + let path = PathBuf::from(path); + tokio::task::spawn_blocking(move || workspace_init::inspect_migration_source(&path)) + .await + .map_err(|e| e.to_string()) +} + +#[tauri::command] +pub async fn run_migration_init( + request: MigrationInitRequest, + app: AppHandle, + state: State<'_, AppState>, +) -> Result { + ensure_idle(&state).await?; + let _guard = state.init_lock.lock().await; + let cfg = current_workspace_config(&state).await; + tokio::task::spawn_blocking(move || { + workspace_init::run_migration_init(&request, &cfg, |event| { + let _ = app.emit("init:migration-progress", event); + }) + }) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string()) +} diff --git a/openplanter-desktop/crates/op-tauri/src/commands/mod.rs b/openplanter-desktop/crates/op-tauri/src/commands/mod.rs index 4f02ad95..b4525a46 100644 --- a/openplanter-desktop/crates/op-tauri/src/commands/mod.rs +++ b/openplanter-desktop/crates/op-tauri/src/commands/mod.rs @@ -1,4 +1,5 @@ pub mod agent; pub mod config; +pub mod init; pub mod session; pub mod wiki; diff --git a/openplanter-desktop/crates/op-tauri/src/main.rs b/openplanter-desktop/crates/op-tauri/src/main.rs index edf948cf..49cefe20 100644 --- a/openplanter-desktop/crates/op-tauri/src/main.rs +++ b/openplanter-desktop/crates/op-tauri/src/main.rs @@ -23,6 +23,11 @@ fn main() { commands::config::list_models, commands::config::save_settings, commands::config::get_credentials_status, + commands::init::get_init_status, + commands::init::run_standard_init, + commands::init::complete_first_run_gate, + commands::init::inspect_migration_source, + commands::init::run_migration_init, commands::session::list_sessions, commands::session::open_session, commands::session::delete_session, diff --git a/openplanter-desktop/crates/op-tauri/src/state.rs b/openplanter-desktop/crates/op-tauri/src/state.rs index f19f1be5..6a649dda 100644 --- a/openplanter-desktop/crates/op-tauri/src/state.rs +++ b/openplanter-desktop/crates/op-tauri/src/state.rs @@ -1,11 +1,11 @@ -use op_core::config::{ - AgentConfig, FOUNDRY_OPENAI_API_KEY_PLACEHOLDER, normalize_web_search_provider, - normalize_zai_plan, resolve_openai_api_key, resolve_zai_base_url, -}; -use op_core::credentials::{ - CredentialBundle, credentials_from_env, discover_env_candidates, parse_env_file, -}; -use op_core::settings::{PersistentSettings, SettingsStore}; +use op_core::config::AgentConfig; +use op_core::config_hydration::{apply_settings_to_config, merge_credentials_into_config}; +use op_core::credentials::CredentialBundle; +use op_core::credentials::{credentials_from_env, discover_env_candidates, parse_env_file}; +#[cfg(test)] +use op_core::settings::PersistentSettings; +use op_core::settings::SettingsStore; +use op_core::workspace_init; use std::env; use std::fs; use std::path::{Path, PathBuf}; @@ -37,124 +37,6 @@ struct LegacyMigrationReport { errors: Vec, } -/// Merge credentials into an AgentConfig. -/// Priority: existing config value > env_creds > file_creds. -pub fn merge_credentials_into_config( - cfg: &mut AgentConfig, - env_creds: &CredentialBundle, - file_creds: &CredentialBundle, -) { - if cfg.openai_oauth_token.is_none() { - cfg.openai_oauth_token = env_creds - .openai_oauth_token - .clone() - .or_else(|| file_creds.openai_oauth_token.clone()); - } - cfg.openai_api_key = cfg - .openai_api_key - .clone() - .filter(|value| { - let trimmed = value.trim(); - !trimmed.is_empty() && trimmed != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER - }) - .or_else(|| env_creds.openai_api_key.clone()) - .or_else(|| file_creds.openai_api_key.clone()) - .or_else(|| cfg.openai_api_key.clone()); - cfg.openai_api_key = resolve_openai_api_key( - cfg.openai_api_key.clone(), - &cfg.openai_base_url, - cfg.openai_oauth_token.clone(), - ); - cfg.api_key = resolve_openai_api_key( - cfg.openai_api_key - .clone() - .filter(|value| { - let trimmed = value.trim(); - !trimmed.is_empty() && trimmed != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER - }) - .or_else(|| { - cfg.api_key.clone().filter(|value| { - let trimmed = value.trim(); - !trimmed.is_empty() && trimmed != FOUNDRY_OPENAI_API_KEY_PLACEHOLDER - }) - }) - .or_else(|| cfg.openai_api_key.clone()) - .or_else(|| cfg.api_key.clone()), - &cfg.base_url, - cfg.openai_oauth_token.clone(), - ); - - macro_rules! merge { - ($field:ident) => { - if cfg.$field.is_none() { - cfg.$field = env_creds - .$field - .clone() - .or_else(|| file_creds.$field.clone()); - } - }; - } - merge!(anthropic_api_key); - merge!(openrouter_api_key); - merge!(cerebras_api_key); - merge!(zai_api_key); - merge!(exa_api_key); - merge!(firecrawl_api_key); - merge!(brave_api_key); - merge!(tavily_api_key); - merge!(voyage_api_key); -} - -fn has_env_value(keys: &[&str]) -> bool { - keys.iter().any(|key| { - env::var(key) - .ok() - .map(|value| !value.trim().is_empty()) - .unwrap_or(false) - }) -} - -fn apply_settings_to_config(cfg: &mut AgentConfig, settings: &PersistentSettings) { - if !has_env_value(&["OPENPLANTER_REASONING_EFFORT"]) { - if let Some(reasoning_effort) = settings.default_reasoning_effort.clone() { - cfg.reasoning_effort = Some(reasoning_effort); - } - } - - if !has_env_value(&["OPENPLANTER_ZAI_PLAN"]) { - if let Some(plan) = settings.zai_plan.as_deref() { - cfg.zai_plan = normalize_zai_plan(Some(plan)); - } - } - - if !has_env_value(&["OPENPLANTER_ZAI_BASE_URL"]) { - cfg.zai_base_url = resolve_zai_base_url( - &cfg.zai_plan, - &cfg.zai_paygo_base_url, - &cfg.zai_coding_base_url, - ); - } - - if !has_env_value(&["OPENPLANTER_WEB_SEARCH_PROVIDER"]) { - if let Some(provider) = settings.web_search_provider.as_deref() { - cfg.web_search_provider = normalize_web_search_provider(Some(provider)); - } - } - - if !has_env_value(&["OPENPLANTER_MODEL"]) { - let saved_model = if cfg.provider == "auto" { - settings.default_model.as_deref() - } else { - settings - .default_model_for_provider(cfg.provider.as_str()) - .or(settings.default_model.as_deref()) - }; - if let Some(model) = saved_model { - cfg.model = model.to_string(); - } - } -} - fn canonicalize_or_self(path: &Path) -> PathBuf { path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) } @@ -372,6 +254,8 @@ pub struct AppState { pub config: Arc>, pub session_id: Arc>>, pub cancel_token: Arc>, + pub agent_running: Arc>, + pub init_lock: Arc>, startup_trace: String, } @@ -381,6 +265,11 @@ impl AppState { let resolved_workspace = resolve_desktop_workspace(); let mut cfg = AgentConfig::from_env(&resolved_workspace.path); let migration = migrate_legacy_desktop_state(&cfg.workspace, &cfg.session_root_dir); + if let Err(err) = + workspace_init::run_standard_init(&cfg.workspace, &cfg.session_root_dir, false) + { + eprintln!("[startup:init] {err}"); + } // Load .env files and merge credentials into config let env_creds = credentials_from_env(); @@ -403,6 +292,8 @@ impl AppState { config: Arc::new(Mutex::new(cfg)), session_id: Arc::new(Mutex::new(None)), cancel_token: Arc::new(Mutex::new(CancellationToken::new())), + agent_running: Arc::new(Mutex::new(false)), + init_lock: Arc::new(Mutex::new(())), startup_trace: format_startup_trace(¤t_dir, &resolved_workspace, &migration), } } diff --git a/openplanter-desktop/frontend/src/api/events.test.ts b/openplanter-desktop/frontend/src/api/events.test.ts index f7620ae7..88ea1cc1 100644 --- a/openplanter-desktop/frontend/src/api/events.test.ts +++ b/openplanter-desktop/frontend/src/api/events.test.ts @@ -17,6 +17,7 @@ import { onAgentDelta, onAgentComplete, onAgentError, + onMigrationProgress, onWikiUpdated, } from "./events"; @@ -98,6 +99,21 @@ describe("event listeners", () => { expect(callback).toHaveBeenCalledWith(graphData); }); + it("onMigrationProgress registers listener and forwards progress payload", async () => { + const callback = vi.fn(); + await onMigrationProgress(callback); + + const handler = listeners.get("init:migration-progress")!; + const payload = { + stage: "copy", + message: "Copying raw content", + current: 1, + total: 3, + }; + handler({ payload }); + expect(callback).toHaveBeenCalledWith(payload); + }); + it("all listeners return unlisten function", async () => { const noop = vi.fn(); const unlistens = await Promise.all([ @@ -106,6 +122,7 @@ describe("event listeners", () => { onAgentDelta(noop), onAgentComplete(noop), onAgentError(noop), + onMigrationProgress(noop), onWikiUpdated(noop), ]); for (const u of unlistens) { diff --git a/openplanter-desktop/frontend/src/api/events.ts b/openplanter-desktop/frontend/src/api/events.ts index 30cb0704..845ba8b9 100644 --- a/openplanter-desktop/frontend/src/api/events.ts +++ b/openplanter-desktop/frontend/src/api/events.ts @@ -1,6 +1,11 @@ /** Tauri event subscriptions. */ import { listen, type UnlistenFn } from "@tauri-apps/api/event"; -import type { AgentEvent, CuratorUpdateEvent, GraphData } from "./types"; +import type { + AgentEvent, + CuratorUpdateEvent, + GraphData, + MigrationProgressEvent, +} from "./types"; export function onAgentTrace( callback: (message: string) => void @@ -51,3 +56,11 @@ export function onCuratorUpdate( callback(e.payload) ); } + +export function onMigrationProgress( + callback: (event: MigrationProgressEvent) => void +): Promise { + return listen("init:migration-progress", (e) => + callback(e.payload) + ); +} diff --git a/openplanter-desktop/frontend/src/api/invoke.test.ts b/openplanter-desktop/frontend/src/api/invoke.test.ts index 965fbbbd..62a2f232 100644 --- a/openplanter-desktop/frontend/src/api/invoke.test.ts +++ b/openplanter-desktop/frontend/src/api/invoke.test.ts @@ -18,7 +18,11 @@ import { openSession, deleteSession, getGraphData, + getInitStatus, + inspectMigrationSource, debugLog, + runMigrationInit, + runStandardInit, } from "./invoke"; describe("invoke wrappers", () => { @@ -211,6 +215,84 @@ describe("invoke wrappers", () => { await debugLog("test message"); }); + it("getInitStatus calls invoke", async () => { + __setHandler("get_init_status", () => ({ + runtime_workspace: "/tmp/ws", + gate_state: "requires_action", + onboarding_completed: false, + has_openplanter_root: true, + has_runtime_wiki: true, + has_runtime_index: true, + init_state_path: "/tmp/ws/.openplanter/init-state.json", + last_migration_target: null, + warnings: [], + })); + const status = await getInitStatus(); + expect(status.runtime_workspace).toBe("/tmp/ws"); + expect(status.gate_state).toBe("requires_action"); + }); + + it("runStandardInit calls invoke", async () => { + __setHandler("run_standard_init", () => ({ + workspace: "/tmp/ws", + created_paths: ["/tmp/ws/.openplanter"], + copied_paths: ["/tmp/ws/.openplanter/wiki/index.md"], + skipped_existing: 0, + errors: [], + onboarding_required: false, + })); + const report = await runStandardInit(); + expect(report.workspace).toBe("/tmp/ws"); + expect(report.created_paths).toHaveLength(1); + }); + + it("inspectMigrationSource sends path", async () => { + __setHandler("inspect_migration_source", ({ path }: any) => { + expect(path).toBe("/tmp/source"); + return { + path, + kind: "manual_research", + has_sessions: false, + has_settings: false, + has_credentials: false, + has_runtime_wiki: false, + has_baseline_wiki: false, + markdown_files: 4, + warnings: [], + }; + }); + const inspection = await inspectMigrationSource("/tmp/source"); + expect(inspection.kind).toBe("manual_research"); + expect(inspection.markdown_files).toBe(4); + }); + + it("runMigrationInit sends request payload", async () => { + __setHandler("run_migration_init", ({ request }: any) => { + expect(request.target_workspace).toBe("/tmp/target"); + expect(request.sources).toEqual([{ path: "/tmp/a" }, { path: "/tmp/b" }]); + return { + target_workspace: "/tmp/target", + sources: ["/tmp/a", "/tmp/b"], + sessions_copied: 2, + sessions_renamed: 1, + settings_merged_fields: ["default_model"], + credentials_merged_fields: ["openai_api_key"], + wiki_files_synthesized: 3, + raw_preservation_root: "/tmp/target/.openplanter/migration/raw", + rewrite_summary: "Curator rewrote 3 wiki files from imported sources.", + restart_required: true, + restart_message: "Restart required", + warnings: [], + }; + }); + const result = await runMigrationInit({ + target_workspace: "/tmp/target", + sources: [{ path: "/tmp/a" }, { path: "/tmp/b" }], + }); + expect(result.sessions_copied).toBe(2); + expect(result.restart_required).toBe(true); + }); + it("unhandled command rejects", async () => { await expect(solve("test", "s1")).rejects.toThrow("No mock for command: solve"); }); diff --git a/openplanter-desktop/frontend/src/api/invoke.ts b/openplanter-desktop/frontend/src/api/invoke.ts index c3094dee..f07662a5 100644 --- a/openplanter-desktop/frontend/src/api/invoke.ts +++ b/openplanter-desktop/frontend/src/api/invoke.ts @@ -3,11 +3,16 @@ import { invoke } from "@tauri-apps/api/core"; import type { ConfigView, GraphData, + InitStatusView, + MigrationInitRequest, + MigrationInitResultView, + MigrationSourceInspection, ModelInfo, PartialConfig, PersistentSettings, ReplayEntry, SessionInfo, + StandardInitReportView, } from "./types"; export async function solve(objective: string, sessionId: string): Promise { @@ -68,3 +73,27 @@ export async function readWikiFile(path: string): Promise { export async function debugLog(msg: string): Promise { return invoke("debug_log", { msg }); } + +export async function getInitStatus(): Promise { + return invoke("get_init_status"); +} + +export async function runStandardInit(): Promise { + return invoke("run_standard_init"); +} + +export async function completeFirstRunGate(): Promise { + return invoke("complete_first_run_gate"); +} + +export async function inspectMigrationSource( + path: string +): Promise { + return invoke("inspect_migration_source", { path }); +} + +export async function runMigrationInit( + request: MigrationInitRequest +): Promise { + return invoke("run_migration_init", { request }); +} diff --git a/openplanter-desktop/frontend/src/api/types.ts b/openplanter-desktop/frontend/src/api/types.ts index 9bc29eb7..22c4d605 100644 --- a/openplanter-desktop/frontend/src/api/types.ts +++ b/openplanter-desktop/frontend/src/api/types.ts @@ -114,6 +114,82 @@ export interface SlashResult { success: boolean; } +export type InitGateState = "ready" | "requires_action" | "blocked"; +export type MigrationSourceKind = "openplanter_workspace" | "manual_research" | "unknown"; +export type MigrationProgressStage = + | "inspect" + | "copy" + | "merge_sessions" + | "merge_settings" + | "merge_credentials" + | "synthesize" + | "rewrite" + | "done"; + +export interface StandardInitReportView { + workspace: string; + created_paths: string[]; + copied_paths: string[]; + skipped_existing: number; + errors: string[]; + onboarding_required: boolean; +} + +export interface InitStatusView { + runtime_workspace: string; + gate_state: InitGateState; + onboarding_completed: boolean; + has_openplanter_root: boolean; + has_runtime_wiki: boolean; + has_runtime_index: boolean; + init_state_path: string; + last_migration_target: string | null; + warnings: string[]; +} + +export interface MigrationSourceInspection { + path: string; + kind: MigrationSourceKind; + has_sessions: boolean; + has_settings: boolean; + has_credentials: boolean; + has_runtime_wiki: boolean; + has_baseline_wiki: boolean; + markdown_files: number; + warnings: string[]; +} + +export interface MigrationSourceInput { + path: string; +} + +export interface MigrationInitRequest { + target_workspace: string; + sources: MigrationSourceInput[]; +} + +export interface MigrationProgressEvent { + stage: MigrationProgressStage; + message: string; + current: number; + total: number; +} + +export interface MigrationInitResultView { + target_workspace: string; + sources: string[]; + sessions_copied: number; + sessions_renamed: number; + settings_merged_fields: string[]; + credentials_merged_fields: string[]; + wiki_files_synthesized: number; + raw_preservation_root: string; + rewrite_summary: string; + restart_required: boolean; + restart_message: string; + warnings: string[]; +} + export interface StepToolCallEntry { name: string; key_arg: string; diff --git a/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts b/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts index 4ef78cf7..42915f39 100644 --- a/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts +++ b/openplanter-desktop/frontend/src/commands/completionRegistry.test.ts @@ -26,6 +26,7 @@ describe("completionRegistry", () => { expect(values).toContain("/zai-plan"); expect(values).toContain("/web-search"); expect(values).toContain("/reasoning"); + expect(values).toContain("/init"); }); it("every item has a non-empty value and description", () => { @@ -114,4 +115,16 @@ describe("completionRegistry", () => { expect(helpCmd).toBeDefined(); expect(helpCmd!.children).toBeUndefined(); }); + + it("/init has expected subcommands", () => { + const initCmd = COMMAND_COMPLETIONS.find((c) => c.value === "/init"); + expect(initCmd).toBeDefined(); + expect(initCmd!.children?.map((child) => child.value)).toEqual([ + "status", + "standard", + "migrate", + "open", + "done", + ]); + }); }); diff --git a/openplanter-desktop/frontend/src/commands/completionRegistry.ts b/openplanter-desktop/frontend/src/commands/completionRegistry.ts index 973dc00e..e7ae9ab8 100644 --- a/openplanter-desktop/frontend/src/commands/completionRegistry.ts +++ b/openplanter-desktop/frontend/src/commands/completionRegistry.ts @@ -78,4 +78,15 @@ export const COMMAND_COMPLETIONS: CompletionItem[] = [ description: "Set reasoning effort", children: REASONING_LEVELS, }, + { + value: "/init", + description: "Workspace initialization and migration", + children: [ + { value: "status", description: "Show init status" }, + { value: "standard", description: "Initialize the current workspace" }, + { value: "migrate", description: "Open the migration init panel" }, + { value: "open", description: "Open the init panel" }, + { value: "done", description: "Mark the first-run gate complete" }, + ], + }, ]; diff --git a/openplanter-desktop/frontend/src/commands/init.ts b/openplanter-desktop/frontend/src/commands/init.ts new file mode 100644 index 00000000..44bff941 --- /dev/null +++ b/openplanter-desktop/frontend/src/commands/init.ts @@ -0,0 +1,133 @@ +import { + completeFirstRunGate, + getInitStatus, + runStandardInit, +} from "../api/invoke"; +import type { InitStatusView } from "../api/types"; +import { appState } from "../state/store"; +import type { CommandResult } from "./model"; + +function statusLines(status: InitStatusView): string[] { + return [ + `Workspace: ${status.runtime_workspace}`, + `Gate: ${status.gate_state}`, + `Initialized: ${status.onboarding_completed ? "yes" : "no"}`, + `Wiki root: ${status.has_runtime_wiki ? "yes" : "no"}`, + `Wiki index: ${status.has_runtime_index ? "yes" : "no"}`, + `Last migration target: ${status.last_migration_target || "—"}`, + ...status.warnings.map((warning) => `Warning: ${warning}`), + ]; +} + +export async function handleInitCommand(args: string): Promise { + const parts = args.trim().split(/\s+/).filter(Boolean); + const subcommand = (parts[0] || "status").toLowerCase(); + + if (appState.get().isInitBusy) { + return { + action: "handled", + lines: ["Initialization is already running. Wait for it to finish first."], + }; + } + + if (subcommand === "status") { + const status = await getInitStatus(); + appState.update((s) => ({ + ...s, + initStatus: status, + initGateState: status.gate_state, + initGateVisible: status.gate_state !== "ready" ? true : s.initGateVisible, + })); + return { action: "handled", lines: statusLines(status) }; + } + + if (subcommand === "standard") { + try { + appState.update((s) => ({ ...s, isInitBusy: true, migrationResult: null })); + const report = await runStandardInit(); + const status = await getInitStatus(); + appState.update((s) => ({ + ...s, + isInitBusy: false, + initStatus: status, + initGateState: status.gate_state, + initGateVisible: status.gate_state !== "ready" ? true : false, + initGateMode: "standard", + migrationProgress: null, + })); + if (typeof window !== "undefined") { + window.dispatchEvent(new CustomEvent("curator-done")); + } + return { + action: "handled", + lines: [ + `Standard init completed for ${report.workspace}.`, + `Created paths: ${report.created_paths.length}`, + `Copied files: ${report.copied_paths.length}`, + `Skipped existing: ${report.skipped_existing}`, + ...statusLines(status), + ], + }; + } catch (error) { + appState.update((s) => ({ ...s, isInitBusy: false })); + return { + action: "handled", + lines: [`Standard init failed: ${error}`], + }; + } + } + + if (subcommand === "migrate") { + appState.update((s) => ({ + ...s, + initGateVisible: true, + initGateMode: "migration", + migrationResult: null, + })); + return { + action: "handled", + lines: ["Opened Migration Init. Add a target workspace and one or more sources in the setup panel."], + }; + } + + if (subcommand === "open") { + appState.update((s) => ({ + ...s, + initGateVisible: true, + initGateMode: s.initGateMode, + })); + return { + action: "handled", + lines: ["Opened the workspace initialization panel."], + }; + } + + if (subcommand === "done") { + try { + appState.update((s) => ({ ...s, isInitBusy: true })); + const status = await completeFirstRunGate(); + appState.update((s) => ({ + ...s, + isInitBusy: false, + initStatus: status, + initGateState: status.gate_state, + initGateVisible: status.gate_state !== "ready", + })); + return { action: "handled", lines: statusLines(status) }; + } catch (error) { + appState.update((s) => ({ ...s, isInitBusy: false })); + return { + action: "handled", + lines: [`Failed to complete onboarding: ${error}`], + }; + } + } + + return { + action: "handled", + lines: [ + `Unknown /init subcommand: ${subcommand}`, + "Use /init status, /init standard, or /init migrate.", + ], + }; +} diff --git a/openplanter-desktop/frontend/src/commands/slash.test.ts b/openplanter-desktop/frontend/src/commands/slash.test.ts index e95062f4..d5db6e44 100644 --- a/openplanter-desktop/frontend/src/commands/slash.test.ts +++ b/openplanter-desktop/frontend/src/commands/slash.test.ts @@ -21,6 +21,7 @@ describe("dispatchSlashCommand", () => { webSearchProvider: "exa", sessionId: "20260101-120000-deadbeef", reasoningEffort: "medium", + initGateState: "ready", }); }); @@ -166,4 +167,46 @@ describe("dispatchSlashCommand", () => { (globalThis as any).window = origWindow; }); + + it("/init status dispatches", async () => { + __setHandler("get_init_status", () => ({ + runtime_workspace: "/tmp/ws", + gate_state: "requires_action", + onboarding_completed: false, + has_openplanter_root: true, + has_runtime_wiki: true, + has_runtime_index: true, + init_state_path: "/tmp/ws/.openplanter/init-state.json", + last_migration_target: null, + warnings: [], + })); + const result = await dispatchSlashCommand("/init status"); + expect(result).not.toBeNull(); + expect(result!.lines.some((l) => l.includes("Gate:"))).toBe(true); + }); + + it("/init standard dispatches", async () => { + __setHandler("run_standard_init", () => ({ + workspace: "/tmp/ws", + created_paths: [], + copied_paths: [], + skipped_existing: 0, + errors: [], + onboarding_required: false, + })); + __setHandler("get_init_status", () => ({ + runtime_workspace: "/tmp/ws", + gate_state: "ready", + onboarding_completed: true, + has_openplanter_root: true, + has_runtime_wiki: true, + has_runtime_index: true, + init_state_path: "/tmp/ws/.openplanter/init-state.json", + last_migration_target: null, + warnings: [], + })); + const result = await dispatchSlashCommand("/init standard"); + expect(result).not.toBeNull(); + expect(result!.lines.some((l) => l.includes("Standard init completed"))).toBe(true); + }); }); diff --git a/openplanter-desktop/frontend/src/commands/slash.ts b/openplanter-desktop/frontend/src/commands/slash.ts index 748b312d..125eeb14 100644 --- a/openplanter-desktop/frontend/src/commands/slash.ts +++ b/openplanter-desktop/frontend/src/commands/slash.ts @@ -5,6 +5,7 @@ import { handleModelCommand, type CommandResult } from "./model"; import { handleReasoningCommand } from "./reasoning"; import { handleWebSearchCommand } from "./webSearch"; import { handleZaiPlanCommand } from "./zaiPlan"; +import { handleInitCommand } from "./init"; /** Dispatch a slash command. Returns null if not a slash command. */ export async function dispatchSlashCommand(input: string): Promise { @@ -38,6 +39,9 @@ export async function dispatchSlashCommand(input: string): Promise --save Set and persist", " /reasoning Show/set reasoning effort", " /reasoning Set level (low, medium, high, off)", + " /init status Show workspace init status", + " /init standard Initialize the current workspace", + " /init migrate Open the migration init panel", ], }; @@ -110,6 +114,9 @@ export async function dispatchSlashCommand(input: string): Promise { beforeEach(() => { uuidCounter = 0; - appState.set({ ...originalState, messages: [], sessionId: null }); + appState.set({ + ...originalState, + messages: [], + sessionId: null, + initGateVisible: false, + initGateState: "ready", + initStatus: null, + isInitBusy: false, + migrationProgress: null, + migrationResult: null, + }); __setHandler("list_sessions", () => [SESSION_B, SESSION_A]); __setHandler("get_credentials_status", () => ({ openai: true, anthropic: true, openrouter: false, @@ -139,6 +149,34 @@ describe("createApp", () => { expect(items[0].textContent).toBe("No sessions yet"); }); }); + + it("renders workspace init gate when requested", async () => { + appState.update((s) => ({ + ...s, + initGateVisible: true, + initGateState: "requires_action", + initStatus: { + runtime_workspace: "/tmp/ws", + gate_state: "requires_action", + onboarding_completed: false, + has_openplanter_root: true, + has_runtime_wiki: true, + has_runtime_index: true, + init_state_path: "/tmp/ws/.openplanter/init-state.json", + last_migration_target: null, + warnings: [], + }, + })); + const root = document.createElement("div"); + document.body.appendChild(root); + createApp(root); + + await vi.waitFor(() => { + const gate = root.querySelector(".workspace-init-gate") as HTMLElement; + expect(gate).not.toBeNull(); + expect(gate.style.display).toBe("flex"); + }); + }); }); describe("session delete confirmation flow", () => { diff --git a/openplanter-desktop/frontend/src/components/App.ts b/openplanter-desktop/frontend/src/components/App.ts index 715c0f38..4ef96820 100644 --- a/openplanter-desktop/frontend/src/components/App.ts +++ b/openplanter-desktop/frontend/src/components/App.ts @@ -2,6 +2,7 @@ import { createStatusBar } from "./StatusBar"; import { createChatPane } from "./ChatPane"; import { createGraphPane } from "./GraphPane"; +import { createWorkspaceInitGate } from "./WorkspaceInitGate"; import { appState } from "../state/store"; import { listSessions, openSession, deleteSession, getCredentialsStatus, getSessionHistory } from "../api/invoke"; import type { ChatMessage } from "../state/store"; @@ -61,6 +62,10 @@ export function createApp(root: HTMLElement): void { const graphPane = createGraphPane(); root.appendChild(graphPane); + // Workspace init gate + const workspaceInitGate = createWorkspaceInitGate(); + root.appendChild(workspaceInitGate); + // Reactive settings display function renderSettings() { const s = appState.get(); diff --git a/openplanter-desktop/frontend/src/components/InputBar.test.ts b/openplanter-desktop/frontend/src/components/InputBar.test.ts index cf5277a5..1bea6eb1 100644 --- a/openplanter-desktop/frontend/src/components/InputBar.test.ts +++ b/openplanter-desktop/frontend/src/components/InputBar.test.ts @@ -21,7 +21,13 @@ describe("createInputBar", () => { beforeEach(() => { uuidCounter = 0; - appState.set({ ...originalState, messages: [], inputHistory: [], inputQueue: [] }); + appState.set({ + ...originalState, + messages: [], + inputHistory: [], + inputQueue: [], + initGateState: "ready", + }); // Default handlers to prevent unhandled rejection __setHandler("solve", () => {}); __setHandler("cancel", () => {}); @@ -386,4 +392,43 @@ describe("createInputBar", () => { document.body.removeChild(bar); }); + + it("blocks normal objective submit until init is ready", async () => { + appState.update((s) => ({ ...s, initGateState: "requires_action" })); + const bar = createInputBar(); + document.body.appendChild(bar); + const textarea = bar.querySelector("textarea")!; + + textarea.value = "blocked objective"; + bar.querySelectorAll("button")[0].click(); + + await vi.waitFor(() => { + expect(appState.get().isRunning).toBe(false); + expect( + appState.get().messages.some((m) => + m.content.includes("Workspace initialization is required") + ) + ).toBe(true); + }); + + document.body.removeChild(bar); + }); + + it("blocks non-init slash commands until init is ready", async () => { + appState.update((s) => ({ ...s, initGateState: "requires_action" })); + const bar = createInputBar(); + document.body.appendChild(bar); + const textarea = bar.querySelector("textarea")!; + + textarea.value = "/status"; + bar.querySelectorAll("button")[0].click(); + + await vi.waitFor(() => { + expect( + appState.get().messages.some((m) => m.content.includes("Use /init first")) + ).toBe(true); + }); + + document.body.removeChild(bar); + }); }); diff --git a/openplanter-desktop/frontend/src/components/InputBar.ts b/openplanter-desktop/frontend/src/components/InputBar.ts index cad43240..4575bb27 100644 --- a/openplanter-desktop/frontend/src/components/InputBar.ts +++ b/openplanter-desktop/frontend/src/components/InputBar.ts @@ -55,6 +55,15 @@ export function createInputBar(): HTMLElement { // Check for slash commands if (text.startsWith("/")) { + const initRequired = appState.get().initGateState !== "ready"; + const lower = text.toLowerCase(); + if (initRequired && !lower.startsWith("/init") && !lower.startsWith("/help")) { + textarea.value = ""; + autoResize(); + addSystemMessage("Workspace initialization is required. Use /init first."); + return; + } + textarea.value = ""; autoResize(); @@ -80,6 +89,13 @@ export function createInputBar(): HTMLElement { return; } + if (appState.get().initGateState !== "ready") { + addSystemMessage( + "Workspace initialization is required before starting an objective. Use /init." + ); + return; + } + // If running, queue the input instead of blocking if (appState.get().isRunning) { appState.update((s) => ({ @@ -259,7 +275,9 @@ export function createInputBar(): HTMLElement { cancelBtn.style.display = running ? "" : "none"; textarea.placeholder = running ? "Type to queue..." - : "Enter objective or /command..."; + : appState.get().initGateState !== "ready" + ? "Complete workspace init or use /init..." + : "Enter objective or /command..."; // Keep textarea enabled during execution for queuing submitBtn.disabled = false; }); diff --git a/openplanter-desktop/frontend/src/components/WorkspaceInitGate.ts b/openplanter-desktop/frontend/src/components/WorkspaceInitGate.ts new file mode 100644 index 00000000..5baac0fd --- /dev/null +++ b/openplanter-desktop/frontend/src/components/WorkspaceInitGate.ts @@ -0,0 +1,402 @@ +import { + getInitStatus, + inspectMigrationSource, + runMigrationInit, + runStandardInit, +} from "../api/invoke"; +import type { MigrationSourceInspection } from "../api/types"; +import { appState } from "../state/store"; + +interface SourceDraft { + path: string; + inspection: MigrationSourceInspection | null; +} + +export function createWorkspaceInitGate(): HTMLElement { + const overlay = document.createElement("div"); + overlay.className = "workspace-init-gate"; + overlay.style.position = "fixed"; + overlay.style.inset = "0"; + overlay.style.display = "none"; + overlay.style.alignItems = "center"; + overlay.style.justifyContent = "center"; + overlay.style.background = "rgba(6, 10, 14, 0.78)"; + overlay.style.zIndex = "999"; + + const panel = document.createElement("div"); + panel.className = "workspace-init-panel"; + panel.style.width = "min(760px, 92vw)"; + panel.style.maxHeight = "88vh"; + panel.style.overflow = "auto"; + panel.style.padding = "20px"; + panel.style.borderRadius = "16px"; + panel.style.background = "var(--bg-secondary)"; + panel.style.border = "1px solid var(--border)"; + panel.style.boxShadow = "0 24px 80px rgba(0, 0, 0, 0.35)"; + overlay.appendChild(panel); + + let targetWorkspace = ""; + let sources: SourceDraft[] = [{ path: "", inspection: null }]; + let localError = ""; + + function ensureDefaultTarget(): void { + const workspace = appState.get().workspace; + if (!targetWorkspace && workspace) { + targetWorkspace = `${workspace}-desktop`; + } + } + + async function refreshStatus(): Promise { + const status = await getInitStatus(); + appState.update((s) => ({ + ...s, + initStatus: status, + initGateState: status.gate_state, + initGateVisible: status.gate_state !== "ready" ? true : s.initGateVisible, + })); + } + + function visibilityState(): boolean { + const state = appState.get(); + return state.initGateVisible || state.initGateState !== "ready"; + } + + function updateBusy(isInitBusy: boolean): void { + appState.update((s) => ({ ...s, isInitBusy })); + } + + async function handleStandardInit(): Promise { + localError = ""; + updateBusy(true); + try { + await runStandardInit(); + await refreshStatus(); + appState.update((s) => ({ + ...s, + initGateVisible: false, + initGateMode: "standard", + migrationProgress: null, + migrationResult: null, + })); + window.dispatchEvent(new CustomEvent("curator-done")); + } catch (error) { + localError = `Standard init failed: ${error}`; + } finally { + updateBusy(false); + render(); + } + } + + async function handleInspect(index: number): Promise { + const draft = sources[index]; + if (!draft || !draft.path.trim()) { + localError = "Enter a source path before inspecting it."; + render(); + return; + } + localError = ""; + updateBusy(true); + try { + const inspection = await inspectMigrationSource(draft.path.trim()); + sources[index] = { ...draft, inspection }; + } catch (error) { + localError = `Inspection failed: ${error}`; + } finally { + updateBusy(false); + render(); + } + } + + async function handleMigration(): Promise { + const trimmedTarget = targetWorkspace.trim(); + const trimmedSources = sources + .map((source) => source.path.trim()) + .filter(Boolean); + if (!trimmedTarget) { + localError = "Enter a target workspace path."; + render(); + return; + } + if (trimmedSources.length === 0) { + localError = "Add at least one source workspace or research directory."; + render(); + return; + } + + localError = ""; + appState.update((s) => ({ + ...s, + isInitBusy: true, + migrationProgress: null, + migrationResult: null, + initGateMode: "migration", + initGateVisible: true, + })); + try { + const result = await runMigrationInit({ + target_workspace: trimmedTarget, + sources: trimmedSources.map((path) => ({ path })), + }); + appState.update((s) => ({ + ...s, + isInitBusy: false, + migrationResult: result, + initGateVisible: true, + })); + } catch (error) { + localError = `Migration failed: ${error}`; + updateBusy(false); + } finally { + render(); + } + } + + function renderSourceRow(index: number, stateBusy: boolean): HTMLElement { + const draft = sources[index]; + const row = document.createElement("div"); + row.style.display = "grid"; + row.style.gridTemplateColumns = "1fr auto auto"; + row.style.gap = "8px"; + row.style.marginBottom = "10px"; + + const input = document.createElement("input"); + input.type = "text"; + input.value = draft.path; + input.placeholder = "/path/to/openplanter-workspace-or-research-dir"; + input.disabled = stateBusy; + input.addEventListener("input", () => { + sources[index] = { path: input.value, inspection: null }; + }); + + const inspectBtn = document.createElement("button"); + inspectBtn.textContent = "Inspect"; + inspectBtn.disabled = stateBusy; + inspectBtn.addEventListener("click", () => { + void handleInspect(index); + }); + + const removeBtn = document.createElement("button"); + removeBtn.textContent = "Remove"; + removeBtn.disabled = stateBusy || sources.length === 1; + removeBtn.addEventListener("click", () => { + sources.splice(index, 1); + render(); + }); + + row.appendChild(input); + row.appendChild(inspectBtn); + row.appendChild(removeBtn); + + if (draft.inspection) { + const details = document.createElement("div"); + details.style.gridColumn = "1 / -1"; + details.style.padding = "8px 10px"; + details.style.border = "1px solid var(--border)"; + details.style.borderRadius = "10px"; + details.style.background = "var(--bg-tertiary)"; + details.textContent = [ + `kind=${draft.inspection.kind}`, + `markdown=${draft.inspection.markdown_files}`, + `sessions=${draft.inspection.has_sessions ? "yes" : "no"}`, + `settings=${draft.inspection.has_settings ? "yes" : "no"}`, + `credentials=${draft.inspection.has_credentials ? "yes" : "no"}`, + `runtime_wiki=${draft.inspection.has_runtime_wiki ? "yes" : "no"}`, + ].join(" | "); + row.appendChild(details); + } + + return row; + } + + function render(): void { + ensureDefaultTarget(); + const state = appState.get(); + const visible = visibilityState(); + overlay.style.display = visible ? "flex" : "none"; + if (!visible) { + return; + } + + panel.replaceChildren(); + + const title = document.createElement("h2"); + title.textContent = "Workspace Initialization"; + panel.appendChild(title); + + const intro = document.createElement("p"); + intro.textContent = + state.initGateState !== "ready" + ? "Choose Standard Init to prepare the current workspace, or Migration Init to build a new Desktop workspace from one or more existing sources." + : "Manage the current workspace or open a migration flow to build a new Desktop workspace."; + panel.appendChild(intro); + + const modeBar = document.createElement("div"); + modeBar.style.display = "flex"; + modeBar.style.gap = "8px"; + modeBar.style.marginBottom = "14px"; + + const standardTab = document.createElement("button"); + standardTab.textContent = "Standard Init"; + standardTab.disabled = state.isInitBusy; + standardTab.style.fontWeight = state.initGateMode === "standard" ? "700" : "400"; + standardTab.addEventListener("click", () => { + appState.update((s) => ({ ...s, initGateMode: "standard", migrationResult: null })); + }); + + const migrationTab = document.createElement("button"); + migrationTab.textContent = "Migration Init"; + migrationTab.disabled = state.isInitBusy; + migrationTab.style.fontWeight = state.initGateMode === "migration" ? "700" : "400"; + migrationTab.addEventListener("click", () => { + appState.update((s) => ({ ...s, initGateMode: "migration" })); + }); + + modeBar.appendChild(standardTab); + modeBar.appendChild(migrationTab); + panel.appendChild(modeBar); + + if (state.initStatus) { + const status = document.createElement("div"); + status.style.padding = "10px 12px"; + status.style.border = "1px solid var(--border)"; + status.style.borderRadius = "12px"; + status.style.background = "var(--bg-tertiary)"; + status.style.marginBottom = "14px"; + status.textContent = [ + `workspace=${state.initStatus.runtime_workspace}`, + `gate=${state.initStatus.gate_state}`, + `wiki=${state.initStatus.has_runtime_index ? "ready" : "missing"}`, + `last_migration=${state.initStatus.last_migration_target || "—"}`, + ].join(" | "); + panel.appendChild(status); + } + + if (state.migrationProgress) { + const progress = document.createElement("div"); + progress.style.padding = "10px 12px"; + progress.style.border = "1px solid var(--border)"; + progress.style.borderRadius = "12px"; + progress.style.background = "rgba(57, 148, 255, 0.08)"; + progress.style.marginBottom = "14px"; + progress.textContent = `[${state.migrationProgress.stage}] ${state.migrationProgress.message}`; + panel.appendChild(progress); + } + + if (state.migrationResult) { + const result = document.createElement("div"); + result.style.padding = "12px"; + result.style.border = "1px solid var(--border)"; + result.style.borderRadius = "12px"; + result.style.background = "rgba(56, 184, 90, 0.10)"; + result.style.marginBottom = "14px"; + result.textContent = [ + `Target: ${state.migrationResult.target_workspace}`, + `Sessions copied: ${state.migrationResult.sessions_copied}`, + `Sessions renamed: ${state.migrationResult.sessions_renamed}`, + `Wiki pages available: ${state.migrationResult.wiki_files_synthesized}`, + `Curator summary: ${state.migrationResult.rewrite_summary}`, + state.migrationResult.restart_message, + ].join("\n"); + panel.appendChild(result); + } + + if (localError) { + const error = document.createElement("div"); + error.style.padding = "10px 12px"; + error.style.border = "1px solid rgba(255, 99, 99, 0.45)"; + error.style.borderRadius = "12px"; + error.style.background = "rgba(255, 99, 99, 0.10)"; + error.style.marginBottom = "14px"; + error.textContent = localError; + panel.appendChild(error); + } + + if (state.initGateMode === "standard") { + const block = document.createElement("div"); + const body = document.createElement("p"); + body.textContent = + "Standard Init prepares the current workspace, creates the runtime wiki skeleton, and marks the Desktop onboarding flow complete."; + const button = document.createElement("button"); + button.textContent = state.isInitBusy ? "Initializing..." : "Initialize Current Workspace"; + button.disabled = state.isInitBusy; + button.addEventListener("click", () => { + void handleStandardInit(); + }); + block.appendChild(body); + block.appendChild(button); + panel.appendChild(block); + } else { + const migration = document.createElement("div"); + + const targetLabel = document.createElement("label"); + targetLabel.textContent = "Target Workspace"; + targetLabel.style.display = "block"; + targetLabel.style.marginBottom = "6px"; + migration.appendChild(targetLabel); + + const targetInput = document.createElement("input"); + targetInput.type = "text"; + targetInput.value = targetWorkspace; + targetInput.placeholder = "/path/to/new-desktop-workspace"; + targetInput.style.width = "100%"; + targetInput.style.marginBottom = "14px"; + targetInput.disabled = state.isInitBusy; + targetInput.addEventListener("input", () => { + targetWorkspace = targetInput.value; + }); + migration.appendChild(targetInput); + + const sourcesHeader = document.createElement("div"); + sourcesHeader.textContent = "Migration Sources"; + sourcesHeader.style.fontWeight = "700"; + sourcesHeader.style.marginBottom = "8px"; + migration.appendChild(sourcesHeader); + + const sourceList = document.createElement("div"); + for (let index = 0; index < sources.length; index += 1) { + sourceList.appendChild(renderSourceRow(index, state.isInitBusy)); + } + migration.appendChild(sourceList); + + const actions = document.createElement("div"); + actions.style.display = "flex"; + actions.style.gap = "8px"; + actions.style.marginTop = "12px"; + + const addBtn = document.createElement("button"); + addBtn.textContent = "Add Source"; + addBtn.disabled = state.isInitBusy; + addBtn.addEventListener("click", () => { + sources.push({ path: "", inspection: null }); + render(); + }); + + const migrateBtn = document.createElement("button"); + migrateBtn.textContent = state.isInitBusy ? "Migrating..." : "Run Migration Init"; + migrateBtn.disabled = state.isInitBusy; + migrateBtn.addEventListener("click", () => { + void handleMigration(); + }); + + actions.appendChild(addBtn); + actions.appendChild(migrateBtn); + migration.appendChild(actions); + panel.appendChild(migration); + } + + if (state.initGateState === "ready") { + const closeBtn = document.createElement("button"); + closeBtn.textContent = "Close"; + closeBtn.style.marginTop = "16px"; + closeBtn.disabled = state.isInitBusy; + closeBtn.addEventListener("click", () => { + appState.update((s) => ({ ...s, initGateVisible: false })); + }); + panel.appendChild(closeBtn); + } + } + + appState.subscribe(render); + render(); + return overlay; +} diff --git a/openplanter-desktop/frontend/src/main.ts b/openplanter-desktop/frontend/src/main.ts index ad9ac303..bb9696a8 100644 --- a/openplanter-desktop/frontend/src/main.ts +++ b/openplanter-desktop/frontend/src/main.ts @@ -1,5 +1,5 @@ import { createApp } from "./components/App"; -import { getConfig } from "./api/invoke"; +import { getConfig, getInitStatus } from "./api/invoke"; import { onAgentTrace, onAgentDelta, @@ -8,6 +8,7 @@ import { onAgentStep, onWikiUpdated, onCuratorUpdate, + onMigrationProgress, } from "./api/events"; import { appState } from "./state/store"; @@ -31,6 +32,7 @@ async function init() { const config = await getConfig(); provider = config.provider; model = config.model; + const initStatus = await getInitStatus(); appState.update((s) => ({ ...s, provider: config.provider, @@ -43,6 +45,9 @@ async function init() { workspace: config.workspace, maxDepth: config.max_depth, maxStepsPerCall: config.max_steps_per_call, + initStatus, + initGateState: initStatus.gate_state, + initGateVisible: initStatus.gate_state !== "ready", })); } catch (e) { console.error("Failed to load config:", e); @@ -82,6 +87,17 @@ async function init() { content: "Type /help for commands. ESC to cancel a running task.", timestamp: Date.now(), }, + ...(state.initGateState !== "ready" + ? [ + { + id: crypto.randomUUID(), + role: "system" as const, + content: + "Workspace initialization is required before running the agent. Use the setup panel or /init.", + timestamp: Date.now(), + }, + ] + : []), ], })); @@ -175,6 +191,14 @@ async function init() { // Notify graph pane to refresh with curator's wiki changes window.dispatchEvent(new CustomEvent("curator-done")); }); + + await onMigrationProgress((event) => { + appState.update((s) => ({ + ...s, + migrationProgress: event, + isInitBusy: event.stage !== "done", + })); + }); } function processQueue() { diff --git a/openplanter-desktop/frontend/src/state/store.test.ts b/openplanter-desktop/frontend/src/state/store.test.ts index 7796926e..41420430 100644 --- a/openplanter-desktop/frontend/src/state/store.test.ts +++ b/openplanter-desktop/frontend/src/state/store.test.ts @@ -71,6 +71,9 @@ describe("appState", () => { expect(state.maxDepth).toBe(4); expect(state.maxStepsPerCall).toBe(100); expect(state.inputQueue).toEqual([]); + expect(state.initGateState).toBe("ready"); + expect(state.isInitBusy).toBe(false); + expect(state.initGateVisible).toBe(false); }); it("message append via update", () => { diff --git a/openplanter-desktop/frontend/src/state/store.ts b/openplanter-desktop/frontend/src/state/store.ts index eafa4c8a..27d1a382 100644 --- a/openplanter-desktop/frontend/src/state/store.ts +++ b/openplanter-desktop/frontend/src/state/store.ts @@ -1,4 +1,10 @@ /** Simple observable state store. */ +import type { + InitStatusView, + MigrationInitResultView, + MigrationProgressEvent, +} from "../api/types"; + type Listener = (value: T) => void; export class Store { @@ -77,6 +83,13 @@ export interface AppState { currentDepth: number; inputHistory: string[]; inputQueue: string[]; + initGateState: "ready" | "requires_action" | "blocked"; + initStatus: InitStatusView | null; + isInitBusy: boolean; + initGateVisible: boolean; + initGateMode: "standard" | "migration"; + migrationProgress: MigrationProgressEvent | null; + migrationResult: MigrationInitResultView | null; } export const appState = new Store({ @@ -98,4 +111,11 @@ export const appState = new Store({ currentDepth: 0, inputHistory: [], inputQueue: [], + initGateState: "ready", + initStatus: null, + isInitBusy: false, + initGateVisible: false, + initGateMode: "standard", + migrationProgress: null, + migrationResult: null, }); From fd08764530cadd1629543e42cb4a7a4e28f00df1 Mon Sep 17 00:00:00 2001 From: Drake Date: Thu, 12 Mar 2026 17:14:24 -0400 Subject: [PATCH 10/58] Remove investigation artifacts from repository --- .gitignore | 40 ++++++++++++++++++++++++++++++++++++---- LICENSE | 21 --------------------- 2 files changed, 36 insertions(+), 25 deletions(-) delete mode 100644 LICENSE diff --git a/.gitignore b/.gitignore index 6c4fc33f..e5a51452 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,43 @@ +# Local environment and workspace state .env .env.* -node_modules/ +.venv/ +.python-version +.direnv/ +.openplanter/ +/workspace/ + +# Python caches and build artifacts __pycache__/ -*.pyc +*.py[cod] *.egg-info/ -dist/ +.pytest_cache/ +.coverage +.coverage.* +htmlcov/ +.mypy_cache/ +.ruff_cache/ +.hypothesis/ +.tox/ +.nox/ build/ +dist/ +pip-wheel-metadata/ + +# Frontend and test artifacts +node_modules/ +coverage/ +playwright-report/ +test-results/ + +# Rust / Tauri build output +target/ + +# Generated captures *.cast *.mp4 -.openplanter/ + +# Editor and OS cruft +.DS_Store +.idea/ +.vscode/ diff --git a/LICENSE b/LICENSE deleted file mode 100644 index e8b35c70..00000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 OpenPlanter Contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. From 4e33f5ab5139ed3730c53cfdc10c30a5af730970 Mon Sep 17 00:00:00 2001 From: Drake Date: Fri, 13 Mar 2026 09:04:19 -0400 Subject: [PATCH 11/58] fix: preserve replay sequence across resumed sessions --- agent/engine.py | 2 +- agent/replay_log.py | 65 ++++++++- .../crates/op-core/src/session/replay.rs | 137 ++++++++++++++++++ tests/test_replay_log.py | 41 +++++- 4 files changed, 242 insertions(+), 3 deletions(-) diff --git a/agent/engine.py b/agent/engine.py index 422dbf99..b762c621 100644 --- a/agent/engine.py +++ b/agent/engine.py @@ -355,7 +355,7 @@ def _solve_recursive( conversation = model.create_conversation(self.system_prompt, initial_message) - if replay_logger and replay_logger._seq == 0: + if replay_logger and replay_logger.needs_header: replay_logger.write_header( provider=type(model).__name__, model=getattr(model, "model", "(unknown)"), diff --git a/agent/replay_log.py b/agent/replay_log.py index 96a399a7..466b53af 100644 --- a/agent/replay_log.py +++ b/agent/replay_log.py @@ -25,6 +25,16 @@ class ReplayLogger: conversation_id: str = "root" _seq: int = field(default=0, init=False) _last_msg_count: int = field(default=0, init=False) + _has_call: bool = field(default=False, init=False) + _has_header: bool = field(default=False, init=False) + + def __post_init__(self) -> None: + self._seq = self._scan_next_seq() + self._hydrate_conversation_state() + + @property + def needs_header(self) -> bool: + return not self._has_header def child(self, depth: int, step: int) -> "ReplayLogger": """Create a child logger for a subtask conversation.""" @@ -56,6 +66,7 @@ def write_header( if temperature is not None: record["temperature"] = temperature self._append(record) + self._has_header = True def log_call( self, @@ -68,6 +79,7 @@ def log_call( output_tokens: int = 0, elapsed_sec: float = 0.0, ) -> None: + self._seq = max(self._seq, self._scan_next_seq()) record: dict[str, Any] = { "type": "call", "conversation_id": self.conversation_id, @@ -76,7 +88,7 @@ def log_call( "step": step, "ts": datetime.now(timezone.utc).isoformat(), } - if self._seq == 0: + if not self._has_call: record["messages_snapshot"] = messages else: record["messages_delta"] = messages[self._last_msg_count:] @@ -86,9 +98,60 @@ def log_call( record["elapsed_sec"] = round(elapsed_sec, 3) self._last_msg_count = len(messages) + self._has_call = True self._seq += 1 self._append(record) + def _scan_next_seq(self) -> int: + if not self.path.exists(): + return 0 + next_seq = 0 + for raw_line in self.path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + seq = record.get("seq") + if isinstance(seq, int) and seq >= next_seq: + next_seq = seq + 1 + return next_seq + + def _hydrate_conversation_state(self) -> None: + if not self.path.exists(): + return + msg_count = 0 + has_call = False + has_header = False + for raw_line in self.path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if record.get("conversation_id") != self.conversation_id: + continue + if record.get("type") == "header": + has_header = True + continue + if record.get("type") != "call": + continue + has_call = True + snapshot = record.get("messages_snapshot") + if isinstance(snapshot, list): + msg_count = len(snapshot) + continue + delta = record.get("messages_delta") + if isinstance(delta, list): + msg_count += len(delta) + self._has_call = has_call + self._has_header = has_header + self._last_msg_count = msg_count + def _append(self, record: dict[str, Any]) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) with self.path.open("a", encoding="utf-8") as fh: diff --git a/openplanter-desktop/crates/op-core/src/session/replay.rs b/openplanter-desktop/crates/op-core/src/session/replay.rs index d347874a..df6895aa 100644 --- a/openplanter-desktop/crates/op-core/src/session/replay.rs +++ b/openplanter-desktop/crates/op-core/src/session/replay.rs @@ -58,6 +58,7 @@ impl ReplayLogger { /// Append an entry to the replay log. pub async fn append(&mut self, mut entry: ReplayEntry) -> std::io::Result<()> { + self.seq = self.seq.max(Self::max_seq_from_file(&self.path).await?); self.seq += 1; entry.seq = self.seq; if entry.timestamp.is_empty() { @@ -77,6 +78,29 @@ impl ReplayLogger { Ok(()) } + async fn max_seq_from_file(path: &Path) -> std::io::Result { + if !path.exists() { + return Ok(0); + } + let content = fs::read_to_string(path).await?; + let mut max_seq = 0_u64; + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + match serde_json::from_str::(trimmed) { + Ok(entry) => { + max_seq = max_seq.max(entry.seq); + } + Err(e) => { + eprintln!("[replay] skipping malformed line: {e}"); + } + } + } + Ok(max_seq) + } + /// Read all entries from a session's replay log. pub async fn read_all(session_dir: &Path) -> std::io::Result> { let path = session_dir.join("replay.jsonl"); @@ -292,4 +316,117 @@ mod tests { assert!(!content.contains("step_number")); assert!(!content.contains("step_tool_calls")); } + + #[tokio::test] + async fn test_append_continues_seq_from_existing_file() { + let tmp = tempdir().unwrap(); + let path = tmp.path().join("replay.jsonl"); + let content = format!( + "{}\n{}\n", + serde_json::to_string(&ReplayEntry { + seq: 4, + timestamp: "2026-01-01T00:00:00Z".into(), + role: "user".into(), + content: "first".into(), + tool_name: None, + is_rendered: None, + step_number: None, + step_tokens_in: None, + step_tokens_out: None, + step_elapsed: None, + step_model_preview: None, + step_tool_calls: None, + }) + .unwrap(), + serde_json::to_string(&ReplayEntry { + seq: 6, + timestamp: "2026-01-01T00:01:00Z".into(), + role: "assistant".into(), + content: "second".into(), + tool_name: None, + is_rendered: None, + step_number: None, + step_tokens_in: None, + step_tokens_out: None, + step_elapsed: None, + step_model_preview: None, + step_tool_calls: None, + }) + .unwrap(), + ); + fs::write(&path, content).await.unwrap(); + + let mut logger = ReplayLogger::new(tmp.path()); + logger + .append(ReplayEntry { + seq: 0, + timestamp: String::new(), + role: "user".into(), + content: "third".into(), + tool_name: None, + is_rendered: None, + step_number: None, + step_tokens_in: None, + step_tokens_out: None, + step_elapsed: None, + step_model_preview: None, + step_tool_calls: None, + }) + .await + .unwrap(); + + let entries = ReplayLogger::read_all(tmp.path()).await.unwrap(); + assert_eq!(entries.last().unwrap().seq, 7); + } + + #[tokio::test] + async fn test_append_ignores_malformed_lines_when_scanning_seq() { + let tmp = tempdir().unwrap(); + let path = tmp.path().join("replay.jsonl"); + fs::write( + &path, + format!( + "{}\nnot json\n", + serde_json::to_string(&ReplayEntry { + seq: 2, + timestamp: "2026-01-01T00:00:00Z".into(), + role: "user".into(), + content: "first".into(), + tool_name: None, + is_rendered: None, + step_number: None, + step_tokens_in: None, + step_tokens_out: None, + step_elapsed: None, + step_model_preview: None, + step_tool_calls: None, + }) + .unwrap() + ), + ) + .await + .unwrap(); + + let mut logger = ReplayLogger::new(tmp.path()); + logger + .append(ReplayEntry { + seq: 0, + timestamp: String::new(), + role: "assistant".into(), + content: "next".into(), + tool_name: None, + is_rendered: None, + step_number: None, + step_tokens_in: None, + step_tokens_out: None, + step_elapsed: None, + step_model_preview: None, + step_tool_calls: None, + }) + .await + .unwrap(); + + let entries = ReplayLogger::read_all(tmp.path()).await.unwrap(); + assert_eq!(entries.last().unwrap().seq, 3); + } } diff --git a/tests/test_replay_log.py b/tests/test_replay_log.py index ff31e7a9..97d6c837 100644 --- a/tests/test_replay_log.py +++ b/tests/test_replay_log.py @@ -174,7 +174,7 @@ def test_child_logger(self) -> None: self.assertEqual(records[2]["conversation_id"], "root/d0s2") self.assertEqual(records[2]["model"], "m-child") self.assertEqual(records[3]["conversation_id"], "root/d0s2") - self.assertEqual(records[3]["seq"], 0) + self.assertEqual(records[3]["seq"], 1) self.assertIn("messages_snapshot", records[3]) def test_creates_parent_dirs(self) -> None: @@ -187,6 +187,45 @@ def test_creates_parent_dirs(self) -> None: ) self.assertTrue(p.exists()) + def test_initializes_seq_from_existing_file(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + p = Path(tmpdir) / "replay.jsonl" + p.write_text( + "\n".join([ + json.dumps({"type": "header", "conversation_id": "root"}), + json.dumps({"type": "call", "conversation_id": "root", "seq": 3, "messages_snapshot": [{"role": "user", "content": "hi"}]}), + "{malformed", + json.dumps({"type": "call", "conversation_id": "other", "seq": 8, "messages_snapshot": [{"role": "user", "content": "x"}]}), + ]) + + "\n", + encoding="utf-8", + ) + + logger = ReplayLogger(path=p) + logger.log_call( + depth=0, + step=2, + messages=[ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ], + response={"r": 1}, + ) + + records = [] + for line in p.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + continue + calls = [r for r in records if r.get("type") == "call" and r.get("conversation_id") == "root"] + self.assertEqual(calls[-1]["seq"], 9) + self.assertIn("messages_delta", calls[-1]) + self.assertEqual(calls[-1]["messages_delta"], [{"role": "assistant", "content": "hello"}]) + class ReplayLoggerIntegrationTests(unittest.TestCase): def _read_records(self, path: Path) -> list[dict]: From fe390d4e7fded1d029e8170c796172fcb4144e95 Mon Sep 17 00:00:00 2001 From: Drake Date: Fri, 13 Mar 2026 09:10:51 -0400 Subject: [PATCH 12/58] feat: add runtime loop guardrails and metrics --- agent/engine.py | 102 +++++++++++++++++++++++++++++++++++ agent/runtime.py | 46 +++++++++++++++- tests/test_engine.py | 35 ++++++++++++ tests/test_turn_summaries.py | 69 ++++++++++++++++++++++++ 4 files changed, 251 insertions(+), 1 deletion(-) diff --git a/agent/engine.py b/agent/engine.py index b762c621..434503ea 100644 --- a/agent/engine.py +++ b/agent/engine.py @@ -24,6 +24,30 @@ ContentDeltaCallback = Callable[[str, str], None] +_RECON_TOOL_NAMES = { + "list_files", + "search_files", + "repo_map", + "web_search", + "fetch_url", + "read_file", + "read_image", + "list_artifacts", + "read_artifact", +} +_ARTIFACT_TOOL_NAMES = { + "write_file", + "apply_patch", + "edit_file", + "hashline_edit", +} +_META_FINAL_PATTERNS = ( + re.compile(r"^\s*(here(?:'s| is)\s+(?:my|the)\s+(?:plan|approach|analysis))\b", re.I), + re.compile(r"\b(i\s+(?:will|can|should|need to|want to|am going to|plan to))\b", re.I), + re.compile(r"\b(let me|next,?\s+i\s+will|i\s+should\s+start\s+by)\b", re.I), +) + + def _summarize_args(args: dict[str, Any], max_len: int = 120) -> str: """One-line summary of tool call arguments.""" parts: list[str] = [] @@ -169,6 +193,7 @@ class RLMEngine: _shell_command_counts: dict[tuple[int, str], int] = field(default_factory=dict) _cancel: threading.Event = field(default_factory=threading.Event) _pending_image: threading.local = field(default_factory=threading.local) + last_loop_metrics: dict[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: if not self.system_prompt: @@ -300,6 +325,14 @@ def _judge_result( except Exception as exc: return f"PASS\n(judge error: {exc})" + def _is_meta_final_text(self, text: str) -> bool: + stripped = text.strip() + if not stripped: + return True + if len(stripped.split()) < 5: + return False + return any(pattern.search(stripped) for pattern in _META_FINAL_PATTERNS) + def _solve_recursive( self, objective: str, @@ -355,6 +388,17 @@ def _solve_recursive( conversation = model.create_conversation(self.system_prompt, initial_message) + loop_metrics: dict[str, Any] = { + "steps": 0, + "model_turns": 0, + "tool_calls": 0, + "phase_counts": {"investigate": 0, "build": 0, "iterate": 0, "finalize": 0}, + "recon_streak": 0, + "max_recon_streak": 0, + "guardrail_warnings": 0, + "final_rejections": 0, + } + if replay_logger and replay_logger.needs_header: replay_logger.write_header( provider=type(model).__name__, @@ -420,6 +464,8 @@ def _solve_recursive( if hasattr(model, "on_content_delta"): model.on_content_delta = None elapsed = time.monotonic() - t0 + loop_metrics["steps"] = step + loop_metrics["model_turns"] += 1 if replay_logger: try: @@ -469,6 +515,7 @@ def _solve_recursive( "output_tokens": turn.output_tokens, "elapsed_sec": round(elapsed, 2), "is_final": False, + "phase": "model", } ) except Exception: @@ -476,11 +523,30 @@ def _solve_recursive( # No tool calls + text present = final answer if not turn.tool_calls and turn.text: + if self._is_meta_final_text(turn.text): + loop_metrics["final_rejections"] += 1 + self._emit( + f"[d{depth}/s{step}] rejected meta final-answer text; requesting concrete completion", + on_event, + ) + rejection_result = ToolResult( + tool_call_id="meta-final-reject", + name="system", + content=( + "Final-answer candidate rejected: response is meta/process text. " + "Provide a concrete completion summary (what was produced/changed) " + "instead of describing what you will do next." + ), + ) + model.append_tool_results(conversation, [rejection_result]) + continue + loop_metrics["phase_counts"]["finalize"] += 1 preview = turn.text[:200] + "..." if len(turn.text) > 200 else turn.text self._emit( f"[d{depth}/s{step}] final answer ({len(turn.text)} chars, {elapsed:.1f}s): {preview}", on_event, ) + self.last_loop_metrics = loop_metrics if on_step: try: on_step( @@ -491,6 +557,8 @@ def _solve_recursive( "action": {"name": "final", "arguments": {"text": turn.text}}, "observation": turn.text, "is_final": True, + "phase": "finalize", + "loop_metrics": dict(loop_metrics), } ) except Exception: @@ -510,6 +578,21 @@ def _solve_recursive( # Log tool calls from model tc_names = [tc.name for tc in turn.tool_calls] + loop_metrics["tool_calls"] += len(tc_names) + has_recon = any(name in _RECON_TOOL_NAMES for name in tc_names) + has_artifact = any(name in _ARTIFACT_TOOL_NAMES for name in tc_names) + if has_recon and not has_artifact and all(name in _RECON_TOOL_NAMES for name in tc_names): + loop_metrics["recon_streak"] += 1 + loop_metrics["phase_counts"]["investigate"] += 1 + elif has_artifact: + loop_metrics["recon_streak"] = 0 + loop_metrics["phase_counts"]["build"] += 1 + else: + loop_metrics["recon_streak"] = 0 + loop_metrics["phase_counts"]["iterate"] += 1 + loop_metrics["max_recon_streak"] = max( + int(loop_metrics["max_recon_streak"]), int(loop_metrics["recon_streak"]) + ) self._emit( f"[d{depth}/s{step}] model returned {len(turn.tool_calls)} tool call(s) ({elapsed:.1f}s): {', '.join(tc_names)}", on_event, @@ -618,6 +701,24 @@ def _solve_recursive( image=rl.image, ) + if ( + final_answer is None + and results + and int(loop_metrics["recon_streak"]) >= 3 + and not has_artifact + ): + loop_metrics["guardrail_warnings"] += 1 + soft_warning = ToolResult( + "recon-guardrail", + "system", + ( + "Soft guardrail: you've spent multiple consecutive steps in read/list/search mode " + "without producing artifacts. Move to implementation now (edit files, run targeted " + "validation, and return concrete outputs)." + ), + ) + results.append(soft_warning) + # Plan injection — find newest *.plan.md in session dir, append to last result if self.session_dir is not None and results and final_answer is None: try: @@ -650,6 +751,7 @@ def _solve_recursive( if final_answer is not None: self._emit(f"[d{depth}] completed in {step} step(s)", on_event) + self.last_loop_metrics = loop_metrics return final_answer for r in results: diff --git a/agent/runtime.py b/agent/runtime.py index d28b070e..2dfafcb9 100644 --- a/agent/runtime.py +++ b/agent/runtime.py @@ -228,6 +228,7 @@ class SessionRuntime: max_persisted_observations: int = 400 turn_history: list[TurnSummary] | None = None max_turn_summaries: int = 50 + loop_metrics: dict[str, Any] | None = None @classmethod def bootstrap( @@ -265,6 +266,19 @@ def bootstrap( except (KeyError, TypeError): pass max_turns = max(1, config.max_turn_summaries) + raw_loop_metrics = state.get("loop_metrics", {}) + loop_metrics: dict[str, Any] = raw_loop_metrics if isinstance(raw_loop_metrics, dict) else {} + loop_metrics.setdefault("turns", 0) + loop_metrics.setdefault("steps", 0) + loop_metrics.setdefault("model_turns", 0) + loop_metrics.setdefault("tool_calls", 0) + loop_metrics.setdefault("guardrail_warnings", 0) + loop_metrics.setdefault("final_rejections", 0) + loop_metrics.setdefault("phase_counts", {}) + if not isinstance(loop_metrics["phase_counts"], dict): + loop_metrics["phase_counts"] = {} + for phase in ("investigate", "build", "iterate", "finalize"): + loop_metrics["phase_counts"].setdefault(phase, 0) runtime = cls( engine=engine, @@ -274,6 +288,7 @@ def bootstrap( max_persisted_observations=max_obs, turn_history=turn_history[-max_turns:], max_turn_summaries=max_turns, + loop_metrics=loop_metrics, ) try: runtime.store.append_event( @@ -373,6 +388,34 @@ def _combined_on_step(step_event: dict[str, Any]) -> None: ) self.context = updated_context + latest_loop_metrics = self.engine.last_loop_metrics if isinstance(self.engine.last_loop_metrics, dict) else {} + if self.loop_metrics is None: + self.loop_metrics = { + "turns": 0, + "steps": 0, + "model_turns": 0, + "tool_calls": 0, + "guardrail_warnings": 0, + "final_rejections": 0, + "phase_counts": {"investigate": 0, "build": 0, "iterate": 0, "finalize": 0}, + } + self.loop_metrics["turns"] = int(self.loop_metrics.get("turns", 0)) + 1 + self.loop_metrics["steps"] = int(self.loop_metrics.get("steps", 0)) + int(latest_loop_metrics.get("steps", 0)) + self.loop_metrics["model_turns"] = int(self.loop_metrics.get("model_turns", 0)) + int(latest_loop_metrics.get("model_turns", 0)) + self.loop_metrics["tool_calls"] = int(self.loop_metrics.get("tool_calls", 0)) + int(latest_loop_metrics.get("tool_calls", 0)) + self.loop_metrics["guardrail_warnings"] = int(self.loop_metrics.get("guardrail_warnings", 0)) + int(latest_loop_metrics.get("guardrail_warnings", 0)) + self.loop_metrics["final_rejections"] = int(self.loop_metrics.get("final_rejections", 0)) + int(latest_loop_metrics.get("final_rejections", 0)) + phase_counts = self.loop_metrics.setdefault("phase_counts", {}) + latest_phase_counts = latest_loop_metrics.get("phase_counts", {}) + if not isinstance(phase_counts, dict): + phase_counts = {} + self.loop_metrics["phase_counts"] = phase_counts + if not isinstance(latest_phase_counts, dict): + latest_phase_counts = {} + for phase in ("investigate", "build", "iterate", "finalize"): + phase_counts[phase] = int(phase_counts.get(phase, 0)) + int(latest_phase_counts.get(phase, 0)) + self.loop_metrics["last_turn"] = latest_loop_metrics + # Generate turn summary if self.turn_history is None: self.turn_history = [] @@ -414,5 +457,6 @@ def _persist_state(self) -> None: } if self.turn_history: state["turn_history"] = [t.to_dict() for t in self.turn_history] + if self.loop_metrics: + state["loop_metrics"] = self.loop_metrics self.store.save_state(self.session_id, state) - diff --git a/tests/test_engine.py b/tests/test_engine.py index c0780fb9..5527bf5f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -136,6 +136,41 @@ def test_runtime_policy_blocks_repeated_shell_command(self) -> None: "expected policy block observation in context", ) + def test_meta_text_not_accepted_as_final_answer(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + cfg = AgentConfig(workspace=root, max_depth=1, max_steps_per_call=4, acceptance_criteria=False) + tools = WorkspaceTools(root=root) + model = ScriptedModel( + scripted_turns=[ + ModelTurn(text="Here is my plan: I will inspect files and then implement.", stop_reason="end_turn"), + ModelTurn(text="Concrete result delivered.", stop_reason="end_turn"), + ] + ) + engine = RLMEngine(model=model, tools=tools, config=cfg) + result = engine.solve("meta final rejection") + self.assertEqual(result, "Concrete result delivered.") + self.assertEqual(engine.last_loop_metrics.get("final_rejections"), 1) + + def test_soft_guardrail_for_repeated_recon(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + cfg = AgentConfig(workspace=root, max_depth=1, max_steps_per_call=6, acceptance_criteria=False) + tools = WorkspaceTools(root=root) + model = ScriptedModel( + scripted_turns=[ + ModelTurn(tool_calls=[_tc("list_files")]), + ModelTurn(tool_calls=[_tc("search_files", query="x")]), + ModelTurn(tool_calls=[_tc("repo_map")]), + ModelTurn(text="done", stop_reason="end_turn"), + ] + ) + engine = RLMEngine(model=model, tools=tools, config=cfg) + result, ctx = engine.solve_with_context("trigger recon guardrail") + self.assertEqual(result, "done") + self.assertTrue(any("Soft guardrail" in obs for obs in ctx.observations)) + self.assertGreaterEqual(int(engine.last_loop_metrics.get("guardrail_warnings", 0)), 1) + class CustomSystemPromptTests(unittest.TestCase): def test_custom_system_prompt_override(self) -> None: diff --git a/tests/test_turn_summaries.py b/tests/test_turn_summaries.py index c7e99828..008f095d 100644 --- a/tests/test_turn_summaries.py +++ b/tests/test_turn_summaries.py @@ -282,6 +282,75 @@ def test_backward_compat_old_state_no_turn_history(self) -> None: self.assertEqual(len(rt.turn_history), 1) self.assertEqual(rt.turn_history[0].turn_number, 1) + def test_loop_metrics_persisted_and_loaded_additively(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + cfg = self._make_config(root) + + model1 = ScriptedModel( + scripted_turns=[ + ModelTurn(tool_calls=[_tc("list_files")]), + ModelTurn(text="done-1", stop_reason="end_turn"), + ] + ) + engine1 = RLMEngine(model=model1, tools=WorkspaceTools(root=root), config=cfg) + rt1 = SessionRuntime.bootstrap( + engine=engine1, config=cfg, session_id="sess-loop", resume=False, + ) + rt1.solve("first") + + state_path = root / ".openplanter" / "sessions" / "sess-loop" / "state.json" + state_after_first = json.loads(state_path.read_text(encoding="utf-8")) + self.assertIn("loop_metrics", state_after_first) + self.assertEqual(state_after_first["loop_metrics"]["turns"], 1) + + model2 = ScriptedModel( + scripted_turns=[ModelTurn(text="done-2", stop_reason="end_turn")] + ) + engine2 = RLMEngine(model=model2, tools=WorkspaceTools(root=root), config=cfg) + rt2 = SessionRuntime.bootstrap( + engine=engine2, config=cfg, session_id="sess-loop", resume=True, + ) + self.assertIn("turns", rt2.loop_metrics) + rt2.solve("second") + + state_after_second = json.loads(state_path.read_text(encoding="utf-8")) + self.assertEqual(state_after_second["loop_metrics"]["turns"], 2) + self.assertIn("last_turn", state_after_second["loop_metrics"]) + + def test_backward_compat_old_state_no_loop_metrics(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + cfg = self._make_config(root) + + session_dir = root / ".openplanter" / "sessions" / "sess-no-loop" + session_dir.mkdir(parents=True) + (session_dir / "artifacts").mkdir() + (session_dir / "metadata.json").write_text( + json.dumps({"session_id": "sess-no-loop", "workspace": str(root)}), + encoding="utf-8", + ) + (session_dir / "state.json").write_text( + json.dumps({ + "session_id": "sess-no-loop", + "saved_at": "2026-01-01T00:00:00Z", + "external_observations": [], + }), + encoding="utf-8", + ) + + model = ScriptedModel( + scripted_turns=[ModelTurn(text="resumed", stop_reason="end_turn")] + ) + engine = RLMEngine(model=model, tools=WorkspaceTools(root=root), config=cfg) + rt = SessionRuntime.bootstrap( + engine=engine, config=cfg, session_id="sess-no-loop", resume=True, + ) + self.assertIsNotNone(rt.loop_metrics) + self.assertEqual(rt.loop_metrics.get("turns"), 0) + rt.solve("new turn") + self.assertEqual(rt.loop_metrics.get("turns"), 1) + if __name__ == "__main__": unittest.main() From c17a1bef9fc0ee3aca2920209b6473c4201081ff Mon Sep 17 00:00:00 2001 From: Drake Date: Fri, 13 Mar 2026 10:43:31 -0400 Subject: [PATCH 13/58] fix: resolve runtime loop metrics merge cleanup --- agent/engine.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/agent/engine.py b/agent/engine.py index 434503ea..cd12e735 100644 --- a/agent/engine.py +++ b/agent/engine.py @@ -397,8 +397,11 @@ def _solve_recursive( "max_recon_streak": 0, "guardrail_warnings": 0, "final_rejections": 0, + "last_guardrail_streak": 0, } + self.last_loop_metrics = loop_metrics + if replay_logger and replay_logger.needs_header: replay_logger.write_header( provider=type(model).__name__, @@ -413,9 +416,11 @@ def _solve_recursive( for step in range(1, self.config.max_steps_per_call + 1): if self._cancel.is_set(): self._emit(f"[d{depth}] cancelled by user", on_event) + self.last_loop_metrics = loop_metrics return "Task cancelled." if deadline and time.monotonic() > deadline: self._emit(f"[d{depth}] wall-clock limit reached", on_event) + self.last_loop_metrics = loop_metrics return "Time limit exceeded. Try a more focused objective." self._emit(f"[d{depth}/s{step}] calling model...", on_event) t0 = time.monotonic() @@ -427,6 +432,7 @@ def _solve_recursive( while True: if self._cancel.is_set(): self._emit(f"[d{depth}] cancelled by user", on_event) + self.last_loop_metrics = loop_metrics return "Task cancelled." try: turn = model.complete(conversation) @@ -434,6 +440,7 @@ def _solve_recursive( except RateLimitError as exc: if rate_limit_retries >= self.config.rate_limit_max_retries: self._emit(f"[d{depth}/s{step}] model error: {exc}", on_event) + self.last_loop_metrics = loop_metrics return f"Model error at depth {depth}, step {step}: {exc}" rate_limit_retries += 1 delay: float | None = None @@ -448,6 +455,7 @@ def _solve_recursive( delay = min(delay, self.config.rate_limit_backoff_max_sec) if deadline and (time.monotonic() + delay) > deadline: self._emit(f"[d{depth}] wall-clock limit reached", on_event) + self.last_loop_metrics = loop_metrics return "Time limit exceeded. Try a more focused objective." provider_code = f" ({exc.provider_code})" if exc.provider_code is not None else "" self._emit( @@ -459,6 +467,7 @@ def _solve_recursive( time.sleep(delay) except ModelError as exc: self._emit(f"[d{depth}/s{step}] model error: {exc}", on_event) + self.last_loop_metrics = loop_metrics return f"Model error at depth {depth}, step {step}: {exc}" finally: if hasattr(model, "on_content_delta"): @@ -706,8 +715,10 @@ def _solve_recursive( and results and int(loop_metrics["recon_streak"]) >= 3 and not has_artifact + and int(loop_metrics.get("last_guardrail_streak", 0)) != int(loop_metrics["recon_streak"]) ): loop_metrics["guardrail_warnings"] += 1 + loop_metrics["last_guardrail_streak"] = int(loop_metrics["recon_streak"]) soft_warning = ToolResult( "recon-guardrail", "system", @@ -757,6 +768,7 @@ def _solve_recursive( for r in results: context.add(f"[depth {depth} step {step}]\n{r.content}") + self.last_loop_metrics = loop_metrics return ( f"Step budget exhausted at depth {depth} for objective: {objective}\n" "Please try with a more specific task, higher step budget, or deeper recursion." From c96f6395d48c50f5f8f69f97a567a6e78411e018 Mon Sep 17 00:00:00 2001 From: Drake Thomsen <120344051+ThomsenDrake@users.noreply.github.com> Date: Fri, 13 Mar 2026 11:55:54 -0400 Subject: [PATCH 14/58] docs: add RFC for typed ontology-first InvestigationState --- docs/rfcs/0001-typed-investigation-state.md | 399 ++++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100644 docs/rfcs/0001-typed-investigation-state.md diff --git a/docs/rfcs/0001-typed-investigation-state.md b/docs/rfcs/0001-typed-investigation-state.md new file mode 100644 index 00000000..ac7c22e1 --- /dev/null +++ b/docs/rfcs/0001-typed-investigation-state.md @@ -0,0 +1,399 @@ +# RFC 0001: Typed `InvestigationState` (Ontology-First Session Memory) + +- **Status:** Proposed +- **Authors:** OpenPlanter team +- **Created:** 2026-03-13 +- **Target release:** staged rollout over 3 milestones +- **Scope:** session persistence (`state.json` successor), event/replay projection, runtime APIs for Python + Rust + +## 1. Summary + +This RFC defines an implementation-ready, typed `InvestigationState` to replace today’s mostly append-only text memory model with an ontology-first graph model centered on: + +- entities +- links +- claims +- evidence +- hypotheses +- open questions +- tasks/actions +- provenance +- confidence + +The current session state is predominantly `external_observations: string[]` with optional turn summaries and loop metrics, which biases memory toward late synthesis and makes structured reasoning (e.g., “which evidence supports this claim?”) difficult to perform incrementally. The new state introduces typed records with stable IDs, lifecycle fields, and confidence/provenance semantics that can be updated throughout the investigation. + +## 2. Motivation and Current Gaps + +## 2.1 Current Python session state is string-heavy and late-structured + +`SessionRuntime._persist_state()` persists `external_observations` as plain strings, plus `turn_history` and `loop_metrics`; no typed entities/claims/evidence graph exists in persisted state. The runtime loads this into `ExternalContext(observations=list[str])`, then injects summaries into prompts for later synthesis. This is useful for continuity, but it is not ontology-native. + +## 2.2 Current events and replay logs are rich but not canonicalized into typed state + +- `events.jsonl` captures `objective`, `trace`, `step`, `result`, and artifacts. +- `replay.jsonl` captures model call records (`header`, `call`, message snapshots/deltas, responses, token usage). + +These logs provide temporal traceability, but they are not normalized into first-class analytical objects (claims/evidence/hypotheses/tasks) that can be reasoned over directly. + +## 2.3 Python/Rust state model divergence + +Rust’s `ExternalContext` currently expects `observations: Vec` from `state.json`, while Python writes `external_observations: string[]`. This creates an interoperability mismatch and makes cross-runtime typed state consumption brittle. + +## 2.4 Consequences + +- hard to query support/opposition relationships for claims +- weak provenance granularity (source spans, extraction method, derived-from chain) +- confidence tracked informally in text, not as updateable fields +- poor lifecycle tracking for open questions, hypotheses, and tasks +- expensive/fragile “read all logs, then synthesize” behavior + +## 3. Goals and Non-Goals + +### 3.1 Goals + +1. Define a versioned, typed, ontology-first `InvestigationState` schema. +2. Preserve append-only logs (`events.jsonl`, `replay.jsonl`) as immutable trace, while introducing a mutable canonical state projection. +3. Provide deterministic migration from legacy `state.json` and optional bootstrap from replay/events logs. +4. Define runtime consumption contracts for both Python and Rust. +5. Enable incremental updates throughout the loop (investigate/build/iterate/finalize), not only final summarization. + +### 3.2 Non-Goals + +1. Replacing replay/events logging. +2. Building a global cross-session knowledge graph in this RFC. +3. Defining UI-level rendering details beyond data contract implications. + +## 4. Proposed Data Model + +## 4.1 File layout + +Within each session directory: + +- `investigation_state.json` (**new canonical typed state**) +- `state.json` (legacy compatibility; transitional) +- `events.jsonl` (append-only trace, unchanged) +- `replay.jsonl` (append-only model transcript, unchanged) + +## 4.2 Top-level schema + +```json +{ + "schema_version": "1.0.0", + "session_id": "20260313-120000-abc123", + "created_at": "2026-03-13T12:00:00Z", + "updated_at": "2026-03-13T12:05:00Z", + "objective": "Investigate relationships between X and Y", + "ontology": { + "namespace": "openplanter.core", + "version": "2026-03" + }, + "entities": {}, + "links": {}, + "claims": {}, + "evidence": {}, + "hypotheses": {}, + "questions": {}, + "tasks": {}, + "actions": {}, + "provenance_nodes": {}, + "confidence_profiles": {}, + "timeline": [], + "indexes": { + "by_external_ref": {}, + "by_tag": {} + }, + "legacy": { + "external_observations": [], + "turn_history": [], + "loop_metrics": {} + } +} +``` + +Design choice: object maps keyed by stable IDs (`ent_`, `clm_`, `ev_`, etc.) rather than only arrays to allow O(1) merge/update and conflict resolution. + +## 4.3 Core record types + +### 4.3.1 Entity + +Represents person/org/location/asset/document/event/concept. + +Required fields: + +- `id`, `kind`, `canonical_name`, `status` +- `created_at`, `updated_at` +- `provenance_ids[]` +- `confidence_id` + +Optional: + +- aliases, attributes, external_refs, tags + +```json +{ + "id": "ent_01H...", + "kind": "organization", + "canonical_name": "Acme Holdings LLC", + "aliases": ["Acme Holdings"], + "attributes": {"jurisdiction": "DE"}, + "external_refs": [{"system": "sec_cik", "value": "0000123456"}], + "status": "active", + "provenance_ids": ["prov_..."], + "confidence_id": "conf_...", + "created_at": "...", + "updated_at": "..." +} +``` + +### 4.3.2 Link + +Typed relationship between two entities (or entity↔claim where needed). + +- `source_entity_id`, `target_entity_id`, `predicate` +- `directional` (bool), `valid_time` (optional interval) +- provenance + confidence + +### 4.3.3 Claim + +Atomic proposition that may be supported or contradicted. + +- `text`, `claim_type` (`factual`, `attribution`, `quantitative`, etc.) +- `subject_refs[]` (entity/link IDs) +- `status` (`proposed`, `supported`, `contested`, `retracted`) +- `evidence_support_ids[]`, `evidence_contra_ids[]` +- provenance + confidence + +### 4.3.4 Evidence + +Observation/excerpt/document-derived fact unit. + +- `evidence_type` (`document`, `api_response`, `tool_output`, `human_note`) +- `content` (normalized value or excerpt) +- `source_uri`/`artifact_path`/`event_ref` +- `extraction` metadata (`method`, `extractor_version`, `span`) +- `hash` (optional dedupe) +- provenance + confidence + +### 4.3.5 Hypothesis + +Testable explanatory model composed of one or more claims. + +- `statement` +- `claim_ids[]` +- `status` (`open`, `plausible`, `weakened`, `rejected`, `accepted`) +- `test_plan_task_ids[]` +- provenance + confidence + +### 4.3.6 Open Question + +Resolvable question with lifecycle. + +- `question_text` +- `priority` (`low|medium|high|critical`) +- `status` (`open|in_progress|blocked|resolved|won't_fix`) +- `resolution_claim_id` (optional) +- `related_entity_ids[]`, `related_hypothesis_ids[]` +- provenance + confidence + +### 4.3.7 Task / Action + +Task = planned unit of work. Action = executed step/tool invocation. + +Task fields: + +- `title`, `description`, `status`, `assignee` (agent/human/system) +- `depends_on_task_ids[]`, `produced_ids[]`, `consumed_ids[]` +- `opened_by_question_id`/`opened_by_hypothesis_id` + +Action fields: + +- `task_id`, `action_type` (`tool_call`, `manual_edit`, `analysis_step`) +- `started_at`, `ended_at`, `outcome` +- `event_refs[]`, `replay_refs[]`, `artifact_paths[]` + +### 4.3.8 Provenance node + +First-class provenance object for source and transformation lineage. + +- `source_kind` (`event_log`, `replay_log`, `artifact`, `external_api`, `user_input`) +- `source_ref` (e.g., `events.jsonl#line:120`, URI, file path) +- `captured_at` +- `derived_from_ids[]` +- `method` (parser/model/tool), `method_version` + +### 4.3.9 Confidence profile + +Shared representation for confidence + rationale. + +- `score` (0.0-1.0) +- `grade` (`very_low|low|medium|high|very_high`) +- `dimensions` (source reliability, corroboration, recency, extraction certainty) +- `rationale` (short text) +- `updated_by` (agent/tool/user) + +## 4.4 Cross-object invariants + +1. All referenced IDs MUST exist. +2. `updated_at >= created_at`. +3. Closed objects (`resolved/rejected/retracted`) MUST include closure metadata (`closed_at`, `closed_reason`). +4. Claim status transition to `supported` requires at least one support evidence reference. +5. Evidence used by claims MUST include provenance. +6. Confidence profile referenced by object MUST exist (or explicit `null` if unknown is allowed by configuration). + +## 5. Lifecycle Model + +Each turn updates typed state continuously: + +1. **Ingest**: parse tool outputs/events into candidate evidence/entities. +2. **Normalize**: dedupe, entity resolution, link extraction. +3. **Assert**: create/update claims and hypothesis weights. +4. **Plan**: open/close questions; generate/update tasks. +5. **Act**: execute actions and attach provenance/replay refs. +6. **Review**: recompute confidence and status transitions. +7. **Persist**: atomic write of `investigation_state.json` + event emission. + +State updates are **idempotent upserts** keyed by IDs or deterministic signatures. + +## 6. Migration Plan + +## 6.1 Legacy inputs + +- `state.json` (primary): `external_observations`, `turn_history`, `loop_metrics` +- `events.jsonl` (optional enrichment) +- `replay.jsonl` (optional deep enrichment) + +## 6.2 Migration phases + +### Phase A (compatibility + scaffold) + +- Introduce writer for `investigation_state.json` with top-level metadata and `legacy` block copied from current `state.json`. +- Build pseudo-evidence from each legacy observation: + - `evidence_type = "legacy_observation"` + - content = observation string + - provenance source = `state.json#external_observations[i]` + - confidence = default baseline (e.g., 0.4, low) + +### Phase B (log projection backfill) + +- Parse `events.jsonl` to synthesize tasks/actions timeline: + - `objective` -> task roots + - `step` -> action nodes + - `result` -> claim/hypothesis candidate notes +- Parse `replay.jsonl` for optional high-fidelity provenance edges: + - map model/tool turns to `action.replay_refs` + - attach token/time diagnostics to action metadata + +### Phase C (native typed operation) + +- Runtime writes typed objects directly during investigation loop. +- Legacy `state.json` becomes derived compatibility projection (or frozen fallback). + +## 6.3 Deterministic ID strategy + +Use ULID/UUIDv7 for new runtime objects; for migrated objects optionally derive stable hash IDs from `(session_id, source_ref, normalized_content)` to avoid duplicate backfills. + +## 6.4 Conflict handling + +- If object exists: merge by field precedence (`new structured parse` > `legacy text parse` > `defaults`). +- If confidence differs: keep latest score and append to confidence history (optional extension field). + +## 7. Runtime Consumption Contracts + +## 7.1 Python runtime contract + +Add a typed state layer in Python: + +- `InvestigationState` dataclasses / pydantic models. +- Loader order: + 1. load `investigation_state.json` if present and version-compatible + 2. else migrate from `state.json` (+ optional logs) +- During `solve()`, update typed graph incrementally from steps/results. +- Persist both: + - canonical `investigation_state.json` + - compatibility `state.json` (minimal projection for older consumers) + +Recommended module boundaries: + +- `agent/investigation_state/schema.py` +- `agent/investigation_state/store.py` +- `agent/investigation_state/migrate.py` +- `agent/investigation_state/projectors.py` (events/replay -> typed) + +## 7.2 Rust runtime contract + +Replace/extend `engine::context::ExternalContext` usage with typed equivalents: + +- `InvestigationState` serde structs mirroring schema version 1. +- tolerant deserialization with `#[serde(default)]` for forward-compatible additive fields. +- loader order identical to Python. +- provide read APIs for prompt assembly: + - high-confidence active claims + - unresolved high-priority questions + - active hypotheses + recent supporting evidence + +Recommended modules: + +- `op-core/src/engine/investigation_state.rs` +- `op-core/src/engine/investigation_migrate.rs` +- keep `context.rs` as compatibility facade during transition + +## 7.3 Interop guarantees + +1. Shared JSON schema version and semantic rules. +2. Unknown fields ignored, known fields validated. +3. Both runtimes can round-trip without lossy deletion of unknown extension fields. + +## 8. Schema Governance and Validation + +- Publish JSON Schema at `docs/schemas/investigation_state.schema.json` (follow-up RFC task). +- Enforce `schema_version` and migration matrix. +- Add golden session fixtures (legacy + migrated + native typed) for Python/Rust parity tests. + +## 9. Rollout Plan + +### Milestone 1 (1-2 sprints) + +- Write/read scaffold + migration from `state.json`. +- No prompt changes required yet. + +### Milestone 2 (1-2 sprints) + +- Event/replay projector for tasks/actions/provenance. +- Prompt/context assembly begins consuming typed slices. + +### Milestone 3 (2+ sprints) + +- Full ontology-native loop updates and confidence lifecycle. +- `state.json` reduced to compatibility export; deprecation notice. + +## 10. Backward Compatibility + +- Existing sessions remain readable. +- If only `state.json` exists, runtime auto-migrates in-memory and writes typed file. +- Legacy clients can continue reading `state.json` until formal removal. + +## 11. Risks and Mitigations + +- **Risk:** schema over-complexity slows iteration. + - **Mitigation:** strict v1 core + extension points. +- **Risk:** noisy auto-extraction creates low-quality entities/claims. + - **Mitigation:** confidence gating and status `proposed` until corroborated. +- **Risk:** Python/Rust drift. + - **Mitigation:** shared fixture suite + contract tests in CI. + +## 12. Open Design Questions + +1. Should confidence history be first-class now or deferred to v1.1? +2. Should we store denormalized indexes on disk or rebuild at load? +3. What minimum evidence requirements are needed before a claim can influence final answers? + +## 13. Implementation Checklist + +- [ ] Add canonical typed state file and loader in Python +- [ ] Add migration path from legacy `state.json` +- [ ] Add optional projectors from `events.jsonl` and `replay.jsonl` +- [ ] Add canonical typed state structs and loader in Rust +- [ ] Add compatibility projection writer to legacy `state.json` +- [ ] Add schema validation + fixtures + parity tests +- [ ] Update prompt/context assembly to consume typed state slices + From 0d9dd54a57116ded0fab675aaf0b490a6c7debda Mon Sep 17 00:00:00 2001 From: Drake Date: Fri, 13 Mar 2026 12:01:58 -0400 Subject: [PATCH 15/58] Improve desktop loop governance telemetry --- .../crates/op-core/src/engine/mod.rs | 231 +++++++++++++++++- .../crates/op-core/src/events.rs | 82 +++++++ .../op-core/tests/test_model_streaming.rs | 197 +++++++++++++-- .../crates/op-tauri/src/bridge.rs | 50 +++- .../frontend/src/api/events.test.ts | 64 ++++- .../frontend/src/api/events.ts | 29 ++- openplanter-desktop/frontend/src/api/types.ts | 44 +++- .../frontend/src/commands/slash.ts | 2 + .../frontend/src/components/App.ts | 4 + .../frontend/src/components/InputBar.ts | 8 + .../frontend/src/components/StatusBar.test.ts | 32 +++ .../frontend/src/components/StatusBar.ts | 14 +- openplanter-desktop/frontend/src/main.ts | 20 +- .../frontend/src/state/store.ts | 6 + 14 files changed, 739 insertions(+), 44 deletions(-) diff --git a/openplanter-desktop/crates/op-core/src/engine/mod.rs b/openplanter-desktop/crates/op-core/src/engine/mod.rs index 0df7fe9d..1803cf10 100644 --- a/openplanter-desktop/crates/op-core/src/engine/mod.rs +++ b/openplanter-desktop/crates/op-core/src/engine/mod.rs @@ -16,7 +16,7 @@ use tokio_util::sync::CancellationToken; use crate::builder::build_model; use crate::config::AgentConfig; -use crate::events::{DeltaEvent, DeltaKind, StepEvent, TokenUsage}; +use crate::events::{DeltaEvent, DeltaKind, LoopMetrics, LoopPhase, StepEvent, TokenUsage}; use crate::model::{BaseModel, Message, ModelTurn, RateLimitError}; use crate::prompts::build_system_prompt; use crate::tools::WorkspaceTools; @@ -208,8 +208,17 @@ pub trait SolveEmitter: Send + Sync { fn emit_trace(&self, message: &str); fn emit_delta(&self, event: DeltaEvent); fn emit_step(&self, event: StepEvent); - fn emit_complete(&self, result: &str); + fn emit_complete(&self, result: &str, loop_metrics: Option); fn emit_error(&self, message: &str); + fn emit_loop_health( + &self, + _depth: u32, + _step: u32, + _phase: LoopPhase, + _metrics: LoopMetrics, + _is_final: bool, + ) { + } /// Called when a background curator finishes updating wiki files. /// Default no-op — override in TauriEmitter/LoggingEmitter. fn emit_curator_update(&self, _summary: &str, _files_changed: u32) {} @@ -256,6 +265,21 @@ pub async fn demo_solve(objective: &str, emitter: &dyn SolveEmitter, cancel: Can tokio::time::sleep(std::time::Duration::from_millis(50)).await; } + let loop_metrics = LoopMetrics { + steps: 1, + model_turns: 1, + tool_calls: 0, + investigate_steps: 0, + build_steps: 0, + iterate_steps: 0, + finalize_steps: 1, + recon_streak: 0, + max_recon_streak: 0, + guardrail_warnings: 0, + final_rejections: 0, + }; + emitter.emit_loop_health(0, 1, LoopPhase::Finalize, loop_metrics.clone(), true); + // Emit step summary emitter.emit_step(StepEvent { depth: 0, @@ -267,9 +291,11 @@ pub async fn demo_solve(objective: &str, emitter: &dyn SolveEmitter, cancel: Can }, elapsed_ms: 350, is_final: true, + loop_phase: Some(LoopPhase::Finalize), + loop_metrics: Some(loop_metrics.clone()), }); - emitter.emit_complete(&response); + emitter.emit_complete(&response, Some(loop_metrics)); } /// Rough token estimate: ~4 chars per token. @@ -396,6 +422,102 @@ async fn chat_stream_with_rate_limit_retries( } } +fn is_meta_final_text(text: &str) -> bool { + let stripped = text.trim(); + if stripped.is_empty() { + return true; + } + let lower = stripped.to_ascii_lowercase(); + let meta_starts = [ + "here is my plan", + "here's my plan", + "here is the plan", + "here's the plan", + "here is my approach", + "here's my approach", + "here is the approach", + "here's the approach", + "here is my analysis", + "here's my analysis", + "here is the analysis", + "here's the analysis", + "let me", + "next, i will", + "next i will", + ]; + if meta_starts.iter().any(|p| lower.starts_with(p)) { + return true; + } + if stripped.split_whitespace().count() < 5 { + return false; + } + let padded = format!(" {lower} "); + [ + " i will ", + " i can ", + " i should ", + " i need to ", + " i want to ", + " i am going to ", + " plan to ", + " let me ", + " next, i will ", + " next i will ", + " i should start by ", + ] + .iter() + .any(|needle| padded.contains(needle)) +} + +fn is_recon_tool(name: &str) -> bool { + matches!( + name, + "list_files" + | "search_files" + | "repo_map" + | "web_search" + | "fetch_url" + | "read_file" + | "read_image" + | "list_artifacts" + | "read_artifact" + ) +} + +fn is_artifact_tool(name: &str) -> bool { + matches!( + name, + "write_file" | "apply_patch" | "edit_file" | "hashline_edit" + ) +} + +fn classify_loop_phase(tool_calls: &[crate::model::ToolCall], is_final: bool) -> LoopPhase { + if is_final { + return LoopPhase::Finalize; + } + if tool_calls.is_empty() { + return LoopPhase::Iterate; + } + let has_recon = tool_calls.iter().any(|tc| is_recon_tool(&tc.name)); + let has_artifact = tool_calls.iter().any(|tc| is_artifact_tool(&tc.name)); + if has_artifact { + LoopPhase::Build + } else if has_recon && tool_calls.iter().all(|tc| is_recon_tool(&tc.name)) { + LoopPhase::Investigate + } else { + LoopPhase::Iterate + } +} + +fn increment_phase(metrics: &mut LoopMetrics, phase: &LoopPhase) { + match phase { + LoopPhase::Investigate => metrics.investigate_steps += 1, + LoopPhase::Build => metrics.build_steps += 1, + LoopPhase::Iterate => metrics.iterate_steps += 1, + LoopPhase::Finalize => metrics.finalize_steps += 1, + } +} + /// Real solve flow with a multi-step agentic loop. /// /// Calls the model with tool definitions. If the model returns tool calls, @@ -441,6 +563,8 @@ pub async fn solve( ]; let max_steps = config.max_steps_per_call as usize; + let mut loop_metrics = LoopMetrics::default(); + let mut last_guardrail_streak = 0u32; // 3. Background curator channel let (curator_tx, mut curator_rx) = mpsc::unbounded_channel::(); @@ -500,6 +624,9 @@ pub async fn solve( } }; + loop_metrics.steps = step as u32; + loop_metrics.model_turns += 1; + // Append assistant message to conversation let tool_calls_opt = if turn.tool_calls.is_empty() { None @@ -511,8 +638,30 @@ pub async fn solve( tool_calls: tool_calls_opt, }); - // No tool calls → final answer + // No tool calls → final answer (unless rejected by governance) if turn.tool_calls.is_empty() { + if turn.text.trim().is_empty() { + emitter.emit_trace(&format!( + "[d0/s{step}] empty model response, requesting tool use or concrete final answer" + )); + messages.push(Message::User { + content: "No tool calls and no final answer were returned. Continue solving: use tools if needed or return the concrete final deliverable.".to_string(), + }); + continue; + } + if is_meta_final_text(&turn.text) { + loop_metrics.final_rejections += 1; + emitter.emit_trace(&format!( + "[d0/s{step}] rejected meta final answer; requesting concrete deliverable" + )); + messages.push(Message::User { + content: "Your previous response was process/meta commentary rather than a concrete final answer. Continue solving: use tools if needed and return a direct final deliverable.".to_string(), + }); + continue; + } + let phase = LoopPhase::Finalize; + increment_phase(&mut loop_metrics, &phase); + emitter.emit_loop_health(0, step as u32, phase.clone(), loop_metrics.clone(), true); let tool_name = None; emitter.emit_step(StepEvent { depth: 0, @@ -524,8 +673,10 @@ pub async fn solve( }, elapsed_ms: step_start.elapsed().as_millis() as u64, is_final: true, + loop_phase: Some(phase), + loop_metrics: Some(loop_metrics.clone()), }); - emitter.emit_complete(&turn.text); + emitter.emit_complete(&turn.text, Some(loop_metrics.clone())); tools.cleanup(); // Wait for in-flight curators before exiting finish_curators( @@ -542,6 +693,8 @@ pub async fn solve( return; } + loop_metrics.tool_calls += turn.tool_calls.len() as u32; + // Execute each tool call and collect results for tc in &turn.tool_calls { if cancel.is_cancelled() { @@ -568,6 +721,30 @@ pub async fn solve( }); } + let phase = classify_loop_phase(&turn.tool_calls, false); + if matches!(phase, LoopPhase::Investigate) { + loop_metrics.recon_streak += 1; + } else { + loop_metrics.recon_streak = 0; + } + loop_metrics.max_recon_streak = + loop_metrics.max_recon_streak.max(loop_metrics.recon_streak); + increment_phase(&mut loop_metrics, &phase); + if matches!(phase, LoopPhase::Investigate) + && loop_metrics.recon_streak >= 3 + && loop_metrics.recon_streak != last_guardrail_streak + { + loop_metrics.guardrail_warnings += 1; + last_guardrail_streak = loop_metrics.recon_streak; + emitter.emit_trace(&format!( + "[d0/s{step}] soft guardrail: multiple consecutive recon steps without artifacts; nudging toward implementation" + )); + messages.push(Message::User { + content: "Soft guardrail: you've spent multiple consecutive steps in read/list/search mode without producing artifacts. Move to implementation now: edit files, run targeted validation, and return concrete outputs.".to_string(), + }); + } + emitter.emit_loop_health(0, step as u32, phase.clone(), loop_metrics.clone(), false); + // Emit step (non-final) AFTER tools execute so the frontend // can refresh the wiki graph with newly written files. let first_tool = turn.tool_calls.first().map(|tc| tc.name.clone()); @@ -581,6 +758,8 @@ pub async fn solve( }, elapsed_ms: step_start.elapsed().as_millis() as u64, is_final: false, + loop_phase: Some(phase), + loop_metrics: Some(loop_metrics.clone()), }); // Spawn background curator after each non-final step @@ -640,6 +819,14 @@ mod tests { use super::*; use std::sync::{Arc, Mutex}; + fn tool_call(name: &str) -> crate::model::ToolCall { + crate::model::ToolCall { + id: format!("call-{name}"), + name: name.to_string(), + arguments: "{}".to_string(), + } + } + #[derive(Debug, Clone)] #[allow(dead_code)] enum RecordedEvent { @@ -685,7 +872,7 @@ mod tests { self.events.lock().unwrap().push(RecordedEvent::Step(event)); } - fn emit_complete(&self, result: &str) { + fn emit_complete(&self, result: &str, _loop_metrics: Option) { self.events .lock() .unwrap() @@ -1018,4 +1205,36 @@ mod tests { assert_eq!(content.len(), 8000, "recent tool result should be intact"); } } + + #[test] + fn test_is_meta_final_text_rejects_empty_and_meta_prefixes() { + assert!(is_meta_final_text("")); + assert!(is_meta_final_text( + "Here is my plan for finishing the task." + )); + assert!(is_meta_final_text( + "I should start by checking the workspace layout." + )); + assert!(!is_meta_final_text( + "Completed the fix and updated the failing test." + )); + } + + #[test] + fn test_classify_loop_phase_recon_only_is_investigate() { + let phase = classify_loop_phase(&[tool_call("read_file"), tool_call("list_files")], false); + assert_eq!(phase, LoopPhase::Investigate); + } + + #[test] + fn test_classify_loop_phase_artifact_tools_are_build() { + let phase = classify_loop_phase(&[tool_call("read_file"), tool_call("write_file")], false); + assert_eq!(phase, LoopPhase::Build); + } + + #[test] + fn test_classify_loop_phase_mixed_recon_and_non_recon_is_iterate() { + let phase = classify_loop_phase(&[tool_call("read_file"), tool_call("run_shell")], false); + assert_eq!(phase, LoopPhase::Iterate); + } } diff --git a/openplanter-desktop/crates/op-core/src/events.rs b/openplanter-desktop/crates/op-core/src/events.rs index 22c111d8..a5c8a834 100644 --- a/openplanter-desktop/crates/op-core/src/events.rs +++ b/openplanter-desktop/crates/op-core/src/events.rs @@ -18,6 +18,37 @@ pub struct StepEvent { pub tokens: TokenUsage, pub elapsed_ms: u64, pub is_final: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub loop_phase: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub loop_metrics: Option, +} + +/// High-level phase classification for the current loop step. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LoopPhase { + Investigate, + Build, + Iterate, + Finalize, +} + +/// Cumulative loop telemetry for health and governance UX. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct LoopMetrics { + pub steps: u32, + pub model_turns: u32, + pub tool_calls: u32, + pub investigate_steps: u32, + pub build_steps: u32, + pub iterate_steps: u32, + pub finalize_steps: u32, + pub recon_streak: u32, + pub max_recon_streak: u32, + pub guardrail_warnings: u32, + pub final_rejections: u32, } /// Token usage counters. @@ -48,6 +79,18 @@ pub enum DeltaKind { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CompleteEvent { pub result: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub loop_metrics: Option, +} + +/// Periodic loop health telemetry event. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LoopHealthEvent { + pub depth: u32, + pub step: u32, + pub phase: LoopPhase, + pub metrics: LoopMetrics, + pub is_final: bool, } /// Agent encountered an error. @@ -112,6 +155,7 @@ pub enum AgentEvent { Complete(CompleteEvent), Error(ErrorEvent), WikiUpdated(GraphData), + LoopHealth(LoopHealthEvent), } /// Configuration view sent to the frontend. @@ -364,6 +408,8 @@ mod tests { }, elapsed_ms: 2345, is_final: false, + loop_phase: None, + loop_metrics: None, }; let json = serde_json::to_string(&step).unwrap(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); @@ -373,6 +419,42 @@ mod tests { assert_eq!(parsed["tokens"]["input_tokens"], 1234); } + #[test] + fn test_loop_metrics_deserialize_backfills_new_fields() { + let parsed: LoopMetrics = serde_json::from_str( + r#"{ + "steps": 2, + "model_turns": 2, + "tool_calls": 1, + "investigate_steps": 1, + "build_steps": 0, + "iterate_steps": 0, + "finalize_steps": 1, + "recon_streak": 0, + "max_recon_streak": 1, + "final_rejections": 1 + }"#, + ) + .unwrap(); + + assert_eq!( + parsed, + LoopMetrics { + steps: 2, + model_turns: 2, + tool_calls: 1, + investigate_steps: 1, + build_steps: 0, + iterate_steps: 0, + finalize_steps: 1, + recon_streak: 0, + max_recon_streak: 1, + guardrail_warnings: 0, + final_rejections: 1, + } + ); + } + #[test] fn test_init_gate_state_serialization() { assert_eq!( diff --git a/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs b/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs index ae880264..fb43b3ef 100644 --- a/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs +++ b/openplanter-desktop/crates/op-core/tests/test_model_streaming.rs @@ -549,7 +549,7 @@ async fn test_solve_with_mock_anthropic() { fn emit_step(&self, event: StepEvent) { self.events.lock().unwrap().push(Ev::Step(event)); } - fn emit_complete(&self, result: &str) { + fn emit_complete(&self, result: &str, _loop_metrics: Option) { self.events .lock() .unwrap() @@ -657,7 +657,7 @@ async fn test_solve_with_mock_openai() { fn emit_step(&self, event: StepEvent) { self.events.lock().unwrap().push(Ev2::Step(event)); } - fn emit_complete(&self, result: &str) { + fn emit_complete(&self, result: &str, _loop_metrics: Option) { self.events .lock() .unwrap() @@ -754,7 +754,7 @@ async fn test_solve_http_error_emits_error() { fn emit_trace(&self, _: &str) {} fn emit_delta(&self, _: DeltaEvent) {} fn emit_step(&self, _: StepEvent) {} - fn emit_complete(&self, _: &str) {} + fn emit_complete(&self, _: &str, _: Option) {} fn emit_error(&self, msg: &str) { self.errors.lock().unwrap().push(msg.to_string()); } @@ -812,7 +812,7 @@ async fn test_solve_rate_limit_retry_eventually_completes() { fn emit_step(&self, _: StepEvent) {} - fn emit_complete(&self, result: &str) { + fn emit_complete(&self, result: &str, _loop_metrics: Option) { self.events .lock() .unwrap() @@ -902,7 +902,7 @@ async fn test_solve_cancel_emits_cancelled() { fn emit_trace(&self, _: &str) {} fn emit_delta(&self, _: DeltaEvent) {} fn emit_step(&self, _: StepEvent) {} - fn emit_complete(&self, _: &str) {} + fn emit_complete(&self, _: &str, _: Option) {} fn emit_error(&self, msg: &str) { self.events.lock().unwrap().push(msg.to_string()); } @@ -947,7 +947,7 @@ async fn test_solve_demo_mode_bypasses_llm() { fn emit_trace(&self, _: &str) {} fn emit_delta(&self, _: DeltaEvent) {} fn emit_step(&self, _: StepEvent) {} - fn emit_complete(&self, result: &str) { + fn emit_complete(&self, result: &str, _loop_metrics: Option) { self.events.lock().unwrap().push(result.to_string()); } fn emit_error(&self, msg: &str) { @@ -988,7 +988,7 @@ async fn test_solve_missing_key_emits_error() { fn emit_trace(&self, _: &str) {} fn emit_delta(&self, _: DeltaEvent) {} fn emit_step(&self, _: StepEvent) {} - fn emit_complete(&self, _: &str) {} + fn emit_complete(&self, _: &str, _: Option) {} fn emit_error(&self, msg: &str) { self.errors.lock().unwrap().push(msg.to_string()); } @@ -1089,7 +1089,7 @@ async fn start_stateful_mock_server(responses: Vec<&'static str>) -> SocketAddr async fn test_solve_multi_step_agentic_loop() { use op_core::config::AgentConfig; use op_core::engine::{SolveEmitter, solve}; - use op_core::events::StepEvent; + use op_core::events::{LoopMetrics, LoopPhase, StepEvent}; // Mock server: first call → tool call, second call → final answer let addr = @@ -1101,7 +1101,10 @@ async fn test_solve_multi_step_agentic_loop() { Trace(String), Delta(DeltaEvent), Step(StepEvent), - Complete(String), + Complete { + result: String, + loop_metrics: Option, + }, Error(String), } @@ -1121,11 +1124,11 @@ async fn test_solve_multi_step_agentic_loop() { fn emit_step(&self, event: StepEvent) { self.events.lock().unwrap().push(Ev3::Step(event)); } - fn emit_complete(&self, result: &str) { - self.events - .lock() - .unwrap() - .push(Ev3::Complete(result.to_string())); + fn emit_complete(&self, result: &str, loop_metrics: Option) { + self.events.lock().unwrap().push(Ev3::Complete { + result: result.to_string(), + loop_metrics, + }); } fn emit_error(&self, message: &str) { self.events @@ -1185,9 +1188,34 @@ async fn test_solve_multi_step_agentic_loop() { Some("list_files"), "first step should show list_files tool" ); + assert_eq!(steps[0].loop_phase, Some(LoopPhase::Investigate)); + assert_eq!( + steps[0] + .loop_metrics + .as_ref() + .map(|metrics| metrics.tool_calls), + Some(1) + ); + assert_eq!( + steps[0] + .loop_metrics + .as_ref() + .map(|metrics| metrics.recon_streak), + Some(1) + ); // Last step should be final assert!(steps.last().unwrap().is_final, "last step should be final"); + assert_eq!(steps.last().unwrap().loop_phase, Some(LoopPhase::Finalize)); + assert_eq!( + steps + .last() + .unwrap() + .loop_metrics + .as_ref() + .map(|metrics| metrics.tool_calls), + Some(1) + ); // Should have tool execution trace let has_tool_trace = recorded @@ -1217,9 +1245,12 @@ async fn test_solve_multi_step_agentic_loop() { // Should complete with the final answer text assert!( - recorded - .iter() - .any(|e| matches!(e, Ev3::Complete(t) if t.contains("Here is the answer"))), + recorded.iter().any(|e| matches!( + e, + Ev3::Complete { result, loop_metrics } + if result.contains("Here is the answer") + && loop_metrics.as_ref().map(|metrics| metrics.tool_calls) == Some(1) + )), "should complete with the final answer" ); @@ -1237,3 +1268,135 @@ async fn test_solve_multi_step_agentic_loop() { errors ); } + +const ANTHROPIC_SSE_META_FINAL: &str = "\ +event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_meta_1\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"usage\":{\"input_tokens\":40}}}\n\n\ +event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n\ +event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Here is my plan for finishing the task.\"}}\n\n\ +event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n\ +event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":9}}\n\n\ +event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"; + +const ANTHROPIC_SSE_CONCRETE_FINAL: &str = "\ +event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_meta_2\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"usage\":{\"input_tokens\":55}}}\n\n\ +event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n\ +event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Completed the task and produced the requested answer.\"}}\n\n\ +event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n\ +event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":11}}\n\n\ +event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"; + +#[tokio::test] +async fn test_solve_rejects_meta_final_until_concrete_completion() { + use op_core::config::AgentConfig; + use op_core::engine::{SolveEmitter, solve}; + use op_core::events::{LoopMetrics, StepEvent}; + + let addr = + start_stateful_mock_server(vec![ANTHROPIC_SSE_META_FINAL, ANTHROPIC_SSE_CONCRETE_FINAL]) + .await; + + #[derive(Debug, Clone)] + #[allow(dead_code)] + enum Ev4 { + Trace(String), + Step(StepEvent), + Complete { + result: String, + loop_metrics: Option, + }, + Error(String), + } + + struct TestEmitter4 { + events: Arc>>, + } + + impl SolveEmitter for TestEmitter4 { + fn emit_trace(&self, message: &str) { + self.events + .lock() + .unwrap() + .push(Ev4::Trace(message.to_string())); + } + + fn emit_delta(&self, _: DeltaEvent) {} + + fn emit_step(&self, event: StepEvent) { + self.events.lock().unwrap().push(Ev4::Step(event)); + } + + fn emit_complete(&self, result: &str, loop_metrics: Option) { + self.events.lock().unwrap().push(Ev4::Complete { + result: result.to_string(), + loop_metrics, + }); + } + + fn emit_error(&self, message: &str) { + self.events + .lock() + .unwrap() + .push(Ev4::Error(message.to_string())); + } + } + + let events = Arc::new(Mutex::new(Vec::new())); + let emitter = TestEmitter4 { + events: events.clone(), + }; + + let cfg = AgentConfig { + provider: "anthropic".into(), + model: "claude-sonnet-4-5".into(), + anthropic_api_key: Some("test-key".into()), + anthropic_base_url: format!("http://{addr}"), + demo: false, + ..Default::default() + }; + + let cancel = CancellationToken::new(); + solve("Produce the final answer directly", &cfg, &emitter, cancel).await; + + let recorded = events.lock().unwrap().clone(); + assert!( + recorded.iter().any(|event| matches!( + event, + Ev4::Trace(message) if message.contains("rejected meta final answer") + )), + "expected a meta-final rejection trace, got: {recorded:?}" + ); + + let steps: Vec<&StepEvent> = recorded + .iter() + .filter_map(|event| match event { + Ev4::Step(step) => Some(step), + _ => None, + }) + .collect(); + assert_eq!(steps.len(), 1, "only the concrete final should emit a step"); + assert!( + steps[0].is_final, + "the emitted step should be the concrete final" + ); + assert_eq!( + steps[0] + .loop_metrics + .as_ref() + .map(|metrics| metrics.final_rejections), + Some(1) + ); + + assert!( + recorded.iter().any(|event| matches!( + event, + Ev4::Complete { result, loop_metrics } + if result.contains("Completed the task") + && loop_metrics.as_ref().map(|metrics| metrics.final_rejections) == Some(1) + )), + "expected completion after the rejection loop, got: {recorded:?}" + ); + assert!( + !recorded.iter().any(|event| matches!(event, Ev4::Error(_))), + "did not expect errors, got: {recorded:?}" + ); +} diff --git a/openplanter-desktop/crates/op-tauri/src/bridge.rs b/openplanter-desktop/crates/op-tauri/src/bridge.rs index e522dbdc..4a436e59 100644 --- a/openplanter-desktop/crates/op-tauri/src/bridge.rs +++ b/openplanter-desktop/crates/op-tauri/src/bridge.rs @@ -11,7 +11,8 @@ use tauri::{AppHandle, Emitter}; use op_core::engine::SolveEmitter; use op_core::events::{ - CompleteEvent, CuratorUpdateEvent, DeltaEvent, DeltaKind, ErrorEvent, StepEvent, TraceEvent, + CompleteEvent, CuratorUpdateEvent, DeltaEvent, DeltaKind, ErrorEvent, LoopHealthEvent, + LoopMetrics, LoopPhase, StepEvent, TraceEvent, }; use op_core::session::replay::{ReplayEntry, ReplayLogger, StepToolCallEntry}; @@ -107,12 +108,13 @@ impl SolveEmitter for TauriEmitter { let _ = self.handle.emit("agent:step", event); } - fn emit_complete(&self, result: &str) { + fn emit_complete(&self, result: &str, loop_metrics: Option) { eprintln!("[bridge] complete: {result}"); let _ = self.handle.emit( "agent:complete", CompleteEvent { result: result.to_string(), + loop_metrics, }, ); } @@ -127,6 +129,26 @@ impl SolveEmitter for TauriEmitter { ); } + fn emit_loop_health( + &self, + depth: u32, + step: u32, + phase: LoopPhase, + metrics: LoopMetrics, + is_final: bool, + ) { + let _ = self.handle.emit( + "agent:loop-health", + LoopHealthEvent { + depth, + step, + phase, + metrics, + is_final, + }, + ); + } + fn emit_curator_update(&self, summary: &str, files_changed: u32) { eprintln!("[bridge] curator update: {summary} ({files_changed} files)"); let _ = self.handle.emit( @@ -308,7 +330,7 @@ impl SolveEmitter for LoggingEmitter { self.inner.emit_step(event); } - fn emit_complete(&self, result: &str) { + fn emit_complete(&self, result: &str, loop_metrics: Option) { let entry = ReplayEntry { seq: 0, timestamp: String::new(), @@ -334,13 +356,25 @@ impl SolveEmitter for LoggingEmitter { }); }); - self.inner.emit_complete(result); + self.inner.emit_complete(result, loop_metrics); } fn emit_error(&self, message: &str) { self.inner.emit_error(message); } + fn emit_loop_health( + &self, + depth: u32, + step: u32, + phase: LoopPhase, + metrics: LoopMetrics, + is_final: bool, + ) { + self.inner + .emit_loop_health(depth, step, phase, metrics, is_final); + } + fn emit_curator_update(&self, summary: &str, files_changed: u32) { // Log curator update to replay let entry = ReplayEntry { @@ -386,7 +420,7 @@ mod tests { fn emit_trace(&self, _: &str) {} fn emit_delta(&self, _: DeltaEvent) {} fn emit_step(&self, _: StepEvent) {} - fn emit_complete(&self, _: &str) {} + fn emit_complete(&self, _: &str, _: Option) {} fn emit_error(&self, _: &str) {} } @@ -503,7 +537,7 @@ mod tests { self.deltas.lock().unwrap().push(event); } fn emit_step(&self, _: StepEvent) {} - fn emit_complete(&self, _: &str) {} + fn emit_complete(&self, _: &str, _: Option) {} fn emit_error(&self, _: &str) {} } @@ -527,6 +561,8 @@ mod tests { tokens: Default::default(), elapsed_ms: 1, is_final: false, + loop_phase: None, + loop_metrics: None, }); let entries = ReplayLogger::read_all(tmp.path()).await.unwrap(); @@ -575,6 +611,8 @@ mod tests { tokens: Default::default(), elapsed_ms: 1, is_final: false, + loop_phase: None, + loop_metrics: None, }); let entries = ReplayLogger::read_all(tmp.path()).await.unwrap(); diff --git a/openplanter-desktop/frontend/src/api/events.test.ts b/openplanter-desktop/frontend/src/api/events.test.ts index 88ea1cc1..294aeed5 100644 --- a/openplanter-desktop/frontend/src/api/events.test.ts +++ b/openplanter-desktop/frontend/src/api/events.test.ts @@ -16,7 +16,9 @@ import { onAgentStep, onAgentDelta, onAgentComplete, + onAgentCompleteEvent, onAgentError, + onLoopHealth, onMigrationProgress, onWikiUpdated, } from "./events"; @@ -73,10 +75,40 @@ describe("event listeners", () => { await onAgentComplete(callback); const handler = listeners.get("agent:complete")!; - handler({ payload: { result: "final answer" } }); + handler({ + payload: { + result: "final answer", + loop_metrics: { final_rejections: 1 }, + }, + }); expect(callback).toHaveBeenCalledWith("final answer"); }); + it("onAgentCompleteEvent registers listener and forwards full payload", async () => { + const callback = vi.fn(); + await onAgentCompleteEvent(callback); + + const handler = listeners.get("agent:complete")!; + const payload = { + result: "final answer", + loop_metrics: { + steps: 2, + model_turns: 2, + tool_calls: 1, + investigate_steps: 1, + build_steps: 0, + iterate_steps: 0, + finalize_steps: 1, + recon_streak: 0, + max_recon_streak: 1, + guardrail_warnings: 0, + final_rejections: 1, + }, + }; + handler({ payload }); + expect(callback).toHaveBeenCalledWith(payload); + }); + it("onAgentError registers listener and extracts message", async () => { const callback = vi.fn(); await onAgentError(callback); @@ -114,6 +146,34 @@ describe("event listeners", () => { expect(callback).toHaveBeenCalledWith(payload); }); + it("onLoopHealth registers listener and forwards payload", async () => { + const callback = vi.fn(); + await onLoopHealth(callback); + + const handler = listeners.get("agent:loop-health")!; + const payload = { + depth: 0, + step: 3, + phase: "investigate", + metrics: { + steps: 3, + model_turns: 3, + tool_calls: 2, + investigate_steps: 2, + build_steps: 0, + iterate_steps: 0, + finalize_steps: 0, + recon_streak: 2, + max_recon_streak: 2, + guardrail_warnings: 1, + final_rejections: 1, + }, + is_final: false, + }; + handler({ payload }); + expect(callback).toHaveBeenCalledWith(payload); + }); + it("all listeners return unlisten function", async () => { const noop = vi.fn(); const unlistens = await Promise.all([ @@ -121,7 +181,9 @@ describe("event listeners", () => { onAgentStep(noop), onAgentDelta(noop), onAgentComplete(noop), + onAgentCompleteEvent(noop), onAgentError(noop), + onLoopHealth(noop), onMigrationProgress(noop), onWikiUpdated(noop), ]); diff --git a/openplanter-desktop/frontend/src/api/events.ts b/openplanter-desktop/frontend/src/api/events.ts index 845ba8b9..0801c234 100644 --- a/openplanter-desktop/frontend/src/api/events.ts +++ b/openplanter-desktop/frontend/src/api/events.ts @@ -2,11 +2,16 @@ import { listen, type UnlistenFn } from "@tauri-apps/api/event"; import type { AgentEvent, + CompleteEvent, CuratorUpdateEvent, GraphData, + LoopHealthEvent, MigrationProgressEvent, } from "./types"; +type AgentStepEvent = Extract; +type AgentDeltaEvent = Extract; + export function onAgentTrace( callback: (message: string) => void ): Promise { @@ -16,23 +21,27 @@ export function onAgentTrace( } export function onAgentStep( - callback: (event: AgentEvent & { type: "step" }) => void + callback: (event: AgentStepEvent) => void ): Promise { - return listen("agent:step", (e) => callback(e.payload as any)); + return listen("agent:step", (e) => callback(e.payload)); } export function onAgentDelta( - callback: (event: AgentEvent & { type: "delta" }) => void + callback: (event: AgentDeltaEvent) => void +): Promise { + return listen("agent:delta", (e) => callback(e.payload)); +} + +export function onAgentCompleteEvent( + callback: (event: CompleteEvent) => void ): Promise { - return listen("agent:delta", (e) => callback(e.payload as any)); + return listen("agent:complete", (e) => callback(e.payload)); } export function onAgentComplete( callback: (result: string) => void ): Promise { - return listen<{ result: string }>("agent:complete", (e) => - callback(e.payload.result) - ); + return onAgentCompleteEvent((event) => callback(event.result)); } export function onAgentError( @@ -64,3 +73,9 @@ export function onMigrationProgress( callback(e.payload) ); } + +export function onLoopHealth( + callback: (event: LoopHealthEvent) => void +): Promise { + return listen("agent:loop-health", (e) => callback(e.payload)); +} diff --git a/openplanter-desktop/frontend/src/api/types.ts b/openplanter-desktop/frontend/src/api/types.ts index 22c4d605..2f13046b 100644 --- a/openplanter-desktop/frontend/src/api/types.ts +++ b/openplanter-desktop/frontend/src/api/types.ts @@ -9,6 +9,22 @@ export interface TraceEvent { message: string; } +export type LoopPhase = "investigate" | "build" | "iterate" | "finalize"; + +export interface LoopMetrics { + steps: number; + model_turns: number; + tool_calls: number; + investigate_steps: number; + build_steps: number; + iterate_steps: number; + finalize_steps: number; + recon_streak: number; + max_recon_streak: number; + guardrail_warnings: number; + final_rejections: number; +} + export interface StepEvent { depth: number; step: number; @@ -16,6 +32,8 @@ export interface StepEvent { tokens: TokenUsage; elapsed_ms: number; is_final: boolean; + loop_phase?: LoopPhase; + loop_metrics?: LoopMetrics; } export type DeltaKind = "text" | "thinking" | "tool_call_start" | "tool_call_args"; @@ -27,6 +45,15 @@ export interface DeltaEvent { export interface CompleteEvent { result: string; + loop_metrics?: LoopMetrics; +} + +export interface LoopHealthEvent { + depth: number; + step: number; + phase: LoopPhase; + metrics: LoopMetrics; + is_final: boolean; } export interface ErrorEvent { @@ -213,8 +240,19 @@ export interface ReplayEntry { export type AgentEvent = | { type: "trace"; message: string } - | { type: "step"; depth: number; step: number; tool_name: string | null; tokens: TokenUsage; elapsed_ms: number; is_final: boolean } + | { + type: "step"; + depth: number; + step: number; + tool_name: string | null; + tokens: TokenUsage; + elapsed_ms: number; + is_final: boolean; + loop_phase?: LoopPhase; + loop_metrics?: LoopMetrics; + } | { type: "delta"; kind: DeltaKind; text: string } - | { type: "complete"; result: string } + | { type: "complete"; result: string; loop_metrics?: LoopMetrics } | { type: "error"; message: string } - | { type: "wiki_updated"; nodes: GraphNode[]; edges: GraphEdge[] }; + | { type: "wiki_updated"; nodes: GraphNode[]; edges: GraphEdge[] } + | { type: "loop_health"; depth: number; step: number; phase: LoopPhase; metrics: LoopMetrics; is_final: boolean }; diff --git a/openplanter-desktop/frontend/src/commands/slash.ts b/openplanter-desktop/frontend/src/commands/slash.ts index 125eeb14..99118010 100644 --- a/openplanter-desktop/frontend/src/commands/slash.ts +++ b/openplanter-desktop/frontend/src/commands/slash.ts @@ -56,6 +56,8 @@ export async function dispatchSlashCommand(input: string): Promise { outputTokens: 0, currentStep: 0, currentDepth: 0, + loopHealth: null, + lastLoopMetrics: null, inputQueue: [], })); // Dispatch event to clear ChatPane DOM @@ -162,6 +164,8 @@ async function switchToSession(sessionId: string, sessionList: HTMLElement): Pro outputTokens: 0, currentStep: 0, currentDepth: 0, + loopHealth: null, + lastLoopMetrics: null, inputQueue: [], })); // Dispatch event to clear ChatPane DOM diff --git a/openplanter-desktop/frontend/src/components/InputBar.ts b/openplanter-desktop/frontend/src/components/InputBar.ts index 4575bb27..3b39aa84 100644 --- a/openplanter-desktop/frontend/src/components/InputBar.ts +++ b/openplanter-desktop/frontend/src/components/InputBar.ts @@ -112,6 +112,10 @@ export function createInputBar(): HTMLElement { appState.update((s) => ({ ...s, isRunning: true, + currentStep: 0, + currentDepth: 0, + loopHealth: null, + lastLoopMetrics: null, messages: [ ...s.messages, { @@ -240,6 +244,10 @@ export function createInputBar(): HTMLElement { appState.update((s) => ({ ...s, isRunning: true, + currentStep: 0, + currentDepth: 0, + loopHealth: null, + lastLoopMetrics: null, messages: [ ...s.messages, { diff --git a/openplanter-desktop/frontend/src/components/StatusBar.test.ts b/openplanter-desktop/frontend/src/components/StatusBar.test.ts index 34aba5ca..8e83288d 100644 --- a/openplanter-desktop/frontend/src/components/StatusBar.test.ts +++ b/openplanter-desktop/frontend/src/components/StatusBar.test.ts @@ -97,6 +97,38 @@ describe("createStatusBar", () => { expect(bar.querySelector(".session")!.textContent).toBe("step 3 depth 1"); }); + it("shows loop health details when telemetry is present", () => { + appState.update((s) => ({ + ...s, + isRunning: true, + currentStep: 4, + currentDepth: 0, + loopHealth: { + depth: 0, + step: 4, + phase: "investigate", + metrics: { + steps: 4, + model_turns: 4, + tool_calls: 2, + investigate_steps: 3, + build_steps: 0, + iterate_steps: 0, + finalize_steps: 0, + recon_streak: 3, + max_recon_streak: 3, + guardrail_warnings: 1, + final_rejections: 2, + }, + is_final: false, + }, + })); + const bar = createStatusBar(); + expect(bar.querySelector(".session")!.textContent).toBe( + "step 4 depth 0 investigate recon:3 reject:2 guard:1" + ); + }); + it("renders token counts", () => { appState.update((s) => ({ ...s, inputTokens: 5000, outputTokens: 2500 })); const bar = createStatusBar(); diff --git a/openplanter-desktop/frontend/src/components/StatusBar.ts b/openplanter-desktop/frontend/src/components/StatusBar.ts index f2f119ad..2bd3aa36 100644 --- a/openplanter-desktop/frontend/src/components/StatusBar.ts +++ b/openplanter-desktop/frontend/src/components/StatusBar.ts @@ -47,7 +47,19 @@ export function createStatusBar(): HTMLElement { sessionEl.textContent = s.sessionId ? `session ${s.sessionId.slice(0, 8)}` : ""; if (s.isRunning && s.currentStep > 0) { - sessionEl.textContent = `step ${s.currentStep} depth ${s.currentDepth}`; + const health = s.loopHealth; + if (health) { + const guardrailText = + health.metrics.guardrail_warnings > 0 + ? ` guard:${health.metrics.guardrail_warnings}` + : ""; + sessionEl.textContent = + `step ${s.currentStep} depth ${s.currentDepth} ` + + `${health.phase} recon:${health.metrics.recon_streak} ` + + `reject:${health.metrics.final_rejections}${guardrailText}`; + } else { + sessionEl.textContent = `step ${s.currentStep} depth ${s.currentDepth}`; + } } const inK = (s.inputTokens / 1000).toFixed(1); diff --git a/openplanter-desktop/frontend/src/main.ts b/openplanter-desktop/frontend/src/main.ts index bb9696a8..c5b61f9a 100644 --- a/openplanter-desktop/frontend/src/main.ts +++ b/openplanter-desktop/frontend/src/main.ts @@ -3,11 +3,12 @@ import { getConfig, getInitStatus } from "./api/invoke"; import { onAgentTrace, onAgentDelta, - onAgentComplete, + onAgentCompleteEvent, onAgentError, onAgentStep, onWikiUpdated, onCuratorUpdate, + onLoopHealth, onMigrationProgress, } from "./api/events"; import { appState } from "./state/store"; @@ -113,6 +114,7 @@ async function init() { outputTokens: s.outputTokens + event.tokens.output_tokens, currentStep: event.step, currentDepth: event.depth, + lastLoopMetrics: event.loop_metrics ?? s.lastLoopMetrics, })); // Dispatch to ChatPane for rich step summary rendering @@ -126,18 +128,20 @@ async function init() { window.dispatchEvent(detail); }); - await onAgentComplete((result) => { + await onAgentCompleteEvent((event) => { appState.update((s) => ({ ...s, isRunning: false, currentStep: 0, currentDepth: 0, + loopHealth: null, + lastLoopMetrics: event.loop_metrics ?? s.lastLoopMetrics, messages: [ ...s.messages, { id: crypto.randomUUID(), role: "assistant" as const, - content: result, + content: event.result, timestamp: Date.now(), isRendered: true, }, @@ -154,6 +158,7 @@ async function init() { isRunning: false, currentStep: 0, currentDepth: 0, + loopHealth: null, messages: [ ...s.messages, { @@ -192,6 +197,15 @@ async function init() { window.dispatchEvent(new CustomEvent("curator-done")); }); + + await onLoopHealth((event) => { + appState.update((s) => ({ + ...s, + loopHealth: event, + lastLoopMetrics: event.metrics, + })); + }); + await onMigrationProgress((event) => { appState.update((s) => ({ ...s, diff --git a/openplanter-desktop/frontend/src/state/store.ts b/openplanter-desktop/frontend/src/state/store.ts index 27d1a382..bf21d516 100644 --- a/openplanter-desktop/frontend/src/state/store.ts +++ b/openplanter-desktop/frontend/src/state/store.ts @@ -1,6 +1,8 @@ /** Simple observable state store. */ import type { InitStatusView, + LoopMetrics, + LoopHealthEvent, MigrationInitResultView, MigrationProgressEvent, } from "../api/types"; @@ -81,6 +83,8 @@ export interface AppState { maxStepsPerCall: number; currentStep: number; currentDepth: number; + loopHealth: LoopHealthEvent | null; + lastLoopMetrics: LoopMetrics | null; inputHistory: string[]; inputQueue: string[]; initGateState: "ready" | "requires_action" | "blocked"; @@ -109,6 +113,8 @@ export const appState = new Store({ maxStepsPerCall: 100, currentStep: 0, currentDepth: 0, + loopHealth: null, + lastLoopMetrics: null, inputHistory: [], inputQueue: [], initGateState: "ready", From 12ac998deb7a33bbe0152d856d02c49d42b5db7c Mon Sep 17 00:00:00 2001 From: Drake Thomsen <120344051+ThomsenDrake@users.noreply.github.com> Date: Fri, 13 Mar 2026 12:06:35 -0400 Subject: [PATCH 16/58] docs: add RFC for evidence normalization and action layer --- ...research-normalization-and-action-layer.md | 374 ++++++++++++++++++ 1 file changed, 374 insertions(+) create mode 100644 docs/rfcs/0001-research-normalization-and-action-layer.md diff --git a/docs/rfcs/0001-research-normalization-and-action-layer.md b/docs/rfcs/0001-research-normalization-and-action-layer.md new file mode 100644 index 00000000..4358f0da --- /dev/null +++ b/docs/rfcs/0001-research-normalization-and-action-layer.md @@ -0,0 +1,374 @@ +# RFC 0001: Research Normalization and Future Action Layer + +- **Status:** Draft +- **Authors:** OpenPlanter contributors +- **Last Updated:** 2026-03-13 +- **Audience:** Agent/runtime, ontology, and workflow maintainers + +## 1) Summary + +This RFC defines a two-part architecture for ontology-first investigations: + +1. **Research normalization**: all incoming research artifacts (local files, web fetches, transcripts, API responses, search results) are normalized into a single **Evidence** model. +2. **Future action layer**: unresolved questions derived from evidence/claims become explicit **NextAction** records with expected payoff, required inputs, and provenance-backed rationale. + +The design is intentionally provenance-heavy: every normalized object and every action recommendation must preserve where it came from, when it was observed, and how confident we are in its freshness and relevance. + +## 2) Motivation + +Investigations currently involve heterogeneous inputs with inconsistent metadata and ad hoc follow-up planning. This causes: + +- brittle downstream extraction/claiming logic, +- weak comparability between evidence types, +- missing or lossy provenance, +- and no unified, inspectable queue of “what to do next.” + +For an ontology-first workflow, we need stable primitives: + +- **Evidence** as the canonical atomic observation unit, +- **Claim** as a typed assertion grounded in evidence, +- **Question** as explicit uncertainty, +- **NextAction** as executable resolution path with expected payoff. + +## 3) Goals + +1. Define a canonical evidence model that all ingestion paths map to. +2. Preserve complete provenance chains (source, retrieval, transforms, extractor versions). +3. Track freshness/temporal validity separately from extraction confidence. +4. Standardize extracted entities and links between evidence and claims. +5. Convert unresolved questions into prioritized, auditable next actions. +6. Keep the model implementation-agnostic enough for CLI and desktop workflows. + +## 4) Non-goals + +- Prescribing a specific storage backend (SQLite, graph DB, document store). +- Replacing existing dataset-specific fetchers. +- Defining UI pixel-level behavior for action rendering. +- Mandating one ranking model for payoff estimation. + +## 5) Design Principles + +1. **Ontology first**: entities, relations, claims, and questions use typed ontology IDs before free-form tags. +2. **Provenance by default**: no evidence/claim/action without source and processing lineage. +3. **Lossless normalization**: preserve source-native payloads; add normalized projections. +4. **Temporal explicitness**: distinguish publication date, retrieval date, and validity window. +5. **Actionability over verbosity**: unresolved uncertainty should produce concrete, bounded next actions. +6. **Composable confidence**: extraction confidence, source reliability, and freshness decay are separate signals. + +## 6) Canonical Evidence Model + +`Evidence` is the normalized envelope for every incoming artifact. + +```yaml +Evidence: + evidence_id: ev_ + kind: [local_file, web_fetch, transcript, api_response, search_result] + modality: [text, html, json, pdf, audio, video, table, mixed] + + content: + raw_ref: + normalized_text: + normalized_structured: + chunks: [ + { + chunk_id: ch_, + type: [paragraph, table_row, json_path, timestamped_utterance], + locator: , + text: , + hash: + } + ] + + provenance: + source_type: [filesystem, http, api, search_index, transcript_pipeline] + source_uri: + source_title: + publisher: + acquisition: + observed_at: + retrieved_at: + retrieval_method: + request_fingerprint: + response_fingerprint: + processing_lineage: + - stage: [decode, ocr, asr, parse, chunk, extract] + tool: + version: + run_id: + timestamp: + + freshness: + published_at: + effective_from: + effective_to: + stale_after: + recency_score: <0..1> + decay_policy: [none, linear, exponential, source_defined] + + reliability: + source_reliability_score: <0..1> + extraction_confidence: <0..1> + integrity: + checksum: + signature_verified: + + ontology_links: + entities: [ + { + entity_id: ent_, + ontology_type: , + mention_span: , + confidence: <0..1>, + resolution_state: [resolved, candidate, unresolved] + } + ] + relations: [ + { + relation_id: rel_, + predicate: , + subject_entity_id: ent_..., + object_entity_id: ent_..., + confidence: <0..1> + } + ] + + claim_links: + supports: [cl_] + contradicts: [cl_] + mentions: [cl_] + + governance: + sensitivity: [public, internal, restricted] + license: +``` + +### Required fields + +At minimum: `evidence_id`, `kind`, `provenance.source_uri`, `provenance.acquisition.retrieved_at`, and one content representation (`raw_ref`, `normalized_text`, or `normalized_structured`). + +## 7) Source-Specific Normalization Contracts + +Each ingestion path maps into the same `Evidence` envelope with source-specific adapters. + +### 7.1 Local files + +- `kind=local_file` +- `source_uri=file://` +- fingerprint from file bytes + inode metadata snapshot +- if structured file (CSV/JSON/Parquet), populate `normalized_structured` +- if text-like, also populate `normalized_text` and paragraph chunks + +### 7.2 Web fetches + +- `kind=web_fetch` +- `source_uri=https://...` after redirect resolution +- store HTTP metadata in provenance extension (status, etag, cache-control) +- keep raw HTML/PDF bytes immutable, plus extracted text projection +- capture canonical URL and retrieval agent identity + +### 7.3 Transcripts (audio/video/meeting/call) + +- `kind=transcript` +- include ASR engine/version in `processing_lineage` +- chunk type defaults to `timestamped_utterance` +- provenance should include media source and diarization metadata when available + +### 7.4 API responses + +- `kind=api_response` +- `source_uri=api:///` and request fingerprint +- normalized structured projection is primary +- capture pagination context and token scopes in provenance extension + +### 7.5 Search results + +- `kind=search_result` +- represent each result item as independent evidence with query provenance +- include ranking metadata (rank, score, provider) +- link result evidence to follow-up fetched evidence via derivation edges + +## 8) Provenance and Freshness Semantics + +### 8.1 Provenance chain + +Every derived artifact stores: + +- parent evidence IDs, +- transformation stage, +- tool version, +- timestamp. + +This enables full replay from claim/action back to raw source. + +### 8.2 Freshness semantics + +Freshness is not binary. We compute: + +- `recency_score` from source-specific decay policy, +- `stale_after` from explicit source directives if present, +- investigation-time override for domains where historical records remain valid. + +Claims should consume freshness as a weighting factor, not a hard validity gate. + +## 9) Entities, Claims, and Linking + +## 9.1 Entity extraction and resolution + +Entity mentions are extracted per chunk and mapped to ontology types. Resolution pipeline states: + +1. `unresolved` (new mention) +2. `candidate` (one or more possible canonical entities) +3. `resolved` (canonical entity assigned) + +Each state transition writes provenance (`who/what/when/how`). + +### 9.2 Claim model (minimal) + +```yaml +Claim: + claim_id: cl_ + claim_type: + subject_entity_id: ent_... + predicate: + object: + status: [proposed, supported, disputed, rejected] + support_evidence_ids: [ev_...] + contradiction_evidence_ids: [ev_...] + confidence: <0..1> + last_evaluated_at: +``` + +Evidence links to claims through `supports`, `contradicts`, or `mentions`. + +### 9.3 Contradiction handling + +Contradictions are first-class edges, not overwrite events. Investigations should preserve both competing evidence sets and open a resolving question if conflict materially affects conclusions. + +## 10) Unresolved Questions → Next Actions + +`Question` records represent uncertainty or missing information; `NextAction` records represent concrete attempts to resolve it. + +### 10.1 Question model + +```yaml +Question: + question_id: q_ + text: + ontology_scope: [entity_ids, claim_ids, predicates] + blocking_level: [critical, high, medium, low] + created_from: + evidence_ids: [ev_...] + claim_ids: [cl_...] + status: [open, in_progress, resolved, abandoned] +``` + +### 10.2 NextAction model + +```yaml +NextAction: + action_id: act_ + question_id: q_ + action_type: [fetch, search, extract, resolve_entity, verify_claim, request_human_input] + hypothesis: + + required_inputs: + required_evidence_kinds: [api_response, web_fetch, ...] + required_entities: [ent_...] + required_claims: [cl_...] + external_dependencies: [api_key:provider, tool:ocr_v2] + + expected_payoff: + uncertainty_reduction: <0..1> + decision_impact: <0..1> + graph_expansion_value: <0..1> + estimated_cost: