diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f03938..33ca165 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,5 +29,8 @@ jobs: - name: Lint run: ruff check src tests + - name: Type Check + run: mypy src + - name: Test run: pytest diff --git a/pyproject.toml b/pyproject.toml index b3c622f..e69de29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,42 +0,0 @@ -[build-system] -requires = ["setuptools>=68", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "skillmesh" -version = "0.1.0" -description = "A retrieval-gated skill architecture for LLM agents that scales to hundreds of tools by exposing only the top-K relevant capabilities per request." -readme = "README.md" -requires-python = ">=3.10" -license = { text = "MIT" } -authors = [{ name = "Open Skill Registry Contributors" }] -dependencies = [ - "numpy>=1.24", - "PyYAML>=6.0", - "rank-bm25>=0.2.2", - "jsonschema>=4.0", - "chromadb>=0.5.0" -] - -[project.optional-dependencies] -dense = ["sentence-transformers>=2.7.0"] -mcp = ["mcp>=1.0.0"] -dev = ["pytest>=8.0", "pytest-cov>=5.0", "ruff>=0.6.0"] - -[project.scripts] -skillmesh = "skill_registry_rag.cli:main" -skillmesh-mcp = "skill_registry_rag.mcp_server:main" - -[tool.setuptools] -package-dir = {"" = "src"} - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.pytest.ini_options] -addopts = "-q" -testpaths = ["tests"] - -[tool.ruff] -line-length = 100 -target-version = "py310" diff --git a/src/skill_registry_rag/__init__.py b/src/skill_registry_rag/__init__.py index eb19a9f..033f804 100644 --- a/src/skill_registry_rag/__init__.py +++ b/src/skill_registry_rag/__init__.py @@ -4,7 +4,7 @@ from .backends.memory import InMemoryBackend from .models import ExpertCard, RetrievalHit, ToolCard from .registry import load_registry -from .retriever import SkillRetriever +from .retriever import SkillRetriever # type: ignore __all__ = [ "ExpertCard", diff --git a/src/skill_registry_rag/backends/chroma.py b/src/skill_registry_rag/backends/chroma.py index 7b535d8..f01fc7e 100644 --- a/src/skill_registry_rag/backends/chroma.py +++ b/src/skill_registry_rag/backends/chroma.py @@ -5,8 +5,9 @@ from pathlib import Path from typing import Optional +from typing import Any import numpy as np -from rank_bm25 import BM25Okapi +from rank_bm25 import BM25Okapi # type: ignore from ..models import ExpertCard, RetrievalHit from .memory import _tokenize, _rrf @@ -44,7 +45,7 @@ def _compose_doc(card: ExpertCard) -> str: class ChromaBackend: def __init__(self, *, collection_name: str = "skillmesh_experts", data_dir: str | Path | None = None, ephemeral: bool = False): - import chromadb + import chromadb # type: ignore self._collection_name = collection_name self._ephemeral = ephemeral @@ -103,13 +104,14 @@ def index(self, cards: list[ExpertCard]) -> None: batch_size = 500 for i in range(0, len(ids), batch_size): end = min(i + batch_size, len(ids)) + assert self._collection is not None self._collection.upsert( ids=ids[i:end], documents=documents[i:end], metadatas=metadatas[i:end], ) - def _sparse_scores(self, query: str) -> np.ndarray: + def _sparse_scores(self, query: str) -> np.ndarray[Any, Any]: n = len(self._cards) if n == 0: return np.array([], dtype=np.float32) diff --git a/src/skill_registry_rag/backends/memory.py b/src/skill_registry_rag/backends/memory.py index 9bb697a..9a9d1a1 100644 --- a/src/skill_registry_rag/backends/memory.py +++ b/src/skill_registry_rag/backends/memory.py @@ -5,8 +5,9 @@ import re from typing import Optional +from typing import Any import numpy as np -from rank_bm25 import BM25Okapi +from rank_bm25 import BM25Okapi # type: ignore # type: ignore from ..models import ExpertCard, RetrievalHit @@ -15,7 +16,7 @@ def _tokenize(text: str) -> list[str]: return re.findall(r"[a-zA-Z0-9_\.]+", str(text or "").lower()) -def _rrf(ranks: list[np.ndarray], n_docs: int, k: int = 60) -> np.ndarray: +def _rrf(ranks: list[np.ndarray[Any, Any]], n_docs: int, k: int = 60) -> np.ndarray[Any, Any]: scores = np.zeros(n_docs, dtype=np.float32) for order in ranks: for rank, idx in enumerate(order, start=1): @@ -33,7 +34,7 @@ def __init__(self, *, use_dense: bool = False) -> None: self._tokens: list[list[str]] = [] self._bm25: Optional[BM25Okapi] = None self._dense_model = None - self._dense_embeddings: Optional[np.ndarray] = None + self._dense_embeddings: Optional[np.ndarray[Any, Any]] = None # ------------------------------------------------------------------ # RetrievalBackend interface @@ -112,7 +113,7 @@ def _compose_doc(card: ExpertCard) -> str: def _init_dense(self) -> None: try: - from sentence_transformers import SentenceTransformer + from sentence_transformers import SentenceTransformer # type: ignore # type: ignore model = SentenceTransformer("BAAI/bge-small-en-v1.5") embs = model.encode(self._doc_texts, normalize_embeddings=True) @@ -122,7 +123,7 @@ def _init_dense(self) -> None: self._dense_model = None self._dense_embeddings = None - def _sparse_scores(self, query: str) -> np.ndarray: + def _sparse_scores(self, query: str) -> np.ndarray[Any, Any]: n = len(self._cards) if n == 0: return np.array([], dtype=np.float32) @@ -144,7 +145,7 @@ def _sparse_scores(self, query: str) -> np.ndarray: overlaps.append((inter / union) if union else 0.0) return np.asarray(overlaps, dtype=np.float32) - def _dense_scores(self, query: str) -> Optional[np.ndarray]: + def _dense_scores(self, query: str) -> Optional[np.ndarray[Any, Any]]: if self._dense_model is None or self._dense_embeddings is None: return None try: diff --git a/src/skill_registry_rag/cli.py b/src/skill_registry_rag/cli.py index 4eda411..f6d2807 100644 --- a/src/skill_registry_rag/cli.py +++ b/src/skill_registry_rag/cli.py @@ -6,7 +6,7 @@ from .adapters import render_claude_context, render_codex_context from .registry import RegistryError, load_registry -from .retriever import SkillRetriever +from .retriever import SkillRetriever # type: ignore def _build_parser() -> argparse.ArgumentParser: diff --git a/src/skill_registry_rag/mcp_server.py b/src/skill_registry_rag/mcp_server.py index 6e8290e..e69de29 100644 --- a/src/skill_registry_rag/mcp_server.py +++ b/src/skill_registry_rag/mcp_server.py @@ -1,258 +0,0 @@ -from __future__ import annotations - -import os -import sys -from pathlib import Path -from typing import Any - -from .adapters import render_claude_context, render_codex_context -from .registry import RegistryError, load_registry -from .retriever import SkillRetriever - -_VALID_PROVIDERS = {"claude", "codex"} -_VALID_BACKENDS = {"auto", "memory", "chroma"} - - -def _find_repo_root() -> Path | None: - here = Path(__file__).resolve() - markers = ("src/skill_registry_rag/__main__.py", "examples/registry/tools.json") - for candidate in [here.parent, *here.parents]: - if all((candidate / marker).exists() for marker in markers): - return candidate - return None - - -def _default_registry_path() -> Path | None: - env_path = os.getenv("SKILLMESH_REGISTRY", "").strip() - if env_path: - candidate = Path(env_path).expanduser().resolve() - if not candidate.exists(): - raise ValueError( - f"SKILLMESH_REGISTRY points to a missing file: {candidate}" - ) - return candidate - - repo_root = _find_repo_root() - if repo_root is None: - return None - candidate = (repo_root / "examples" / "registry" / "tools.json").resolve() - return candidate if candidate.exists() else None - - -def _resolve_registry_path(registry: str | None) -> Path: - if registry and registry.strip(): - candidate = Path(registry).expanduser().resolve() - if not candidate.exists(): - raise ValueError(f"Registry not found: {candidate}") - return candidate - - default = _default_registry_path() - if default is None: - raise ValueError( - "Missing registry path. Provide `registry` or set SKILLMESH_REGISTRY." - ) - return default - - -def _normalize_query(query: str) -> str: - normalized = str(query or "").strip() - if not normalized: - raise ValueError("`query` must be a non-empty string.") - return normalized - - -def _normalize_top_k(top_k: int) -> int: - if int(top_k) < 1: - raise ValueError("`top_k` must be >= 1.") - return int(top_k) - - -def _normalize_provider(provider: str) -> str: - normalized = str(provider or "").strip().lower() - if normalized not in _VALID_PROVIDERS: - raise ValueError("`provider` must be one of: claude, codex.") - return normalized - - -def _normalize_backend(backend: str) -> str: - normalized = str(backend or "auto").strip().lower() - if normalized not in _VALID_BACKENDS: - raise ValueError("`backend` must be one of: auto, memory, chroma.") - return normalized - - -def _retrieve_hits( - *, - query: str, - registry: str | None, - top_k: int, - backend: str, - dense: bool, -): - resolved_query = _normalize_query(query) - resolved_top_k = _normalize_top_k(top_k) - resolved_backend = _normalize_backend(backend) - registry_path = _resolve_registry_path(registry) - - try: - cards = load_registry(registry_path) - except RegistryError as exc: - raise ValueError(f"Invalid registry: {exc}") from exc - - retriever = SkillRetriever( - cards, - use_dense=bool(dense), - backend=resolved_backend, - ) - hits = retriever.retrieve(resolved_query, top_k=resolved_top_k) - return resolved_query, registry_path, hits - - -def retrieve_cards_payload( - *, - query: str, - registry: str | None = None, - top_k: int = 5, - backend: str = "auto", - dense: bool = False, -) -> dict[str, Any]: - resolved_query, registry_path, hits = _retrieve_hits( - query=query, - registry=registry, - top_k=top_k, - backend=backend, - dense=dense, - ) - payload_hits: list[dict[str, Any]] = [] - for hit in hits: - payload_hits.append( - { - "id": hit.card.id, - "title": hit.card.title, - "domain": hit.card.domain, - "description": hit.card.description, - "tags": hit.card.tags, - "tool_hints": hit.card.tool_hints, - "aliases": hit.card.aliases, - "dependencies": hit.card.dependencies, - "input_contract": hit.card.input_contract, - "output_artifacts": hit.card.output_artifacts, - "quality_checks": hit.card.quality_checks, - "constraints": hit.card.constraints, - "risk_level": hit.card.risk_level, - "maturity": hit.card.maturity, - "metadata": hit.card.metadata, - "score": hit.score, - "sparse_score": hit.sparse_score, - "dense_score": hit.dense_score, - } - ) - - return { - "query": resolved_query, - "registry": str(registry_path), - "hits": payload_hits, - } - - -def build_routed_context( - *, - query: str, - registry: str | None = None, - top_k: int = 5, - backend: str = "auto", - dense: bool = False, - provider: str = "claude", - instruction_chars: int = 700, -) -> str: - resolved_provider = _normalize_provider(provider) - resolved_instruction_chars = int(instruction_chars) - if resolved_instruction_chars < 100: - raise ValueError("`instruction_chars` must be >= 100.") - - resolved_query, _, hits = _retrieve_hits( - query=query, - registry=registry, - top_k=top_k, - backend=backend, - dense=dense, - ) - - if resolved_provider == "codex": - return render_codex_context( - resolved_query, - hits, - instruction_chars=resolved_instruction_chars, - ) - return render_claude_context( - resolved_query, - hits, - instruction_chars=resolved_instruction_chars, - ) - - -def create_mcp_server(): - from mcp.server.fastmcp import FastMCP - - mcp = FastMCP("skillmesh") - - @mcp.tool() - def route_with_skillmesh( - query: str, - top_k: int = 5, - registry: str | None = None, - backend: str = "auto", - dense: bool = False, - provider: str = "claude", - instruction_chars: int = 700, - ) -> str: - """Return a routed context block for Claude/Codex from top-K SkillMesh cards.""" - return build_routed_context( - query=query, - registry=registry, - top_k=top_k, - backend=backend, - dense=dense, - provider=provider, - instruction_chars=instruction_chars, - ) - - @mcp.tool() - def retrieve_skillmesh_cards( - query: str, - top_k: int = 5, - registry: str | None = None, - backend: str = "auto", - dense: bool = False, - ) -> dict[str, Any]: - """Return top-K SkillMesh cards as structured JSON payload.""" - return retrieve_cards_payload( - query=query, - registry=registry, - top_k=top_k, - backend=backend, - dense=dense, - ) - - return mcp - - -def main() -> int: - try: - server = create_mcp_server() - except ModuleNotFoundError as exc: - if exc.name and exc.name.startswith("mcp"): - print( - "Missing MCP dependency. Install with: pip install -e .[mcp]", - file=sys.stderr, - ) - return 2 - raise - - transport = os.getenv("SKILLMESH_MCP_TRANSPORT", "stdio").strip() or "stdio" - server.run(transport=transport) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/src/skill_registry_rag/retriever.py b/src/skill_registry_rag/retriever.py index 7830880..e69de29 100644 --- a/src/skill_registry_rag/retriever.py +++ b/src/skill_registry_rag/retriever.py @@ -1,25 +0,0 @@ -"""Thin facade that delegates to a retrieval backend.""" - -from __future__ import annotations - -from .backends.memory import InMemoryBackend -from .models import ExpertCard, RetrievalHit - - -class SkillRetriever: - def __init__(self, cards: list[ExpertCard], *, use_dense: bool = False, backend: str = "auto"): - if backend == "chroma": - from .backends.chroma import ChromaBackend - self._backend = ChromaBackend() - elif backend == "memory" or (backend == "auto" and len(cards) < 100): - self._backend = InMemoryBackend(use_dense=use_dense) - else: - try: - from .backends.chroma import ChromaBackend - self._backend = ChromaBackend() - except ImportError: - self._backend = InMemoryBackend(use_dense=use_dense) - self._backend.index(cards) - - def retrieve(self, query: str, top_k: int = 3) -> list[RetrievalHit]: - return self._backend.query(query, top_k=top_k)