fix(docs): DO NOT MERGE - sales demo only

examples/sko-workshop-demo/agent-control-sales-workshop-demo/custom_evaluator_acme/pyproject.toml

@@ -0,0 +1,23 @@
+[project]
+name = "agent-control-evaluator-acme"
+version = "0.1.0"
+description = "ACME custom evaluators for AgentControl demos"
+requires-python = ">=3.12"
+license = { text = "Apache-2.0" }
+authors = [{ name = "ACME" }]
+dependencies = [
+  "agent-control-evaluators>=5.0.0,<7.0.0",
+  "agent-control-models>=5.0.0,<7.0.0",
+  "httpx>=0.27.0",
+]
+
+[project.entry-points."agent_control.evaluators"]
+"acme.tiered-discount" = "agent_control_evaluator_acme.tiered_discount:TieredDiscountEvaluator"
+"acme.llm-relevance" = "agent_control_evaluator_acme.llm_relevance:LLMRelevanceEvaluator"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/agent_control_evaluator_acme"]

examples/sko-workshop-demo/agent-control-sales-workshop-demo/custom_evaluator_acme/src/agent_control_evaluator_acme/__init__.py

@@ -0,0 +1,11 @@
+"""ACME custom evaluators package."""
+
+from .llm_relevance import LLMRelevanceConfig, LLMRelevanceEvaluator
+from .tiered_discount import TieredDiscountConfig, TieredDiscountEvaluator
+
+__all__ = [
+    "TieredDiscountConfig",
+    "TieredDiscountEvaluator",
+    "LLMRelevanceConfig",
+    "LLMRelevanceEvaluator",
+]

examples/sko-workshop-demo/agent-control-sales-workshop-demo/custom_evaluator_acme/src/agent_control_evaluator_acme/llm_relevance/__init__.py

@@ -0,0 +1,6 @@
+"""LLM relevance evaluator for AgentControl demos."""
+
+from .config import LLMRelevanceConfig
+from .evaluator import LLMRelevanceEvaluator
+
+__all__ = ["LLMRelevanceConfig", "LLMRelevanceEvaluator"]

examples/sko-workshop-demo/agent-control-sales-workshop-demo/custom_evaluator_acme/src/agent_control_evaluator_acme/llm_relevance/config.py

@@ -0,0 +1,20 @@
+from typing import Literal
+
+from agent_control_evaluators import EvaluatorConfig
+
+
+class LLMRelevanceConfig(EvaluatorConfig):
+    """Config for LLM-as-judge relevance scoring.
+
+    - model: LLM model name
+    - threshold: minimum score (0-1) required to pass
+    - api_key_env: env var that holds the API key
+    - base_url: OpenAI-compatible base URL (e.g., https://api.openai.com/v1)
+    - on_error: allow (fail open) or deny (fail closed)
+    """
+
+    model: str = "gpt-4o-mini"
+    threshold: float = 0.7
+    api_key_env: str = "OPENAI_API_KEY"
+    base_url: str = "https://api.openai.com/v1"
+    on_error: Literal["allow", "deny"] = "allow"

examples/sko-workshop-demo/agent-control-sales-workshop-demo/custom_evaluator_acme/src/agent_control_evaluator_acme/llm_relevance/evaluator.py

@@ -0,0 +1,131 @@
+import json
+import os
+from typing import Any
+
+import httpx
+from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator
+from agent_control_models import EvaluatorResult
+
+from .config import LLMRelevanceConfig
+
+
+_SYSTEM_PROMPT = (
+    "You are a strict evaluator. Given a QUESTION and an ANSWER, "
+    "score how relevant the answer is to the question on a 0 to 1 scale. "
+    "Return JSON with keys: score (float 0-1), rationale (string)."
+)
+
+
+@register_evaluator
+class LLMRelevanceEvaluator(Evaluator[LLMRelevanceConfig]):
+    """LLM-as-judge relevance evaluator (OpenAI-compatible API)."""
+
+    metadata = EvaluatorMetadata(
+        name="acme.llm-relevance",
+        version="1.0.0",
+        description="LLM-as-judge relevance scoring (0-1)",
+        requires_api_key=True,
+        timeout_ms=30000,
+    )
+    config_model = LLMRelevanceConfig
+
+    async def evaluate(self, data: Any) -> EvaluatorResult:
+        try:
+            question, answer = _extract_qa(data)
+            if not question or not answer:
+                return EvaluatorResult(
+                    matched=True,
+                    confidence=1.0,
+                    message="Missing question or answer",
+                )
+
+            api_key = os.getenv(self.config.api_key_env, "")
+            if not api_key:
+                return _error_result(
+                    "Missing API key",
+                    on_error=self.config.on_error,
+                )
+
+            payload = {
+                "model": self.config.model,
+                "messages": [
+                    {"role": "system", "content": _SYSTEM_PROMPT},
+                    {
+                        "role": "user",
+                        "content": f"QUESTION:\n{question}\n\nANSWER:\n{answer}",
+                    },
+                ],
+                "temperature": 0,
+            }
+
+            headers = {"Authorization": f"Bearer {api_key}"}
+            url = self.config.base_url.rstrip("/") + "/chat/completions"
+
+            async with httpx.AsyncClient(timeout=30.0) as client:
+                resp = await client.post(url, json=payload, headers=headers)
+                resp.raise_for_status()
+                data = resp.json()
+
+            content = (
+                data.get("choices", [{}])[0]
+                .get("message", {})
+                .get("content", "")
+            )
+            score, rationale = _parse_score(content)
+
+            if score is None:
+                return _error_result("Could not parse score", on_error=self.config.on_error)
+
+            matched = score < self.config.threshold
+            msg = (
+                f"Relevance score {score:.2f} (threshold {self.config.threshold:.2f})"
+            )
+            return EvaluatorResult(
+                matched=matched,
+                confidence=score,
+                message=msg,
+                metadata={"score": score, "rationale": rationale},
+            )
+
+        except Exception as e:
+            return _error_result(str(e), on_error=self.config.on_error)
+
+
+def _extract_qa(data: Any) -> tuple[str, str]:
+    """Extract question/answer from input payload."""
+    if isinstance(data, dict):
+        question = data.get("input") or data.get("question") or ""
+        answer = data.get("output") or data.get("answer") or ""
+        # If input is nested dict, try common keys
+        if isinstance(question, dict):
+            question = (
+                question.get("question")
+                or question.get("query")
+                or question.get("prompt")
+                or str(question)
+            )
+        return str(question), str(answer)
+
+    # If raw string, treat as answer only
+    return "", str(data)
+
+
+def _parse_score(content: str) -> tuple[float | None, str]:
+    """Parse JSON response to extract score + rationale."""
+    try:
+        obj = json.loads(content)
+        score = float(obj.get("score"))
+        rationale = str(obj.get("rationale", ""))
+        return score, rationale
+    except Exception:
+        return None, ""
+
+
+def _error_result(error_msg: str, *, on_error: str) -> EvaluatorResult:
+    matched = on_error == "deny"
+    return EvaluatorResult(
+        matched=matched,
+        confidence=0.0,
+        message=f"LLM relevance evaluator error: {error_msg}",
+        error=error_msg,
+    )

examples/sko-workshop-demo/agent-control-sales-workshop-demo/pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "agentcontrol-sales-workshop-demo"
+version = "0.1.0"
+description = "Sales workshop demo for AgentControl"
+requires-python = ">=3.12"
+dependencies = [
+  "agent-control-sdk>=5.0.0,<7.0.0",
+  "agent-control-models>=5.0.0,<7.0.0",
+  "agent-control-evaluators>=5.0.0,<7.0.0",
+]
+
+[project.optional-dependencies]
+rag = [
+  "chromadb>=0.5.0",
+  "langgraph>=0.2.0",
+  "langchain-openai>=0.2.0",
+  "langchain-core>=0.2.0",
+  "streamlit>=1.33.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+include = [
+  "*.py",
+  "README.md",
+]

examples/sko-workshop-demo/agent-control-sales-workshop-demo/rag_qa_demo.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Interactive RAG Q&A demo using ChromaDB + OpenAI + AgentControl."""
+
+import asyncio
+import os
+import sys
+from typing import Any, Dict, List
+
+# SDK fallback path (monorepo checkout)
+SDK_FALLBACK = "/Users/namrataghadi/code/agentcontrol/agent-control/sdks/python/src"
+if SDK_FALLBACK not in sys.path:
+    sys.path.insert(0, SDK_FALLBACK)
+
+import agent_control
+from agent_control import ControlViolationError, control
+
+from chromadb import Client
+from chromadb.config import Settings
+from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction
+from langchain_openai import ChatOpenAI
+from langchain_core.tools import tool
+
+
+AGENT_NAME = "RAG Q&A Agent"
+AGENT_ID = "9e9a1c8e-8c3f-4c6d-9d2a-0d3d5e8a1b77"
+SERVER_URL = os.getenv("AGENT_CONTROL_URL", "http://localhost:8000")
+
+# --- Initialize AgentControl ---
+agent_control.init(
+    agent_name=AGENT_NAME,
+    agent_id=AGENT_ID,
+    agent_description="Sales assistant demo agent (RAG Q&A)",
+    server_url=SERVER_URL,
+)
+
+# --- LLM ---
+llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.2)
+
+# --- ChromaDB ---
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
+if not OPENAI_API_KEY:
+    raise RuntimeError("OPENAI_API_KEY is required for embeddings and LLM")
+
+embedding_fn = OpenAIEmbeddingFunction(
+    api_key=OPENAI_API_KEY,
+    model_name="text-embedding-3-small",
+)
+
+client = Client(Settings(anonymized_telemetry=False))
+collection = client.get_or_create_collection(
+    name="sales_knowledge",
+    embedding_function=embedding_fn,
+)
+
+DOCS = [
+    (
+        "pricing-1",
+        "Pricing: Standard plan is $50k/year with 10% max discount. Premium is $120k/year with 30% max discount.",
+    ),
+    (
+        "security-1",
+        "Security: SOC2 Type II, GDPR compliant, data encrypted at rest and in transit.",
+    ),
+    (
+        "roi-1",
+        "ROI: Customers typically see 20% faster sales cycles and 15% higher win rates.",
+    ),
+    (
+        "support-1",
+        "Support: 24/7 support for Premium tier, business-hours support for Standard tier.",
+    ),
+]
+
+# Index docs (idempotent)
+existing = set(collection.get(include=[]) .get("ids", []))
+for doc_id, text in DOCS:
+    if doc_id not in existing:
+        collection.add(ids=[doc_id], documents=[text])
+
+
+# --- Controlled retrieval tool ---
+@tool("retrieve_docs")
+async def _retrieve_docs(query: str) -> List[str]:
+    """Retrieve top docs from ChromaDB."""
+    results = collection.query(query_texts=[query], n_results=3)
+    docs = results.get("documents", [[]])[0]
+    return docs
+
+# Wrap tool with AgentControl
+_retrieve_docs.name = "retrieve_docs"  # type: ignore[attr-defined]
+_retrieve_docs.tool_name = "retrieve_docs"  # type: ignore[attr-defined]
+retrieve_docs = control()(_retrieve_docs)
+
+
+# --- Controlled answer generation ---
+@control()
+async def answer_question(question: str, context: str) -> str:
+    prompt = (
+        "You are a sales Q&A assistant. Answer the question using the context below. "
+        "If the context does not contain the answer, say you don't know.\n\n"
+        f"Context:\n{context}\n\nQuestion: {question}"
+    )
+    resp = await llm.ainvoke(prompt)
+    return resp.content
+
+
+async def run_qa(question: str) -> str:
+    docs = await retrieve_docs(question)
+    context = "\n".join(docs)
+    return await answer_question(question, context)
+
+
+async def main() -> None:
+    print("=" * 70)
+    print("RAG Q&A Demo (ChromaDB + OpenAI + AgentControl)")
+    print("=" * 70)
+    print("Try questions like:")
+    print("  - What is the pricing for Standard?\n  - Are you GDPR compliant?\n  - What ROI do customers see?\n")
+    print("Type 'exit' to quit.\n")
+
+    while True:
+        try:
+            q = input("You: ").strip()
+        except (KeyboardInterrupt, EOFError):
+            print("\nGoodbye!")
+            break
+
+        if not q:
+            continue
+        if q.lower() in ("exit", "quit"):
+            print("Goodbye!")
+            break
+
+        try:
+            ans = await run_qa(q)
+            print(f"Agent: {ans}")
+        except ControlViolationError as e:
+            print(f"Blocked by control: {e.control_name} ({e.message})")
+        except Exception as e:
+            print(f"Error: {type(e).__name__}: {e}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())