Separate reward and pass semantics in reporting

Author: sjarmak

docs/REPORT_CONTEXT.md

@@ -148,7 +148,10 @@ The evaluation uses a multi-layer pipeline:
    and task rewards.
 
 4. **Report generator**: Aggregates all layers into structured JSON and
-   Markdown reports.
+   Markdown reports. Reporting should preserve continuous `reward`,
+   authoritative `passed` / `pass_threshold`, scorer family, and output
+   contract separately rather than collapsing unlike verifier families into
+   one implicitly calibrated scalar.
 
 ### 3.3 Scoring Types
 
@@ -165,6 +168,12 @@ Different task categories use different verifier types:
 | **navigation-verified** | 0.0--1.0 | Regression proving (fail-on-buggy + pass-after-patch) |
 | **external** | 0.0--1.0 | TheAgentCompany tasks |
 
+Canonical reporting now treats these families as separate semantic buckets.
+Mean reward remains useful within a family or benchmark, but aggregate views
+should either partition by `scorer_family` or clearly caveat mixed-family
+comparisons. Solved/pass status should come from verifier `passed`, not from
+recomputing `reward > 0`.
+
 ### 3.4 CodeScaleBench-Org Oracle Evaluation
 
 Org tasks use a closed-world oracle system with 7 deterministic

scripts/csb_metrics/__init__.py

@@ -5,8 +5,11 @@
 from .extractors import extract_run_config
 from .task_selection import (
     load_selected_tasks,
+    load_canonical_evaluation_audit,
     build_task_index,
+    build_task_contract_index,
     enrich_runs,
+    enrich_run_contracts,
     filter_runs_to_selected,
 )
 
@@ -18,7 +21,10 @@
     "collect_retrieval_data",
     "extract_run_config",
     "load_selected_tasks",
+    "load_canonical_evaluation_audit",
     "build_task_index",
+    "build_task_contract_index",
     "enrich_runs",
+    "enrich_run_contracts",
     "filter_runs_to_selected",
 ]

scripts/csb_metrics/discovery.py

@@ -230,7 +230,11 @@ def _process_task_dir(
         reward = extract_reward_from_file(reward_path)
         if reward is not None:
             tm.reward = reward
-            tm.status = "passed" if reward > 0 else "failed"
+            if tm.passed is None:
+                tm.pass_threshold = 0.0 if tm.pass_threshold is None else tm.pass_threshold
+                tm.passed = reward > 0
+            if tm.status != "error":
+                tm.status = "passed" if tm.passed else "failed"
 
     # SWE-bench partial score
     if is_swebench:

scripts/csb_metrics/extractors.py

@@ -21,6 +21,98 @@
 _WARNED_UNKNOWN_PRICING_MODELS: set[str] = set()
 
 
+def _coerce_float(value: object) -> Optional[float]:
+    try:
+        return float(value)  # type: ignore[arg-type]
+    except (TypeError, ValueError):
+        return None
+
+
+def _coerce_bool(value: object) -> Optional[bool]:
+    if isinstance(value, bool):
+        return value
+    return None
+
+
+def _infer_passed(reward: Optional[float], pass_threshold: Optional[float]) -> Optional[bool]:
+    if reward is None:
+        return None
+    threshold = 0.0 if pass_threshold is None else pass_threshold
+    if threshold <= 0.0:
+        return reward > 0.0
+    return reward >= threshold
+
+
+def _extract_validation_result_payload(result_json_path: Path) -> Optional[dict]:
+    """Load validation_result.json or compatible legacy sidecar payloads."""
+    validation_result_path = result_json_path.parent / "verifier" / "validation_result.json"
+    if not validation_result_path.is_file():
+        return None
+    try:
+        data = json.loads(validation_result_path.read_text())
+    except (OSError, json.JSONDecodeError):
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def _normalize_validation_result(payload: dict) -> dict[str, object]:
+    """Normalize canonical and legacy validation_result payloads."""
+    output_contract = payload.get("output_contract")
+    if not isinstance(output_contract, dict):
+        output_contract = {}
+
+    reward = _coerce_float(payload.get("reward"))
+    if reward is None:
+        reward = _coerce_float(payload.get("overall_score"))
+    if reward is None:
+        reward = _coerce_float(payload.get("score"))
+
+    pass_threshold = _coerce_float(payload.get("pass_threshold"))
+    passed = _coerce_bool(payload.get("passed"))
+    if passed is None:
+        passed = _infer_passed(reward, pass_threshold)
+
+    status = payload.get("status")
+    if not isinstance(status, str):
+        status = "scored" if reward is not None else None
+
+    scorable = _coerce_bool(payload.get("scorable"))
+    if scorable is None:
+        scorable = status == "scored" if status is not None else reward is not None
+
+    scorer_family = payload.get("scorer_family")
+    if not isinstance(scorer_family, str):
+        scorer_family = None
+
+    output_contract_mode = output_contract.get("mode")
+    if not isinstance(output_contract_mode, str):
+        output_contract_mode = output_contract.get("classification")
+    if not isinstance(output_contract_mode, str):
+        output_contract_mode = None
+
+    output_contract_primary_path = output_contract.get("primary_path")
+    if not isinstance(output_contract_primary_path, str):
+        output_contract_primary_path = output_contract.get("primary_output_path")
+    if not isinstance(output_contract_primary_path, str):
+        output_contract_primary_path = None
+
+    output_contract_required_artifact = _coerce_bool(
+        output_contract.get("required_artifact")
+    )
+
+    return {
+        "reward": reward,
+        "passed": passed,
+        "pass_threshold": pass_threshold,
+        "validation_status": status,
+        "validation_scorable": scorable,
+        "scorer_family": scorer_family,
+        "output_contract_mode": output_contract_mode,
+        "output_contract_primary_path": output_contract_primary_path,
+        "output_contract_required_artifact": output_contract_required_artifact,
+    }
+
+
 def _parse_iso(ts: Optional[str]) -> Optional[datetime]:
     """Parse an ISO 8601 timestamp, returning None on failure."""
     if not ts:
@@ -87,6 +179,12 @@ def extract_task_from_result_json(
 
     # Reward
     reward = None
+    validation_payload = _extract_validation_result_payload(path)
+    normalized_validation = (
+        _normalize_validation_result(validation_payload)
+        if validation_payload is not None
+        else None
+    )
     verifier_result = data.get("verifier_result") or {}
     rewards = verifier_result.get("rewards") or {}
     for key in ("reward", "score"):
@@ -96,13 +194,35 @@ def extract_task_from_result_json(
             except (TypeError, ValueError):
                 continue
             break
+    if normalized_validation and normalized_validation.get("reward") is not None:
+        reward = normalized_validation["reward"]  # type: ignore[assignment]
+
+    passed = None
+    pass_threshold = None
+    scorer_family = None
+    output_contract_mode = None
+    output_contract_primary_path = None
+    output_contract_required_artifact = None
+    validation_status = None
+    validation_scorable = None
+    if normalized_validation:
+        passed = normalized_validation.get("passed")
+        pass_threshold = normalized_validation.get("pass_threshold")
+        scorer_family = normalized_validation.get("scorer_family")
+        output_contract_mode = normalized_validation.get("output_contract_mode")
+        output_contract_primary_path = normalized_validation.get("output_contract_primary_path")
+        output_contract_required_artifact = normalized_validation.get("output_contract_required_artifact")
+        validation_status = normalized_validation.get("validation_status")
+        validation_scorable = normalized_validation.get("validation_scorable")
 
     # Status — agent timeouts are scored normally (verifier runs on partial work)
     exc = data.get("exception_info") or {}
     exc_type = exc.get("exception_type", exc.get("type", "")) if isinstance(exc, dict) else ""
     timed_out = bool(exc and exc_type == "AgentTimeoutError")
     if exc and not timed_out:
         status = "error"
+    elif isinstance(passed, bool):
+        status = "passed" if passed else "failed"
     elif reward is not None:
         status = "passed" if reward > 0 else "failed"
     else:
@@ -162,7 +282,25 @@ def extract_task_from_result_json(
         benchmark=benchmark,
         config_name=config_name,
         reward=reward,
+        passed=passed if isinstance(passed, bool) else None,
+        pass_threshold=pass_threshold if isinstance(pass_threshold, float) else None,
         status=status,
+        scorer_family=scorer_family if isinstance(scorer_family, str) else None,
+        output_contract_mode=output_contract_mode if isinstance(output_contract_mode, str) else None,
+        output_contract_primary_path=(
+            output_contract_primary_path
+            if isinstance(output_contract_primary_path, str)
+            else None
+        ),
+        output_contract_required_artifact=(
+            output_contract_required_artifact
+            if isinstance(output_contract_required_artifact, bool)
+            else None
+        ),
+        validation_status=validation_status if isinstance(validation_status, str) else None,
+        validation_scorable=(
+            validation_scorable if isinstance(validation_scorable, bool) else None
+        ),
         timed_out=timed_out,
         wall_clock_seconds=wall_clock,
         agent_execution_seconds=agent_execution_seconds,

scripts/csb_metrics/models.py

@@ -24,7 +24,15 @@ class TaskMetrics:
     # Scoring
     reward: Optional[float] = None
     partial_score: Optional[float] = None
+    passed: Optional[bool] = None
+    pass_threshold: Optional[float] = None
     status: str = "unknown"  # passed / failed / error
+    scorer_family: Optional[str] = None
+    output_contract_mode: Optional[str] = None
+    output_contract_primary_path: Optional[str] = None
+    output_contract_required_artifact: Optional[bool] = None
+    validation_status: Optional[str] = None
+    validation_scorable: Optional[bool] = None
 
     # LLM Judge (optional — separate from verifier reward)
     judge_score: Optional[float] = None
@@ -155,10 +163,16 @@ def mean_partial_score(self) -> Optional[float]:
 
     @property
     def pass_rate(self) -> Optional[float]:
-        scored = [t for t in self.tasks if t.status in ("passed", "failed")]
+        scored = [
+            t for t in self.tasks
+            if t.passed is not None or t.status in ("passed", "failed")
+        ]
         if not scored:
             return None
-        return sum(1 for t in scored if t.status == "passed") / len(scored)
+        return sum(
+            1 for t in scored
+            if t.passed is True or (t.passed is None and t.status == "passed")
+        ) / len(scored)
 
     @property
     def mean_judge_score(self) -> Optional[float]:

scripts/csb_metrics/task_selection.py

@@ -47,6 +47,11 @@ def load_selected_tasks(path: str | Path) -> dict:
     return json.loads(Path(path).read_text())
 
 
+def load_canonical_evaluation_audit(path: str | Path) -> dict:
+    """Load configs/canonical_evaluation_audit.json."""
+    return json.loads(Path(path).read_text())
+
+
 def build_task_index(selection: dict) -> dict[str, dict]:
     """Build a task_id → task metadata lookup from the selection document.
 
@@ -77,6 +82,19 @@ def build_task_index(selection: dict) -> dict[str, dict]:
     return index
 
 
+def build_task_contract_index(audit: dict) -> dict[str, dict]:
+    """Build a task_id → canonical evaluation contract lookup."""
+    index: dict[str, dict] = {}
+    for task in audit.get("tasks", []):
+        tid = _normalize_task_id(task["task_id"])
+        index[tid] = task
+        if tid.startswith(("csb_", "ccb_")):
+            bare = tid[4:]
+            if bare not in index:
+                index[bare] = task
+    return index
+
+
 def enrich_task_metrics(
     tm: TaskMetrics,
     task_index: dict[str, dict],
@@ -104,6 +122,27 @@ def enrich_task_metrics(
     tm.task_files_count = meta.get("files_count")
 
 
+def enrich_task_contract_metrics(
+    tm: TaskMetrics,
+    contract_index: dict[str, dict],
+) -> None:
+    """Enrich TaskMetrics with canonical scorer-family and output-contract metadata."""
+    contract = contract_index.get(_normalize_task_id(tm.task_id))
+    if contract is None:
+        return
+
+    validation_plan = contract.get("validation_result_plan") or {}
+    evaluator = contract.get("evaluator") or {}
+    output_contract = contract.get("output_contract") or {}
+
+    if tm.scorer_family is None:
+        tm.scorer_family = validation_plan.get("scorer_family") or evaluator.get("family")
+    if tm.output_contract_mode is None:
+        tm.output_contract_mode = output_contract.get("classification")
+    if tm.output_contract_primary_path is None:
+        tm.output_contract_primary_path = output_contract.get("primary_output_path")
+
+
 def enrich_runs(
     runs: list[RunMetrics],
     task_index: dict[str, dict],
@@ -119,6 +158,16 @@ def enrich_runs(
             enrich_task_metrics(tm, task_index)
 
 
+def enrich_run_contracts(
+    runs: list[RunMetrics],
+    contract_index: dict[str, dict],
+) -> None:
+    """Enrich all TaskMetrics within a list of RunMetrics with contract metadata."""
+    for run in runs:
+        for tm in run.tasks:
+            enrich_task_contract_metrics(tm, contract_index)
+
+
 def filter_runs_to_selected(
     runs: list[RunMetrics],
     task_index: dict[str, dict],