fix: 3 verifier bugs — k8s-cri expected_changes mismatch, llamacpp RocketChat reset, PEP 508 regex

BUG 1: k8s-cri-containerd-reason-001 expected_changes.json was checking Pod
storage/validation keywords but instruction asks to trace CRI RuntimeService
gRPC to containerd. Aligned task_id, expected_files, expected_content, and
added expected_patterns to match CRI instruction.

BUG 2: llamacpp-file-modify-search-001 and llamacpp-context-window-search-001
test.sh verifiers called init.sh which resets RocketChat services, destroying
the agent's chat messages before eval.py could read them. Both configs scored
0.0. Fix: skip init.sh during verification (services already running from
agent session).

BUG 3: 8 ccb_build validators.py shared a regex that rejected PEP 508
environment markers (e.g., "boto3==1.26.113 ; python_version >= 3.9") and
extras (e.g., "urllib3[socks]>=1.25.4"). The sg_only verifier wrapper converts
Poetry pyproject.toml to requirements.txt with these markers. Fix: strip
markers after ";" and extras in brackets before regex validation.

12 affected invalid SG_full result directories archived to
runs/archive/verifier_bugs_20260220/.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sjarmak

benchmarks/ccb_build/cgen-deps-install-001/tests/validators.py

@@ -118,6 +118,11 @@ def _validate_pyproject_toml(self, path: Path) -> List[str]:
     @staticmethod
     def _is_valid_requirement(line: str) -> bool:
         """Check if line is a valid requirement."""
+        # Strip PEP 508 environment markers (everything after ";")
+        # e.g., 'boto3==1.26.113 ; python_version >= "3.9"' -> 'boto3==1.26.113'
+        line = line.split(";")[0].strip()
+        # Strip extras specifiers e.g., "package[extra1,extra2]>=1.0" -> "package>=1.0"
+        line = re.sub(r"\[.*?\]", "", line)
         # Pattern: package_name with optional version specifiers
         # e.g., "requests", "requests==2.28.0", "requests>=2.28.0,<3.0"
         pattern = r"^[a-zA-Z0-9\-_.]+(\s*[><=!~]+\s*[0-9\.\*]+)?(\s*,\s*[><=!~]+\s*[0-9\.\*]+)*$"

benchmarks/ccb_build/codecoverage-deps-install-001/tests/validators.py

@@ -118,6 +118,11 @@ def _validate_pyproject_toml(self, path: Path) -> List[str]:
     @staticmethod
     def _is_valid_requirement(line: str) -> bool:
         """Check if line is a valid requirement."""
+        # Strip PEP 508 environment markers (everything after ";")
+        # e.g., 'boto3==1.26.113 ; python_version >= "3.9"' -> 'boto3==1.26.113'
+        line = line.split(";")[0].strip()
+        # Strip extras specifiers e.g., "package[extra1,extra2]>=1.0" -> "package>=1.0"
+        line = re.sub(r"\[.*?\]", "", line)
         # Pattern: package_name with optional version specifiers
         # e.g., "requests", "requests==2.28.0", "requests>=2.28.0,<3.0"
         pattern = r"^[a-zA-Z0-9\-_.]+(\s*[><=!~]+\s*[0-9\.\*]+)?(\s*,\s*[><=!~]+\s*[0-9\.\*]+)*$"

benchmarks/ccb_build/dotenv-expand-deps-install-001/tests/validators.py

@@ -118,6 +118,11 @@ def _validate_pyproject_toml(self, path: Path) -> List[str]:
     @staticmethod
     def _is_valid_requirement(line: str) -> bool:
         """Check if line is a valid requirement."""
+        # Strip PEP 508 environment markers (everything after ";")
+        # e.g., 'boto3==1.26.113 ; python_version >= "3.9"' -> 'boto3==1.26.113'
+        line = line.split(";")[0].strip()
+        # Strip extras specifiers e.g., "package[extra1,extra2]>=1.0" -> "package>=1.0"
+        line = re.sub(r"\[.*?\]", "", line)
         # Pattern: package_name with optional version specifiers
         # e.g., "requests", "requests==2.28.0", "requests>=2.28.0,<3.0"
         pattern = r"^[a-zA-Z0-9\-_.]+(\s*[><=!~]+\s*[0-9\.\*]+)?(\s*,\s*[><=!~]+\s*[0-9\.\*]+)*$"

benchmarks/ccb_build/dotnetkoans-deps-install-001/tests/validators.py

@@ -118,6 +118,11 @@ def _validate_pyproject_toml(self, path: Path) -> List[str]:
     @staticmethod
     def _is_valid_requirement(line: str) -> bool:
         """Check if line is a valid requirement."""
+        # Strip PEP 508 environment markers (everything after ";")
+        # e.g., 'boto3==1.26.113 ; python_version >= "3.9"' -> 'boto3==1.26.113'
+        line = line.split(";")[0].strip()
+        # Strip extras specifiers e.g., "package[extra1,extra2]>=1.0" -> "package>=1.0"
+        line = re.sub(r"\[.*?\]", "", line)
         # Pattern: package_name with optional version specifiers
         # e.g., "requests", "requests==2.28.0", "requests>=2.28.0,<3.0"
         pattern = r"^[a-zA-Z0-9\-_.]+(\s*[><=!~]+\s*[0-9\.\*]+)?(\s*,\s*[><=!~]+\s*[0-9\.\*]+)*$"

benchmarks/ccb_build/eslint-markdown-deps-install-001/tests/validators.py

@@ -118,6 +118,11 @@ def _validate_pyproject_toml(self, path: Path) -> List[str]:
     @staticmethod
     def _is_valid_requirement(line: str) -> bool:
         """Check if line is a valid requirement."""
+        # Strip PEP 508 environment markers (everything after ";")
+        # e.g., 'boto3==1.26.113 ; python_version >= "3.9"' -> 'boto3==1.26.113'
+        line = line.split(";")[0].strip()
+        # Strip extras specifiers e.g., "package[extra1,extra2]>=1.0" -> "package>=1.0"
+        line = re.sub(r"\[.*?\]", "", line)
         # Pattern: package_name with optional version specifiers
         # e.g., "requests", "requests==2.28.0", "requests>=2.28.0,<3.0"
         pattern = r"^[a-zA-Z0-9\-_.]+(\s*[><=!~]+\s*[0-9\.\*]+)?(\s*,\s*[><=!~]+\s*[0-9\.\*]+)*$"

benchmarks/ccb_build/iamactionhunter-deps-install-001/tests/validators.py

@@ -118,6 +118,11 @@ def _validate_pyproject_toml(self, path: Path) -> List[str]:
     @staticmethod
     def _is_valid_requirement(line: str) -> bool:
         """Check if line is a valid requirement."""
+        # Strip PEP 508 environment markers (everything after ";")
+        # e.g., 'boto3==1.26.113 ; python_version >= "3.9"' -> 'boto3==1.26.113'
+        line = line.split(";")[0].strip()
+        # Strip extras specifiers e.g., "package[extra1,extra2]>=1.0" -> "package>=1.0"
+        line = re.sub(r"\[.*?\]", "", line)
         # Pattern: package_name with optional version specifiers
         # e.g., "requests", "requests==2.28.0", "requests>=2.28.0,<3.0"
         pattern = r"^[a-zA-Z0-9\-_.]+(\s*[><=!~]+\s*[0-9\.\*]+)?(\s*,\s*[><=!~]+\s*[0-9\.\*]+)*$"

benchmarks/ccb_build/pcap-parser-deps-install-001/tests/validators.py

@@ -118,6 +118,11 @@ def _validate_pyproject_toml(self, path: Path) -> List[str]:
     @staticmethod
     def _is_valid_requirement(line: str) -> bool:
         """Check if line is a valid requirement."""
+        # Strip PEP 508 environment markers (everything after ";")
+        # e.g., 'boto3==1.26.113 ; python_version >= "3.9"' -> 'boto3==1.26.113'
+        line = line.split(";")[0].strip()
+        # Strip extras specifiers e.g., "package[extra1,extra2]>=1.0" -> "package>=1.0"
+        line = re.sub(r"\[.*?\]", "", line)
         # Pattern: package_name with optional version specifiers
         # e.g., "requests", "requests==2.28.0", "requests>=2.28.0,<3.0"
         pattern = r"^[a-zA-Z0-9\-_.]+(\s*[><=!~]+\s*[0-9\.\*]+)?(\s*,\s*[><=!~]+\s*[0-9\.\*]+)*$"

benchmarks/ccb_build/similar-asserts-deps-install-001/tests/validators.py

@@ -118,6 +118,11 @@ def _validate_pyproject_toml(self, path: Path) -> List[str]:
     @staticmethod
     def _is_valid_requirement(line: str) -> bool:
         """Check if line is a valid requirement."""
+        # Strip PEP 508 environment markers (everything after ";")
+        # e.g., 'boto3==1.26.113 ; python_version >= "3.9"' -> 'boto3==1.26.113'
+        line = line.split(";")[0].strip()
+        # Strip extras specifiers e.g., "package[extra1,extra2]>=1.0" -> "package>=1.0"
+        line = re.sub(r"\[.*?\]", "", line)
         # Pattern: package_name with optional version specifiers
         # e.g., "requests", "requests==2.28.0", "requests>=2.28.0,<3.0"
         pattern = r"^[a-zA-Z0-9\-_.]+(\s*[><=!~]+\s*[0-9\.\*]+)?(\s*,\s*[><=!~]+\s*[0-9\.\*]+)*$"

benchmarks/ccb_test/llamacpp-context-window-search-001/tests/test.sh

@@ -16,10 +16,13 @@ fi
 echo "Running TAC evaluator for sde-find-answer-in-codebase-1..."
 cd /utils
 
-# Initialize TAC environment (needed for RocketChat)
-if [ -f "/utils/init.sh" ]; then
-    SERVER_HOSTNAME="${TAC_SERVER_HOSTNAME:-localhost}" bash /utils/init.sh || true
-fi
+# NOTE: init.sh intentionally SKIPPED during verification.
+# init.sh resets RocketChat services, which destroys the agent's chat messages
+# before eval.py can read them. The services are already running from the agent's
+# session, so no re-initialization is needed for the verifier.
+# if [ -f "/utils/init.sh" ]; then
+#     SERVER_HOSTNAME="${TAC_SERVER_HOSTNAME:-localhost}" bash /utils/init.sh || true
+# fi
 
 DECRYPTION_KEY="${DECRYPTION_KEY:-theagentcompany is all you need}" \
 python_default /utils/eval.py \

benchmarks/ccb_test/llamacpp-file-modify-search-001/tests/test.sh

@@ -16,9 +16,13 @@ fi
 echo "Running TAC evaluator for sde-find-answer-in-codebase-2..."
 cd /utils
 
-if [ -f "/utils/init.sh" ]; then
-    SERVER_HOSTNAME="${TAC_SERVER_HOSTNAME:-localhost}" bash /utils/init.sh || true
-fi
+# NOTE: init.sh intentionally SKIPPED during verification.
+# init.sh resets RocketChat services, which destroys the agent's chat messages
+# before eval.py can read them. The services are already running from the agent's
+# session, so no re-initialization is needed for the verifier.
+# if [ -f "/utils/init.sh" ]; then
+#     SERVER_HOSTNAME="${TAC_SERVER_HOSTNAME:-localhost}" bash /utils/init.sh || true
+# fi
 
 DECRYPTION_KEY="${DECRYPTION_KEY:-theagentcompany is all you need}" \
 python_default /utils/eval.py \