chore(py): add back evaluator sample

py/packages/genkit/src/genkit/_ai/_generate.py

@@ -39,10 +39,10 @@
 from genkit._core._action import Action, ActionKind, ActionRunContext
 from genkit._core._error import GenkitError
 from genkit._core._logger import get_logger
+from genkit._core._model import GenerateActionOptions
 from genkit._core._registry import Registry
 from genkit._core._typing import (
     FinishReason,
-    GenerateActionOptions,
     Part,
     Role,
     ToolDefinition,

py/packages/genkit/src/genkit/_ai/_prompt.py

@@ -48,11 +48,10 @@
 from genkit._core._channel import Channel
 from genkit._core._error import GenkitError
 from genkit._core._logger import get_logger
-from genkit._core._model import Document, ModelConfig
+from genkit._core._model import Document, GenerateActionOptions, ModelConfig
 from genkit._core._registry import Registry
 from genkit._core._schema import to_json_schema
 from genkit._core._typing import (
-    GenerateActionOptions,
     GenerateActionOutputConfig,
     OutputConfig,
     Part,
@@ -489,7 +488,7 @@ def _or(opt_val: Any, default: Any) -> Any:  # noqa: ANN401
 
         return GenerateActionOptions(
             model=model,
-            messages=resolved_msgs,  # type: ignore[arg-type]
+            messages=resolved_msgs,
             config=prompt_options.config,
             tools=prompt_options.tools,
             return_tool_requests=prompt_options.return_tool_requests,

py/packages/genkit/src/genkit/_core/_model.py

@@ -38,15 +38,20 @@
     DocumentData,
     DocumentPart,
     FinishReason,
+    GenerateActionOptionsData,
+    GenerateActionOutputConfig,
     GenerateResponseChunk,
     GenerationCommonConfig,
     GenerationUsage,
     Media,
     MediaModel,
     MediaPart,
     MessageData,
+    MiddlewareRef,
     Operation,
     Part,
+    Resume,
+    Role,
     Text,
     TextPart,
     ToolChoice,
@@ -82,11 +87,21 @@ def __init__(
     ) -> None:
         """Initialize from MessageData or keyword arguments."""
         if message is not None:
-            super().__init__(
-                role=message.role,
-                content=message.content,
-                metadata=message.metadata,
-            )
+            if isinstance(message, dict):
+                role = message.get('role')
+                if role is None:
+                    raise ValueError('Message role is required')
+                super().__init__(
+                    role=role,
+                    content=message.get('content', []),
+                    metadata=message.get('metadata'),
+                )
+            else:
+                super().__init__(
+                    role=message.role,
+                    content=message.content,
+                    metadata=message.metadata,
+                )
         else:
             super().__init__(**kwargs)  # type: ignore[arg-type]
 
@@ -116,6 +131,17 @@ def interrupts(self) -> list[ToolRequestPart]:
         return [p for p in self.tool_requests if p.metadata and p.metadata.get('interrupt')]
 
 
+class GenerateActionOptions(GenerateActionOptionsData):
+    """Generate options with messages as list[Message] for type-safe use with ai.generate()."""
+
+    messages: list[Message]
+
+    @field_validator('messages', mode='before')
+    @classmethod
+    def _wrap_messages(cls, v: list[MessageData]) -> list[Message]:
+        return [m if isinstance(m, Message) else Message(m) for m in v]
+
+
 _TEXT_DATA_TYPE: str = 'text'
 
 
@@ -510,6 +536,17 @@ def count_parts(parts: list[Part]) -> tuple[int, int, int, int]:
     )
 
 
+# Rebuild schema after all types (including Message) are fully defined.
+# _types_namespace provides forward-ref resolution for GenerateActionOptionsData fields.
+GenerateActionOptions.model_rebuild(
+    _types_namespace={
+        'GenerateActionOutputConfig': GenerateActionOutputConfig,
+        'MiddlewareRef': MiddlewareRef,
+        'Resume': Resume,
+        'Role': Role,
+    }
+)
+
 # Type aliases for model middleware (Any is intentional - middleware is type-agnostic)
 # Middleware can have two signatures:
 #   Simple (3 params): (req, ctx, next) -> response

py/packages/genkit/src/genkit/_core/_reflection.py

@@ -87,7 +87,11 @@ async def execute(self) -> None:
                 context=self.payload.get('context', {}),
                 on_trace_start=self.on_trace_start,
             )
-            result = output.response.model_dump() if isinstance(output.response, BaseModel) else output.response
+            result = (
+                output.response.model_dump(by_alias=True, exclude_none=True)
+                if isinstance(output.response, BaseModel)
+                else output.response
+            )
             self.queue.put_nowait(
                 json.dumps({
                     'result': result,

py/packages/genkit/src/genkit/_core/_typing.py

@@ -38,7 +38,7 @@ class EvalStatusEnum(StrEnum):
     """EvalStatusEnum data type class."""
 
     UNKNOWN = 'UNKNOWN'
-    PASS_ = 'PASS'
+    PASS = 'PASS'
     FAIL = 'FAIL'
 
 
@@ -226,13 +226,12 @@ class DataPart(GenkitModel):
     resource: Any | None = Field(default=None)
 
 
-class GenerateActionOptions(GenkitModel):
-    """Model for generateactionoptions data."""
+class GenerateActionOptionsData(GenkitModel):
+    """Model for generateactionoptionsdata data."""
 
     model_config: ClassVar[ConfigDict] = ConfigDict(alias_generator=to_camel, extra='forbid', populate_by_name=True)
     model: str | None = None
     docs: list[DocumentData] | None = None
-    messages: list[MessageData] = Field(...)
     tools: list[str] | None = None
     resources: list[str] | None = None
     tool_choice: ToolChoice | None = None

py/packages/genkit/src/genkit/model/__init__.py

@@ -23,6 +23,7 @@
 )
 from genkit._core._background import BackgroundAction
 from genkit._core._model import (
+    GenerateActionOptions,
     Message,
     ModelRef,
     ModelRequest,
@@ -36,7 +37,6 @@
     Constrained,
     Error,
     FinishReason,
-    GenerateActionOptions,
     ModelInfo,
     Operation,
     Stage,

py/packages/genkit/tests/genkit/ai/generate_test.py

@@ -23,11 +23,10 @@
     define_programmable_model,
 )
 from genkit._core._action import ActionRunContext
-from genkit._core._model import ModelRequest
+from genkit._core._model import GenerateActionOptions, ModelRequest
 from genkit._core._typing import (
     DocumentPart,
     FinishReason,
-    GenerateActionOptions,
     Part,
     Role,
     TextPart,

py/packages/genkit/tests/genkit/ai/prompt_test.py

@@ -34,9 +34,8 @@
     define_programmable_model,
 )
 from genkit._core._action import ActionKind
-from genkit._core._model import ModelConfig, ModelRequest
+from genkit._core._model import GenerateActionOptions, ModelConfig, ModelRequest
 from genkit._core._typing import (
-    GenerateActionOptions,
     Part,
     Role,
     TextPart,

py/packages/genkit/tests/genkit/ai/resource_integration_test.py

@@ -25,10 +25,9 @@
 from genkit._ai._generate import generate_action
 from genkit._ai._resource import ResourceInput, ResourceOutput, define_resource, resource
 from genkit._core._action import ActionRunContext
-from genkit._core._model import ModelRequest
+from genkit._core._model import GenerateActionOptions, ModelRequest
 from genkit._core._registry import ActionKind, Registry
 from genkit._core._typing import (
-    GenerateActionOptions,
     Part,
     Resource1,
     ResourcePart,

py/plugins/evaluators/README.md

@@ -0,0 +1,40 @@
+# Genkit Evaluators Plugin
+
+Provides three rule-based evaluators matching the Go and JS implementations:
+
+- **regex** – Tests output against a regex pattern (reference = regex string)
+- **deep_equal** – Tests equality of output against reference
+- **jsonata** – Evaluates a JSONata expression (reference) against output; pass if result is truthy
+
+No LLM or API keys required.
+
+## Installation
+
+```bash
+pip install genkit-plugin-evaluators
+```
+
+## Usage
+
+```python
+from genkit import Genkit
+from genkit.plugins.evaluators import GenkitEval
+
+ai = Genkit(plugins=[GenkitEval()])
+
+# Run evaluation with genkit eval-flow or programmatically
+evaluator = await ai.registry.resolve_evaluator('genkitEval/regex')
+result = await evaluator.run(input={
+    'dataset': [
+        {'input': 'sample', 'output': 'banana', 'reference': 'ba?a?a'},
+        {'input': 'sample', 'output': 'apple', 'reference': 'ba?a?a'},
+    ],
+    'evalRunId': 'test',
+})
+```
+
+## Evaluators
+
+- **genkitEval/regex** – Reference is a regex string. Output (stringified if needed) must match.
+- **genkitEval/deep_equal** – Reference is the expected value. Output must equal reference.
+- **genkitEval/jsonata** – Reference is a JSONata expression. Evaluated against output; pass if truthy.

py/plugins/evaluators/pyproject.toml

@@ -0,0 +1,53 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+[project]
+authors = [{ name = "Google" }]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: Apache Software License",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+  "genkit",
+  "jsonata-python>=0.6.0",
+]
+description = "Genkit Evaluators Plugin (regex, deep_equal, jsonata)"
+keywords = ["genkit", "ai", "evaluator", "eval", "ragas"]
+license = "Apache-2.0"
+name = "genkit-plugin-evaluators"
+readme = "README.md"
+requires-python = ">=3.10"
+version = "0.5.1"
+
+[project.urls]
+"Homepage" = "https://github.com/firebase/genkit"
+"Repository" = "https://github.com/firebase/genkit/tree/main/py"
+
+[build-system]
+build-backend = "hatchling.build"
+requires      = ["hatchling"]
+
+[tool.hatch.build.targets.wheel]
+only-include = ["src/genkit/plugins/evaluators"]
+sources      = ["src"]

py/plugins/evaluators/src/genkit/plugins/evaluators/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Genkit built-in evaluators: regex, deep_equal, jsonata."""
+
+from genkit.plugins.evaluators.plugin import genkit_eval_name, register_genkit_evaluators
+
+__all__ = ['genkit_eval_name', 'register_genkit_evaluators']