From d73bd5c28e30145219d0d90775d1a7e64b1916bf Mon Sep 17 00:00:00 2001 From: Sounho Chung Date: Tue, 27 Aug 2024 18:49:23 -0700 Subject: [PATCH 01/10] fix: set generation name as optional param --- README.md | 37 +++++++++++++++++++++++++++++++++++-- setup.py | 6 ++---- weavel/_body.py | 21 ++++++++++++++++++++- weavel/_worker.py | 12 +++++++++++- weavel/client.py | 18 ++++++++---------- weavel/wrapper.py | 25 ++++++++++++++----------- 6 files changed, 90 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 8004d02..d189979 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ You can find our full documentation [here](https://weavel.ai/docs/python-sdk). ## How to use -### Basic Usage +### Option 1: Using OpenAI wrapper ```python from weavel import WeavelOpenAI as OpenAI @@ -42,7 +42,40 @@ response = openai.chat.completions.create( ``` -### Advanced Usage +### Option 2: Logging inputs/outputs of LLM calls + +```python +from weavel import Weavel +from openai import OpenAI +from pydantic import BaseModel + +openai = OpenAI() +# initialize Weavel +weavel = Weavel() + +class Answer(BaseModel): + reasoning: str + answer: str + +question = "What is x if x + 2 = 4?" +response = openai.beta.chat.completions.parse( + model="gpt-4o-2024-08-06", + messages=[ + {"role": "system", "content": "You are a math teacher."}, + {"role": "user", "content": question} + ], + response_format=Answer +).choices[0].message.parsed + +# log the generation +weavel.generation( + name="solve-math", # optional + inputs={"question": question}, + outputs=response.model_dump() +) +``` + +### Option 3 (Advanced Usage): OTEL-compatible trace logging ```python from weavel import Weavel diff --git a/setup.py b/setup.py index 9f248a7..3977829 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="weavel", - version="1.7.1", + version="1.8.0", packages=find_namespace_packages(), entry_points={}, description="Weavel, Prompt Optimization and Evaluation for LLM Applications", @@ -32,8 +32,6 @@ "termcolor", "watchdog", "readerwriterlock", - "pendulum", - "httpx[http2]", "nest_asyncio", ], python_requires=">=3.8.10", @@ -47,6 +45,6 @@ "dataset curation", "prompt engineering", "prompt optimization", - "AI Prompt Engineer" + "AI Prompt Engineer", ], ) diff --git a/weavel/_body.py b/weavel/_body.py index 8fdcb0e..93a7c2f 100644 --- a/weavel/_body.py +++ b/weavel/_body.py @@ -132,7 +132,10 @@ class CaptureRecordBody(BaseRecordBody): class CaptureObservationBody(BaseObservationBody): - name: str + name: Optional[str] = Field( + default=None, + description="The name of the observation. Optional.", + ) class CaptureMessageBody(CaptureRecordBody): @@ -262,6 +265,22 @@ class CaptureGenerationBody(CaptureObservationBody): default=None, description="The outputs of the generation. Optional.", ) + messages: Optional[List[Dict[str, str]]] = Field( + default=None, + description="The messages of the generation. Optional.", + ) + model: Optional[str] = Field( + default=None, + description="The model of the generation. Optional.", + ) + latency: Optional[float] = Field( + default=None, + description="The latency of the generation. Optional.", + ) + cost: Optional[float] = Field( + default=None, + description="The cost of the generation. Optional.", + ) prompt_name: Optional[str] = Field( default=None, description="The name of the prompt. Optional.", diff --git a/weavel/_worker.py b/weavel/_worker.py index 5411796..c8bad81 100644 --- a/weavel/_worker.py +++ b/weavel/_worker.py @@ -341,11 +341,15 @@ def capture_generation( self, observation_id: str, created_at: datetime, - name: str, + name: Optional[str] = None, record_id: Optional[str] = None, inputs: Optional[Union[Dict[str, Any], List[Any], str]] = None, outputs: Optional[Union[Dict[str, Any], List[Any], str]] = None, + messages: Optional[List[Dict[str, str]]] = None, metadata: Optional[Dict[str, str]] = None, + model: Optional[str] = None, + latency: Optional[float] = None, + cost: Optional[float] = None, parent_observation_id: Optional[str] = None, prompt_name: Optional[str] = None, ): @@ -358,6 +362,10 @@ def capture_generation( name=name, inputs=inputs, outputs=outputs, + messages=messages, + model=model, + latency=latency, + cost=cost, metadata=metadata, parent_observation_id=parent_observation_id, prompt_name=prompt_name, @@ -374,6 +382,7 @@ async def acapture_generation( record_id: Optional[str] = None, inputs: Optional[Union[Dict[str, Any], List[Any], str]] = None, outputs: Optional[Union[Dict[str, Any], List[Any], str]] = None, + messages: Optional[List[Dict[str, str]]] = None, metadata: Optional[Dict[str, str]] = None, parent_observation_id: Optional[str] = None, prompt_name: Optional[str] = None, @@ -387,6 +396,7 @@ async def acapture_generation( name=name, inputs=inputs, outputs=outputs, + messages=messages, metadata=metadata, parent_observation_id=parent_observation_id, prompt_name=prompt_name, diff --git a/weavel/client.py b/weavel/client.py index 7213911..8a965da 100644 --- a/weavel/client.py +++ b/weavel/client.py @@ -190,6 +190,7 @@ def generation( name: Optional[str] = None, inputs: Optional[Union[Dict[str, Any], List[Any], str]] = None, outputs: Optional[Union[Dict[str, Any], List[Any], str]] = None, + messages: Optional[List[Dict[str, str]]] = None, metadata: Optional[Dict[str, Any]] = None, parent_observation_id: Optional[str] = None, prompt_name: Optional[str] = None, @@ -212,16 +213,11 @@ def generation( name=name, inputs=inputs, outputs=outputs, + messages=messages, metadata=metadata, parent_observation_id=parent_observation_id, prompt_name=prompt_name, ) - if ( - record_id or (record_id is None and parent_observation_id) - ) and name is None: - raise ValueError( - "If you want to create a new generation, you must provide a name." - ) if record_id is not None or ( record_id is None and parent_observation_id is not None @@ -240,6 +236,7 @@ def generation( name=name, inputs=inputs, outputs=outputs, + messages=messages, metadata=metadata, parent_observation_id=parent_observation_id, prompt_name=prompt_name, @@ -251,6 +248,7 @@ def generation( name=name, inputs=inputs, outputs=outputs, + messages=messages, metadata=metadata, parent_observation_id=parent_observation_id, prompt_name=prompt_name, @@ -442,8 +440,8 @@ def create_dataset_items( return for item in items: - if not isinstance(item, DatasetItem): - item = DatasetItem(**item) + if isinstance(item, DatasetItem): + item = item.model_dump() self._worker.create_dataset_items(dataset_name, items) @@ -462,8 +460,8 @@ async def acreate_dataset_items( return for item in items: - if not isinstance(item, DatasetItem): - item = DatasetItem(**item) + if isinstance(item, DatasetItem): + item = item.model_dump() await self._worker.acreate_dataset_items(dataset_name, items) diff --git a/weavel/wrapper.py b/weavel/wrapper.py index f923e00..70fb2e5 100644 --- a/weavel/wrapper.py +++ b/weavel/wrapper.py @@ -41,6 +41,7 @@ load_dotenv() + def calculate_cost(model: str, usage: Dict[str, int]) -> float: if model not in pricing: # raise ValueError(f"Unknown model: {model}") @@ -240,6 +241,7 @@ def _handle_non_streaming(self, *args, **kwargs): cost = calculate_cost(response.model, formatted_response["usage"]) formatted_response["cost"] = cost + # TODO: models, latency, cost shouldn't be logged as inputs. Should have separate fields for each. self._capture_generation(inputs=kwargs, outputs=formatted_response) return response @@ -275,11 +277,11 @@ def parse(self, *args, **kwargs): # context manager to suppress duplicate logging with LoggingSuppressor(self._worker): response = self.original.parse(*args, **kwargs) - + end_time = time.time() latency = end_time - start_time latency = round(latency, 6) - + formatted_response = { "id": response.id, "choices": [choice.model_dump() for choice in response.choices], @@ -291,11 +293,12 @@ def parse(self, *args, **kwargs): "system_fingerprint": response.system_fingerprint, "latency": latency, } - + # Calculate and add cost cost = calculate_cost(response.model, formatted_response["usage"]) formatted_response["cost"] = cost - + + # TODO: models, latency, cost shouldn't be logged as inputs. Should have separate fields for each. self._worker.capture_generation( observation_id=str(uuid4()), created_at=datetime.now(timezone.utc), @@ -304,7 +307,7 @@ def parse(self, *args, **kwargs): inputs=kwargs, outputs=formatted_response, ) - + return response self.chat.completions = CustomChatCompletions( @@ -442,7 +445,7 @@ async def _handle_streaming(self, *args, **kwargs): usage = accumulated_response["usage"] cost = calculate_cost(model, usage) accumulated_response["cost"] = cost - + if self._worker.capture_generation is not None: await self._capture_generation( inputs=kwargs, outputs=accumulated_response @@ -478,7 +481,7 @@ async def _handle_non_streaming(self, *args, **kwargs): inputs=kwargs, outputs=formatted_response ) return response - + async def _capture_generation( self, inputs, @@ -507,14 +510,14 @@ def __init__(self, original, weavel_api_key: str, base_url: Optional[str]): async def parse(self, *args, **kwargs): header = kwargs.pop("headers", {}) start_time = time.time() - + async with AsyncLoggingSuppressor(self._worker): response = await self.original.parse(*args, **kwargs) - + end_time = time.time() latency = end_time - start_time latency = round(latency, 6) - + formatted_response = { "id": response.id, "choices": [choice.model_dump() for choice in response.choices], @@ -526,7 +529,7 @@ async def parse(self, *args, **kwargs): "system_fingerprint": response.system_fingerprint, "latency": latency, } - + # Calculate and add cost cost = calculate_cost(response.model, formatted_response["usage"]) formatted_response["cost"] = cost From 400a46881b56894535b1ef641a85fb90ce7ed4bc Mon Sep 17 00:00:00 2001 From: shamuiscoding Date: Mon, 9 Sep 2024 09:50:18 +0900 Subject: [PATCH 02/10] feat: prompt and prompt version fetch, delete, upload and listup --- setup.py | 2 +- weavel/__init__.py | 2 +- weavel/_worker.py | 250 +++++++++++++++++++++++++++++++++- weavel/client.py | 281 ++++++++++++++++++++++++++++++++++++++- weavel/types/datasets.py | 20 ++- 5 files changed, 549 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 3977829..c7100e4 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="weavel", - version="1.8.0", + version="1.8.1", packages=find_namespace_packages(), entry_points={}, description="Weavel, Prompt Optimization and Evaluation for LLM Applications", diff --git a/weavel/__init__.py b/weavel/__init__.py index 0eed0f9..fca40e3 100644 --- a/weavel/__init__.py +++ b/weavel/__init__.py @@ -4,4 +4,4 @@ from .utils import * -__version___ = "1.7.1" +__version___ = "1.8.1" diff --git a/weavel/_worker.py b/weavel/_worker.py index c8bad81..e8e0f25 100644 --- a/weavel/_worker.py +++ b/weavel/_worker.py @@ -40,8 +40,8 @@ from weavel._buffer_storage import BufferStorage from weavel._api_client import APIClient, AsyncAPIClient from weavel.utils import logger -from weavel.types.datasets import DatasetItem, Dataset - +from weavel.types.datasets import DatasetItem, Dataset, Prompt, PromptVersion +from ape.types import ResponseFormat class Worker: _instance = None @@ -577,6 +577,252 @@ def create_test( ) if response.status_code != 200: raise Exception(f"Failed to create test: {response.text}") + + # create, fetch, delete, list prompts + def create_prompt( + self, + name: str, + description: Optional[str] = None, + ) -> None: + response = self.api_client.execute( + self.api_key, + self.endpoint, + "/prompts", + method="POST", + json={ + "name": name, + "description": description, + }, + ) + if response.status_code == 400: + raise Exception(f"Prompt {name} already exists") + if response.status_code != 200: + raise Exception(f"Failed to create prompt: {response.text}") + + async def acreate_prompt( + self, + name: str, + description: Optional[str] = None, + ) -> None: + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + "/prompts", + method="POST", + json={ + "name": name, + "description": description, + }, + ) + if response.status_code != 200: + raise Exception(f"Failed to create prompt: {response.text}") + + def fetch_prompt( + self, + name: str, + ) -> Prompt: + response = self.api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{name}", + method="GET", + ) + + if response.status_code == 200: + return Prompt(**response.json()) + else: + raise Exception(f"Failed to get prompt: {response.text}") + + async def afetch_prompt( + self, + name: str, + ) -> Prompt: + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{name}", + method="GET", + ) + + if response.status_code == 200: + return Prompt(**response.json()) + else: + raise Exception(f"Failed to get prompt: {response.text}") + + def delete_prompt(self, name: str) -> None: + response = self.api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{name}", + method="DELETE", + ) + if response.status_code != 200: + raise Exception(f"Failed to delete prompt: {response.text}") + + async def adelete_prompt(self, name: str) -> None: + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{name}", + method="DELETE", + ) + if response.status_code != 200: + raise Exception(f"Failed to delete prompt: {response.text}") + + def list_prompts(self) -> List[Prompt]: + response = self.api_client.execute( + self.api_key, + self.endpoint, + "/prompts", + method="GET", + ) + + if response.status_code == 200: + return [Prompt(**prompt) for prompt in response.json()] + else: + raise Exception(f"Failed to list prompts: {response.text}") + + async def alist_prompts(self) -> List[Prompt]: + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + "/prompts", + method="GET", + ) + + if response.status_code == 200: + return [Prompt(**prompt) for prompt in response.json()] + else: + raise Exception(f"Failed to list prompts: {response.text}") + + # create, fetch, delete, list prompt versions + def create_prompt_version( + self, + prompt_name: str, + messages: List[Dict[str, Any]], + model: str = 'gpt-4o-mini', + temperature: float = 0.0, + response_format: Optional[ResponseFormat] = None, + input_vars: Optional[Dict[str, Any]] = None, + output_vars: Optional[Dict[str, Any]] = None, + metadata: Optional[Dict[str, Any]] = None + ) -> None: + response = self.api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{prompt_name}/versions", + method="POST", + json={ + "messages": messages, + "model": model, + "temperature": temperature, + "response_format": response_format, + "input_vars": input_vars, + "output_vars": output_vars, + "metadata": metadata + } + ) + if response.status_code != 200: + raise Exception(f"Failed to create prompt version: {response.text}") + + async def acreate_prompt_version( + self, + prompt_name: str, + messages: List[Dict[str, Any]], + model: str = 'gpt-4o-mini', + temperature: float = 0.0, + response_format: Optional[ResponseFormat] = None, + input_vars: Optional[Dict[str, Any]] = None, + output_vars: Optional[Dict[str, Any]] = None, + metadata: Optional[Dict[str, Any]] = None + ) -> None: + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{prompt_name}/versions", + method="POST", + json={ + "messages": messages, + "model": model, + "temperature": temperature, + "response_format": response_format, + "input_vars": input_vars, + "output_vars": output_vars, + "metadata": metadata + } + ) + if response.status_code != 200: + raise Exception(f"Failed to create prompt version: {response.text}") + + def fetch_prompt_version(self, prompt_name: str, version: int) -> PromptVersion: + response = self.api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{prompt_name}/versions/{version}", + method="GET", + ) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to fetch prompt version: {response.text}") + + async def afetch_prompt_version(self, prompt_name: str, version: int) -> PromptVersion: + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{prompt_name}/versions/{version}", + method="GET", + ) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to fetch prompt version: {response.text}") + + def delete_prompt_version(self, prompt_name: str, version: int) -> None: + response = self.api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{prompt_name}/versions/{version}", + method="DELETE", + ) + if response.status_code != 200: + raise Exception(f"Failed to delete prompt version: {response.text}") + + async def adelete_prompt_version(self, prompt_name: str, version: int) -> None: + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{prompt_name}/versions/{version}", + method="DELETE", + ) + if response.status_code != 200: + raise Exception(f"Failed to delete prompt version: {response.text}") + + def list_prompt_versions(self, prompt_name: int) -> List[PromptVersion]: + response = self.api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{prompt_name}/versions", + method="GET", + ) + + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to list prompt versions: {response.text}") + + async def alist_prompt_versions(self, prompt_name: str) -> List[PromptVersion]: + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + f"/prompts/{prompt_name}/versions", + method="GET", + ) + + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to list prompt versions: {response.text}") def send_requests( self, diff --git a/weavel/client.py b/weavel/client.py index 8a965da..5fc92ea 100644 --- a/weavel/client.py +++ b/weavel/client.py @@ -8,6 +8,7 @@ import time from typing import Callable, Dict, List, Optional, Any, Union from uuid import uuid4 +from ape.types import ResponseFormat from dotenv import load_dotenv from weavel._worker import Worker @@ -19,7 +20,7 @@ SpanClient, TraceClient, ) -from weavel.types.datasets import Dataset, DatasetItem +from weavel.types.datasets import Dataset, DatasetItem, Prompt, PromptVersion load_dotenv() @@ -425,6 +426,284 @@ async def aget_dataset(self, name: str) -> Dataset: return await self._worker.afetch_dataset(name) + # create, fetch, delete, and list prompts + def create_prompt( + self, + name: str, + description: Optional[str] = None, + ) -> None: + """Upload a prompt to the Weavel service. + + Args: + name (str): The name of the prompt. + description (str): The description of the prompt. + """ + if self.testing: + return + + self._worker.create_prompt( + name=name, + description=description, + ) + + async def acreate_prompt( + self, + name: str, + description: Optional[str] = None, + ) -> None: + """Upload a prompt to the Weavel service. + + Args: + name (str): The name of the prompt. + description (str): The description of the prompt. + """ + if self.testing: + return + + await self._worker.acreate_prompt( + name=name, + description=description, + ) + + def fetch_prompt(self, name: str) -> Prompt: + """ + Retrieves a prompt with the given name. + + Args: + name (str): The name of the prompt to retrieve. + + Returns: + Prompt: The retrieved prompt. + """ + if self.testing: + return {} + + return self._worker.fetch_prompt(name) + + async def afetch_prompt(self, name: str) -> Prompt: + """ + Retrieves a prompt with the given name. + + Args: + name (str): The name of the prompt to retrieve. + + Returns: + Prompt: The retrieved prompt. + """ + if self.testing: + return {} + + return await self._worker.afetch_prompt(name) + + def delete_prompt(self, name: str) -> None: + """Delete a prompt from the Weavel service. + + Args: + name (str): The name of the prompt to delete. + """ + if self.testing: + return + + self._worker.delete_prompt(name=name) + + async def adelete_prompt(self, name: str) -> None: + """Delete a prompt from the Weavel service asynchronously. + + Args: + name (str): The name of the prompt to delete. + """ + if self.testing: + return + + await self._worker.adelete_prompt(name=name) + + def list_prompts(self) -> List[Prompt]: + """List all prompts that user created. + + Returns: + List[Prompt]: A list of all prompts. + """ + if self.testing: + return [] + + return self._worker.list_prompts() + + async def alist_prompts(self) -> List[Prompt]: + """List all prompts that user created. + + Returns: + List[Prompt]: A list of all prompts. + """ + if self.testing: + return [] + + return await self._worker.alist_prompts() + + # create, fetch, delete, and list prompt versions + def create_prompt_version( + self, + prompt_name: str, + messages: List[Dict[str, Any]], + model: Optional[str] = None, + temperature: Optional[float] = None, + response_format: Optional[ResponseFormat] = None, + input_vars: Optional[Dict[str, Any]] = None, + output_vars: Optional[Dict[str, Any]] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + """Create a new version of a prompt. + + Args: + prompt_name (str): The name of the prompt. + messages (List[Dict[str, Any]]): The messages for the prompt version. + model (Optional[str]): The model to use for this prompt version. Default is 'gpt-4o-mini'. + temperature (Optional[float]): The temperature setting for the model. Default is 0.0. + response_format (Optional[ResponseFormat]): The response format for the prompt. + input_vars (Optional[Dict[str, Any]]): The input variables for the prompt. + output_vars (Optional[Dict[str, Any]]): The output variables for the prompt. + metadata (Optional[Dict[str, Any]]): Additional metadata for the prompt version. + """ + if self.testing: + return + + self._worker.create_prompt_version( + prompt_name=prompt_name, + messages=messages, + model=model, + temperature=temperature, + response_format=response_format, + input_vars=input_vars, + output_vars=output_vars, + metadata=metadata, + ) + + async def acreate_prompt_version( + self, + prompt_name: str, + messages: List[Dict[str, Any]], + model: Optional[str] = None, + temperature: Optional[float] = None, + response_format: Optional[Dict[str, Any]] = None, + input_vars: Optional[Dict[str, Any]] = None, + output_vars: Optional[Dict[str, Any]] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + """Create a new version of a prompt asynchronously. + + Args: + prompt_name (str): The name of the prompt. + messages (List[Dict[str, Any]]): The messages for the prompt version. + model (Optional[str]): The model to use for this prompt version. Default is 'gpt-4o-mini'. + temperature (Optional[float]): The temperature setting for the model. Default is 0.0. + response_format (Optional[Dict[str, Any]]): The response format for the prompt. + input_vars (Optional[Dict[str, Any]]): The input variables for the prompt. + output_vars (Optional[Dict[str, Any]]): The output variables for the prompt. + metadata (Optional[Dict[str, Any]]): Additional metadata for the prompt version. + """ + if self.testing: + return + + await self._worker.acreate_prompt_version( + prompt_name=prompt_name, + messages=messages, + model=model, + temperature=temperature, + response_format=response_format, + input_vars=input_vars, + output_vars=output_vars, + metadata=metadata, + ) + + def fetch_prompt_version(self, prompt_name: str, version: str) -> PromptVersion: + """Fetch a specific version of a prompt. + + Args: + prompt_name (str): The name of the prompt. + version (str): The version identifier to fetch. + + Returns: + PromptVersion: The prompt version details. + """ + if self.testing: + return PromptVersion() + + return self._worker.fetch_prompt_version( + prompt_name=prompt_name, version=version + ) + + async def afetch_prompt_version( + self, prompt_name: str, version: str + ) -> PromptVersion: + """Fetch a specific version of a prompt asynchronously. + + Args: + prompt_name (str): The name of the prompt. + version (str): The version identifier to fetch. + + Returns: + PromptVersion: The prompt version details. + """ + if self.testing: + return PromptVersion() + + return await self._worker.afetch_prompt_version( + prompt_name=prompt_name, version=version + ) + + def list_prompt_versions(self, prompt_name: str) -> List[PromptVersion]: + """List all versions of a specific prompt. + + Args: + prompt_name (str): The name of the prompt. + + Returns: + List[PromptVersion]: A list of version identifiers. + """ + if self.testing: + return [] + + return self._worker.list_prompt_versions(prompt_name=prompt_name) + + async def alist_prompt_versions(self, prompt_name: str) -> List[PromptVersion]: + """List all versions of a specific prompt asynchronously. + + Args: + prompt_name (str): The name of the prompt. + + Returns: + List[PromptVersion]: A list of version identifiers. + """ + if self.testing: + return [] + + return await self._worker.alist_prompt_versions(prompt_name=prompt_name) + + def delete_prompt_version(self, prompt_name: str, version: str) -> None: + """Delete a specific version of a prompt. + + Args: + prompt_name (str): The name of the prompt. + version (str): The version identifier to delete. + """ + if self.testing: + return + + self._worker.delete_prompt_version(prompt_name=prompt_name, version=version) + + async def adelete_prompt_version(self, prompt_name: str, version: str) -> None: + """Delete a specific version of a prompt asynchronously. + + Args: + prompt_name (str): The name of the prompt. + version (str): The version identifier to delete. + """ + if self.testing: + return + + await self._worker.adelete_prompt_version( + prompt_name=prompt_name, version=version + ) + def create_dataset_items( self, dataset_name: str, diff --git a/weavel/types/datasets.py b/weavel/types/datasets.py index ed48da6..a3444b3 100644 --- a/weavel/types/datasets.py +++ b/weavel/types/datasets.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel - +from ape.types import ResponseFormat class DatasetItem(BaseModel): uuid: Optional[str] = None @@ -14,3 +14,21 @@ class Dataset(BaseModel): created_at: str description: Optional[str] = None items: List[DatasetItem] + + +class Prompt(BaseModel): + name: str + description: Optional[str] = None + created_at: str + + +class PromptVersion(BaseModel): + version: int + messages: List[Dict[str, Any]] + model: str + temperature: float + response_format: Optional[ResponseFormat] = None + input_vars: Optional[Dict[str, Any]] = None + output_vars: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None + created_at: str \ No newline at end of file From 2bd194c5121cc31774c394d16168a4beefac24f9 Mon Sep 17 00:00:00 2001 From: shamuiscoding Date: Wed, 11 Sep 2024 06:50:35 +0900 Subject: [PATCH 03/10] fix: enable latest --- weavel/_worker.py | 4 ++-- weavel/client.py | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/weavel/_worker.py b/weavel/_worker.py index e8e0f25..dcd2bbc 100644 --- a/weavel/_worker.py +++ b/weavel/_worker.py @@ -754,7 +754,7 @@ async def acreate_prompt_version( if response.status_code != 200: raise Exception(f"Failed to create prompt version: {response.text}") - def fetch_prompt_version(self, prompt_name: str, version: int) -> PromptVersion: + def fetch_prompt_version(self, prompt_name: str, version: Union[str, int]) -> PromptVersion: response = self.api_client.execute( self.api_key, self.endpoint, @@ -766,7 +766,7 @@ def fetch_prompt_version(self, prompt_name: str, version: int) -> PromptVersion: else: raise Exception(f"Failed to fetch prompt version: {response.text}") - async def afetch_prompt_version(self, prompt_name: str, version: int) -> PromptVersion: + async def afetch_prompt_version(self, prompt_name: str, version: Union[str, int]) -> PromptVersion: response = await self.async_api_client.execute( self.api_key, self.endpoint, diff --git a/weavel/client.py b/weavel/client.py index 5fc92ea..3c9edb6 100644 --- a/weavel/client.py +++ b/weavel/client.py @@ -614,12 +614,14 @@ async def acreate_prompt_version( metadata=metadata, ) - def fetch_prompt_version(self, prompt_name: str, version: str) -> PromptVersion: + def fetch_prompt_version( + self, prompt_name: str, version: Union[str, int] + ) -> PromptVersion: """Fetch a specific version of a prompt. Args: prompt_name (str): The name of the prompt. - version (str): The version identifier to fetch. + version (Union[str, int]): The version identifier to fetch. Get latest version by version = 'latest'. Otherwise, specify the version number. Returns: PromptVersion: The prompt version details. @@ -632,9 +634,10 @@ def fetch_prompt_version(self, prompt_name: str, version: str) -> PromptVersion: ) async def afetch_prompt_version( - self, prompt_name: str, version: str + self, prompt_name: str, version: Union[str, int] ) -> PromptVersion: """Fetch a specific version of a prompt asynchronously. + Get latest version by version = 'latest'. Otherwise, specify the version number. Args: prompt_name (str): The name of the prompt. From 92624efac8d48c2a4f5678e607bc04e89e157cbd Mon Sep 17 00:00:00 2001 From: Sounho Chung Date: Wed, 11 Sep 2024 15:10:59 -0700 Subject: [PATCH 04/10] chore: refactor types files and imports --- weavel/_worker.py | 37 +++++++++++++++++++++---------------- weavel/types/__init__.py | 26 ++++++++++++++++++++++---- weavel/types/datasets.py | 20 +------------------- weavel/types/prompts.py | 21 +++++++++++++++++++++ 4 files changed, 65 insertions(+), 39 deletions(-) create mode 100644 weavel/types/prompts.py diff --git a/weavel/_worker.py b/weavel/_worker.py index dcd2bbc..1905cc6 100644 --- a/weavel/_worker.py +++ b/weavel/_worker.py @@ -40,9 +40,10 @@ from weavel._buffer_storage import BufferStorage from weavel._api_client import APIClient, AsyncAPIClient from weavel.utils import logger -from weavel.types.datasets import DatasetItem, Dataset, Prompt, PromptVersion +from weavel.types import DatasetItem, Dataset, Prompt, PromptVersion from ape.types import ResponseFormat + class Worker: _instance = None @@ -577,7 +578,7 @@ def create_test( ) if response.status_code != 200: raise Exception(f"Failed to create test: {response.text}") - + # create, fetch, delete, list prompts def create_prompt( self, @@ -668,7 +669,7 @@ async def adelete_prompt(self, name: str) -> None: ) if response.status_code != 200: raise Exception(f"Failed to delete prompt: {response.text}") - + def list_prompts(self) -> List[Prompt]: response = self.api_client.execute( self.api_key, @@ -694,18 +695,18 @@ async def alist_prompts(self) -> List[Prompt]: return [Prompt(**prompt) for prompt in response.json()] else: raise Exception(f"Failed to list prompts: {response.text}") - + # create, fetch, delete, list prompt versions def create_prompt_version( self, prompt_name: str, messages: List[Dict[str, Any]], - model: str = 'gpt-4o-mini', + model: str = "gpt-4o-mini", temperature: float = 0.0, response_format: Optional[ResponseFormat] = None, input_vars: Optional[Dict[str, Any]] = None, output_vars: Optional[Dict[str, Any]] = None, - metadata: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None, ) -> None: response = self.api_client.execute( self.api_key, @@ -719,8 +720,8 @@ def create_prompt_version( "response_format": response_format, "input_vars": input_vars, "output_vars": output_vars, - "metadata": metadata - } + "metadata": metadata, + }, ) if response.status_code != 200: raise Exception(f"Failed to create prompt version: {response.text}") @@ -729,12 +730,12 @@ async def acreate_prompt_version( self, prompt_name: str, messages: List[Dict[str, Any]], - model: str = 'gpt-4o-mini', + model: str = "gpt-4o-mini", temperature: float = 0.0, response_format: Optional[ResponseFormat] = None, input_vars: Optional[Dict[str, Any]] = None, output_vars: Optional[Dict[str, Any]] = None, - metadata: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None, ) -> None: response = await self.async_api_client.execute( self.api_key, @@ -748,13 +749,15 @@ async def acreate_prompt_version( "response_format": response_format, "input_vars": input_vars, "output_vars": output_vars, - "metadata": metadata - } + "metadata": metadata, + }, ) if response.status_code != 200: raise Exception(f"Failed to create prompt version: {response.text}") - - def fetch_prompt_version(self, prompt_name: str, version: Union[str, int]) -> PromptVersion: + + def fetch_prompt_version( + self, prompt_name: str, version: Union[str, int] + ) -> PromptVersion: response = self.api_client.execute( self.api_key, self.endpoint, @@ -766,7 +769,9 @@ def fetch_prompt_version(self, prompt_name: str, version: Union[str, int]) -> Pr else: raise Exception(f"Failed to fetch prompt version: {response.text}") - async def afetch_prompt_version(self, prompt_name: str, version: Union[str, int]) -> PromptVersion: + async def afetch_prompt_version( + self, prompt_name: str, version: Union[str, int] + ) -> PromptVersion: response = await self.async_api_client.execute( self.api_key, self.endpoint, @@ -797,7 +802,7 @@ async def adelete_prompt_version(self, prompt_name: str, version: int) -> None: ) if response.status_code != 200: raise Exception(f"Failed to delete prompt version: {response.text}") - + def list_prompt_versions(self, prompt_name: int) -> List[PromptVersion]: response = self.api_client.execute( self.api_key, diff --git a/weavel/types/__init__.py b/weavel/types/__init__.py index da2d553..823b357 100644 --- a/weavel/types/__init__.py +++ b/weavel/types/__init__.py @@ -1,4 +1,22 @@ -from .datasets import * -from .observations import * -from .records import * -from .session import * +from .datasets import Dataset, DatasetItem +from .prompts import Prompt, PromptVersion +from .observations import Observation, Span, Generation, Log +from .records import Record, Message, TrackEvent, Trace +from .session import Session + + +__all__ = [ + "Dataset", + "DatasetItem", + "Prompt", + "PromptVersion", + "Observation", + "Span", + "Generation", + "Log", + "Record", + "Message", + "TrackEvent", + "Trace", + "Session", +] diff --git a/weavel/types/datasets.py b/weavel/types/datasets.py index a3444b3..ed48da6 100644 --- a/weavel/types/datasets.py +++ b/weavel/types/datasets.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel -from ape.types import ResponseFormat + class DatasetItem(BaseModel): uuid: Optional[str] = None @@ -14,21 +14,3 @@ class Dataset(BaseModel): created_at: str description: Optional[str] = None items: List[DatasetItem] - - -class Prompt(BaseModel): - name: str - description: Optional[str] = None - created_at: str - - -class PromptVersion(BaseModel): - version: int - messages: List[Dict[str, Any]] - model: str - temperature: float - response_format: Optional[ResponseFormat] = None - input_vars: Optional[Dict[str, Any]] = None - output_vars: Optional[Dict[str, Any]] = None - metadata: Optional[Dict[str, Any]] = None - created_at: str \ No newline at end of file diff --git a/weavel/types/prompts.py b/weavel/types/prompts.py new file mode 100644 index 0000000..c579a11 --- /dev/null +++ b/weavel/types/prompts.py @@ -0,0 +1,21 @@ +from typing import Any, Dict, List, Optional, Union +from pydantic import BaseModel +from ape.types import ResponseFormat + + +class Prompt(BaseModel): + name: str + description: Optional[str] = None + created_at: str + + +class PromptVersion(BaseModel): + version: int + messages: List[Dict[str, Any]] + model: str + temperature: float + response_format: Optional[ResponseFormat] = None + input_vars: Optional[Dict[str, Any]] = None + output_vars: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None + created_at: str From e82fc889d9267e23a5215ba14c2c941d406a7ea6 Mon Sep 17 00:00:00 2001 From: Sounho Chung Date: Wed, 11 Sep 2024 16:38:12 -0700 Subject: [PATCH 05/10] chore: bump version 1.8.1 -> 1.9.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c7100e4..317679d 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="weavel", - version="1.8.1", + version="1.9.0", packages=find_namespace_packages(), entry_points={}, description="Weavel, Prompt Optimization and Evaluation for LLM Applications", From 9d971899768c04be4857482b0a4fdcc445660d6c Mon Sep 17 00:00:00 2001 From: Sounho Chung Date: Wed, 11 Sep 2024 16:57:19 -0700 Subject: [PATCH 06/10] fix: fix import bug --- setup.py | 2 +- weavel/client.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 317679d..58aaf29 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="weavel", - version="1.9.0", + version="1.9.1", packages=find_namespace_packages(), entry_points={}, description="Weavel, Prompt Optimization and Evaluation for LLM Applications", diff --git a/weavel/client.py b/weavel/client.py index 3c9edb6..54c7d76 100644 --- a/weavel/client.py +++ b/weavel/client.py @@ -13,14 +13,13 @@ from dotenv import load_dotenv from weavel._worker import Worker -# from weavel.types.instances import Session, Span, Trace from weavel.object_clients import ( GenerationClient, SessionClient, SpanClient, TraceClient, ) -from weavel.types.datasets import Dataset, DatasetItem, Prompt, PromptVersion +from weavel.types import Dataset, DatasetItem, Prompt, PromptVersion load_dotenv() From 46255c1696b188f38f7f01a8db8f42b220b26d09 Mon Sep 17 00:00:00 2001 From: shamuiscoding Date: Thu, 12 Sep 2024 09:31:25 +0900 Subject: [PATCH 07/10] fix: remove ape dependency --- setup.py | 2 +- weavel/__init__.py | 2 +- weavel/_worker.py | 2 +- weavel/client.py | 2 +- weavel/types/datasets.py | 2 +- weavel/types/response_format.py | 13 +++++++++++++ 6 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 weavel/types/response_format.py diff --git a/setup.py b/setup.py index c7100e4..e011ee5 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="weavel", - version="1.8.1", + version="1.8.2", packages=find_namespace_packages(), entry_points={}, description="Weavel, Prompt Optimization and Evaluation for LLM Applications", diff --git a/weavel/__init__.py b/weavel/__init__.py index fca40e3..c703a0c 100644 --- a/weavel/__init__.py +++ b/weavel/__init__.py @@ -4,4 +4,4 @@ from .utils import * -__version___ = "1.8.1" +__version___ = "1.8.2" diff --git a/weavel/_worker.py b/weavel/_worker.py index dcd2bbc..fcaa14c 100644 --- a/weavel/_worker.py +++ b/weavel/_worker.py @@ -41,7 +41,7 @@ from weavel._api_client import APIClient, AsyncAPIClient from weavel.utils import logger from weavel.types.datasets import DatasetItem, Dataset, Prompt, PromptVersion -from ape.types import ResponseFormat +from weavel.types.response_format import ResponseFormat class Worker: _instance = None diff --git a/weavel/client.py b/weavel/client.py index 3c9edb6..9a9c8e1 100644 --- a/weavel/client.py +++ b/weavel/client.py @@ -8,7 +8,7 @@ import time from typing import Callable, Dict, List, Optional, Any, Union from uuid import uuid4 -from ape.types import ResponseFormat +from weavel.types.response_format import ResponseFormat from dotenv import load_dotenv from weavel._worker import Worker diff --git a/weavel/types/datasets.py b/weavel/types/datasets.py index a3444b3..82726e9 100644 --- a/weavel/types/datasets.py +++ b/weavel/types/datasets.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel -from ape.types import ResponseFormat +from response_format import ResponseFormat class DatasetItem(BaseModel): uuid: Optional[str] = None diff --git a/weavel/types/response_format.py b/weavel/types/response_format.py new file mode 100644 index 0000000..9871365 --- /dev/null +++ b/weavel/types/response_format.py @@ -0,0 +1,13 @@ +from typing import Any, Dict, Literal, Optional +from pydantic import BaseModel + + +class JsonSchema(BaseModel): + name: str + schema: Dict[str, Any] + strict: bool = True + + +class ResponseFormat(BaseModel): + type: Literal["json_object", "json_schema", "xml"] + json_schema: Optional[JsonSchema] = None From eccf9d699a2c5d140192202a5ed1270c8106f30a Mon Sep 17 00:00:00 2001 From: shamuiscoding Date: Thu, 12 Sep 2024 09:34:22 +0900 Subject: [PATCH 08/10] fix: minor fixes --- weavel/_worker.py | 3 +-- weavel/client.py | 2 +- weavel/types/__init__.py | 1 + weavel/types/datasets.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/weavel/_worker.py b/weavel/_worker.py index fcaa14c..fb3b2e0 100644 --- a/weavel/_worker.py +++ b/weavel/_worker.py @@ -40,8 +40,7 @@ from weavel._buffer_storage import BufferStorage from weavel._api_client import APIClient, AsyncAPIClient from weavel.utils import logger -from weavel.types.datasets import DatasetItem, Dataset, Prompt, PromptVersion -from weavel.types.response_format import ResponseFormat +from weavel.types import DatasetItem, Dataset, Prompt, PromptVersion, ResponseFormat class Worker: _instance = None diff --git a/weavel/client.py b/weavel/client.py index 9a9c8e1..f241da1 100644 --- a/weavel/client.py +++ b/weavel/client.py @@ -8,7 +8,7 @@ import time from typing import Callable, Dict, List, Optional, Any, Union from uuid import uuid4 -from weavel.types.response_format import ResponseFormat +from weavel.types import ResponseFormat from dotenv import load_dotenv from weavel._worker import Worker diff --git a/weavel/types/__init__.py b/weavel/types/__init__.py index da2d553..40bd4f9 100644 --- a/weavel/types/__init__.py +++ b/weavel/types/__init__.py @@ -2,3 +2,4 @@ from .observations import * from .records import * from .session import * +from .response_format import * \ No newline at end of file diff --git a/weavel/types/datasets.py b/weavel/types/datasets.py index 82726e9..a6bdd18 100644 --- a/weavel/types/datasets.py +++ b/weavel/types/datasets.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel -from response_format import ResponseFormat +from weavel.types import ResponseFormat class DatasetItem(BaseModel): uuid: Optional[str] = None From 0fc9ece3559dd8d626d6cb3eb2ac4fbcf35a400f Mon Sep 17 00:00:00 2001 From: engineerA314 Date: Tue, 17 Sep 2024 14:37:49 -0700 Subject: [PATCH 09/10] UPDATE: update version to 1.9.3, remove ape-core dependency --- setup.py | 2 +- weavel/__init__.py | 2 +- weavel/types/prompts.py | 12 ++++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 95c6c27..d0f9fb4 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="weavel", - version="1.9.2", + version="1.9.3", packages=find_namespace_packages(), entry_points={}, description="Weavel, Prompt Optimization and Evaluation for LLM Applications", diff --git a/weavel/__init__.py b/weavel/__init__.py index c703a0c..a684560 100644 --- a/weavel/__init__.py +++ b/weavel/__init__.py @@ -4,4 +4,4 @@ from .utils import * -__version___ = "1.8.2" +__version___ = "1.9.3" diff --git a/weavel/types/prompts.py b/weavel/types/prompts.py index c579a11..8f66aeb 100644 --- a/weavel/types/prompts.py +++ b/weavel/types/prompts.py @@ -1,7 +1,15 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union, Literal from pydantic import BaseModel -from ape.types import ResponseFormat +class JsonSchema(BaseModel): + name: str + schema: Dict[str, Any] + strict: bool = True + + +class ResponseFormat(BaseModel): + type: Literal["json_object", "json_schema", "xml"] + json_schema: Optional[JsonSchema] = None class Prompt(BaseModel): name: str From 71feac026e46226f9de7b74c1cb650e8a304a72e Mon Sep 17 00:00:00 2001 From: shamuiscoding Date: Wed, 18 Sep 2024 20:03:40 +0900 Subject: [PATCH 10/10] update python sdk to have optimize and ape schedule --- setup.py | 2 +- weavel/__init__.py | 2 +- weavel/_worker.py | 236 +++++++++++++++++++++++++++++++++++++++++++++ weavel/client.py | 194 ++++++++++++++++++++++++++++++++++++- 4 files changed, 431 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index d0f9fb4..c8a5e2f 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="weavel", - version="1.9.3", + version="1.9.4", packages=find_namespace_packages(), entry_points={}, description="Weavel, Prompt Optimization and Evaluation for LLM Applications", diff --git a/weavel/__init__.py b/weavel/__init__.py index a684560..b162c18 100644 --- a/weavel/__init__.py +++ b/weavel/__init__.py @@ -4,4 +4,4 @@ from .utils import * -__version___ = "1.9.3" +__version___ = "1.9.4" diff --git a/weavel/_worker.py b/weavel/_worker.py index aaccc1a..de06f29 100644 --- a/weavel/_worker.py +++ b/weavel/_worker.py @@ -827,7 +827,243 @@ async def alist_prompt_versions(self, prompt_name: str) -> List[PromptVersion]: return response.json() else: raise Exception(f"Failed to list prompt versions: {response.text}") + + def optimize_ape( + self, + dataset_name: str, + prompt_name: str, + version: Optional[str] = None, + input_vars: Optional[Dict[str, Any]] = None, + output_vars: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, + evaluation_type: Literal["TEMPLATE", "CUSTOM"] = "TEMPLATE", + evaluation_metric: Optional[Dict[str, Any]] = None, + description: Optional[str] = None, + ) -> None: + """ + Optimize Ape. + + Args: + dataset_name (str): The name of the dataset to use for optimization. + prompt_name (str): The name of the prompt to optimize. + version (Optional[str]): The version of the prompt to use. If not provided, the latest version will be used. + input_vars (Optional[Dict[str, Any]]): The input variables for the prompt. + output_vars (Optional[Dict[str, Any]]): The output variables for the prompt. + response_format (Optional[ResponseFormat]): The response format for the prompt. + evaluation_type (str): The type of evaluation to use. Defaults to "TEMPLATE". + evaluation_metric (Optional[Dict[str, Any]]): The evaluation metric to use. + description (Optional[str]): A description of the optimization task. + """ + body = { + "dataset_name": dataset_name, + "prompt_name": prompt_name, + "version": version, + "input_vars": input_vars, + "output_vars": output_vars, + "response_format": response_format.model_dump() if response_format else None, + "evaluation_type": evaluation_type, + "evaluation_metric": evaluation_metric, + "description": description, + } + response = self.api_client.execute( + self.api_key, + self.endpoint, + "/ape/optimize", + method="POST", + json=body, + ) + if response.status_code != 200: + raise Exception(f"Failed to optimize Ape: {response.text}") + + async def aoptimize_ape( + self, + dataset_name: str, + prompt_name: str, + version: Optional[str] = None, + input_vars: Optional[Dict[str, Any]] = None, + output_vars: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, + evaluation_type: Literal["TEMPLATE", "CUSTOM"] = "TEMPLATE", + evaluation_metric: Optional[Dict[str, Any]] = None, + description: Optional[str] = None, + ) -> None: + """ + Optimize Ape asynchronously. + + Args: + dataset_name (str): The name of the dataset to use for optimization. + prompt_name (str): The name of the prompt to optimize. + version (Optional[str]): The version of the prompt to use. If not provided, the latest version will be used. + input_vars (Optional[Dict[str, Any]]): The input variables for the prompt. + output_vars (Optional[Dict[str, Any]]): The output variables for the prompt. + response_format (Optional[ResponseFormat]): The response format for the prompt. + evaluation_type (str): The type of evaluation to use. Defaults to "TEMPLATE". + evaluation_metric (Optional[Dict[str, Any]]): The evaluation metric to use. + description (Optional[str]): A description of the optimization task. + """ + body = { + "dataset_name": dataset_name, + "prompt_name": prompt_name, + "version": version, + "input_vars": input_vars, + "output_vars": output_vars, + "response_format": response_format.model_dump() if response_format else None, + "evaluation_type": evaluation_type, + "evaluation_metric": evaluation_metric, + "description": description, + } + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + "/ape/optimize", + method="POST", + json=body, + ) + if response.status_code != 200: + raise Exception(f"Failed to optimize Ape: {response.text}") + + def schedule_ape( + self, + prompt_name: str, + dataset_name: str, + interval: int, + trigger_threshold: float, + ignore_keys: Optional[List[str]] = None, + ) -> Dict[str, Any]: + """ + Schedule an Ape task. + + Args: + prompt_name (str): The name of the prompt to schedule. + dataset_name (str): The name of the dataset to use. + interval (int): The interval in days between runs. + trigger_threshold (float): The threshold to trigger the Ape task. + ignore_keys (Optional[List[str]]): Keys to ignore in the dataset. + + Returns: + Dict[str, Any]: The created Ape schedule. + """ + body = { + "prompt_name": prompt_name, + "dataset_name": dataset_name, + "interval": interval, + "trigger_threshold": trigger_threshold, + "ignore_keys": ignore_keys, + } + response = self.api_client.execute( + self.api_key, + self.endpoint, + "/ape/schedule", + method="POST", + json=body, + ) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to schedule Ape task: {response.text}") + + async def aschedule_ape( + self, + prompt_name: str, + dataset_name: str, + interval: int, + trigger_threshold: float, + ignore_keys: Optional[List[str]] = None, + ) -> Dict[str, Any]: + """ + Schedule an Ape task asynchronously. + Args: + prompt_name (str): The name of the prompt to schedule. + dataset_name (str): The name of the dataset to use. + interval (int): The interval in days between runs. + trigger_threshold (float): The threshold to trigger the Ape task. + ignore_keys (Optional[List[str]]): Keys to ignore in the dataset. + + Returns: + Dict[str, Any]: The created Ape schedule. + """ + body = { + "prompt_name": prompt_name, + "dataset_name": dataset_name, + "interval": interval, + "trigger_threshold": trigger_threshold, + "ignore_keys": ignore_keys, + } + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + "/ape/schedule", + method="POST", + json=body, + ) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to schedule Ape task: {response.text}") + + def list_ape_schedule( + self, + prompt_name: Optional[str] = None, + dataset_name: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """ + List Ape schedules. + + Args: + prompt_name (Optional[str]): Filter schedules by prompt name. + dataset_name (Optional[str]): Filter schedules by dataset name. + + Returns: + List[Dict[str, Any]]: The list of Ape schedules. + """ + body = { + "prompt_name": prompt_name, + "dataset_name": dataset_name, + } + response = self.api_client.execute( + self.api_key, + self.endpoint, + "/ape/list", + method="POST", + json=body, + ) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to list Ape schedules: {response.text}") + + async def alist_ape_schedule( + self, + prompt_name: Optional[str] = None, + dataset_name: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """ + List Ape schedules asynchronously. + + Args: + prompt_name (Optional[str]): Filter schedules by prompt name. + dataset_name (Optional[str]): Filter schedules by dataset name. + + Returns: + List[Dict[str, Any]]: The list of Ape schedules. + """ + body = { + "prompt_name": prompt_name, + "dataset_name": dataset_name, + } + response = await self.async_api_client.execute( + self.api_key, + self.endpoint, + "/ape/list", + method="POST", + json=body, + ) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to list Ape schedules: {response.text}") + def send_requests( self, requests: List[UnionRequest], diff --git a/weavel/client.py b/weavel/client.py index b899c98..e96265d 100644 --- a/weavel/client.py +++ b/weavel/client.py @@ -6,7 +6,7 @@ import os from datetime import datetime, timezone import time -from typing import Callable, Dict, List, Optional, Any, Union +from typing import Callable, Dict, List, Literal, Optional, Any, Union from uuid import uuid4 from weavel.types import ResponseFormat @@ -745,6 +745,198 @@ async def acreate_dataset_items( item = item.model_dump() await self._worker.acreate_dataset_items(dataset_name, items) + + def optimize_prompt( + self, + dataset_name: str, + prompt_name: str, + version: Optional[str] = None, + input_vars: Optional[Dict[str, Any]] = None, + output_vars: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, + evaluation_type: Literal["TEMPLATE", "CUSTOM"] = "TEMPLATE", + evaluation_metric: Optional[Dict[str, Any]] = None, + description: Optional[str] = None, + ) -> None: + """ + Optimize Ape. + + Args: + dataset_name (str): The name of the dataset to use for optimization. + prompt_name (str): The name of the prompt to optimize. + version (Optional[str]): The version of the prompt to use. If not provided, the latest version will be used. + input_vars (Optional[Dict[str, Any]]): The input variables for the prompt. + output_vars (Optional[Dict[str, Any]]): The output variables for the prompt. + response_format (Optional[ResponseFormat]): The response format for the prompt. + evaluation_type (str): The type of evaluation to use. Defaults to "TEMPLATE". + evaluation_metric (Optional[Dict[str, Any]]): The evaluation metric to use. + description (Optional[str]): A description of the optimization task. + """ + if self.testing: + return + + self._worker.optimize_ape( + dataset_name=dataset_name, + prompt_name=prompt_name, + version=version, + input_vars=input_vars, + output_vars=output_vars, + response_format=response_format, + evaluation_type=evaluation_type, + evaluation_metric=evaluation_metric, + description=description, + ) + + async def aoptimize_prompt( + self, + dataset_name: str, + prompt_name: str, + version: Optional[str] = None, + input_vars: Optional[Dict[str, Any]] = None, + output_vars: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, + evaluation_type: Literal["TEMPLATE", "CUSTOM"] = "TEMPLATE", + evaluation_metric: Optional[Dict[str, Any]] = None, + description: Optional[str] = None, + ) -> None: + """ + Optimize Ape asynchronously. + + Args: + dataset_name (str): The name of the dataset to use for optimization. + prompt_name (str): The name of the prompt to optimize. + version (Optional[str]): The version of the prompt to use. If not provided, the latest version will be used. + input_vars (Optional[Dict[str, Any]]): The input variables for the prompt. + output_vars (Optional[Dict[str, Any]]): The output variables for the prompt. + response_format (Optional[ResponseFormat]): The response format for the prompt. + evaluation_type (str): The type of evaluation to use. Defaults to "TEMPLATE". + evaluation_metric (Optional[Dict[str, Any]]): The evaluation metric to use. + description (Optional[str]): A description of the optimization task. + """ + if self.testing: + return + + await self._worker.aoptimize_ape( + dataset_name=dataset_name, + prompt_name=prompt_name, + version=version, + input_vars=input_vars, + output_vars=output_vars, + response_format=response_format, + evaluation_type=evaluation_type, + evaluation_metric=evaluation_metric, + description=description, + ) + + def schedule_ape( + self, + prompt_name: str, + dataset_name: str, + interval: int, + trigger_threshold: float, + ignore_keys: Optional[List[str]] = None, + ) -> Dict[str, Any]: + """ + Schedule an Ape task. + + Args: + prompt_name (str): The name of the prompt to schedule. + dataset_name (str): The name of the dataset to use. + interval (int): The interval in days between runs. + trigger_threshold (float): The threshold to trigger the Ape task. + ignore_keys (Optional[List[str]]): Keys to ignore in the dataset. + + Returns: + Dict[str, Any]: The created Ape schedule. + """ + if self.testing: + return {} + + return self._worker.schedule_ape( + prompt_name=prompt_name, + dataset_name=dataset_name, + interval=interval, + trigger_threshold=trigger_threshold, + ignore_keys=ignore_keys, + ) + + async def aschedule_ape( + self, + prompt_name: str, + dataset_name: str, + interval: int, + trigger_threshold: float, + ignore_keys: Optional[List[str]] = None, + ) -> Dict[str, Any]: + """ + Schedule an Ape task asynchronously. + + Args: + prompt_name (str): The name of the prompt to schedule. + dataset_name (str): The name of the dataset to use. + interval (int): The interval in days between runs. + trigger_threshold (float): The threshold to trigger the Ape task. + ignore_keys (Optional[List[str]]): Keys to ignore in the dataset. + + Returns: + Dict[str, Any]: The created Ape schedule. + """ + if self.testing: + return {} + + return await self._worker.aschedule_ape( + prompt_name=prompt_name, + dataset_name=dataset_name, + interval=interval, + trigger_threshold=trigger_threshold, + ignore_keys=ignore_keys, + ) + + def list_ape_schedule( + self, + prompt_name: Optional[str] = None, + dataset_name: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """ + List Ape schedules. + + Args: + prompt_name (Optional[str]): Filter schedules by prompt name. + dataset_name (Optional[str]): Filter schedules by dataset name. + + Returns: + List[Dict[str, Any]]: The list of Ape schedules. + """ + if self.testing: + return [] + + return self._worker.list_ape_schedule( + prompt_name=prompt_name, + dataset_name=dataset_name, + ) + + async def alist_ape_schedule( + self, + prompt_name: Optional[str] = None, + dataset_name: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """ + List Ape schedules asynchronously. + + Args: + prompt_name (Optional[str]): Filter schedules by prompt name. + dataset_name (Optional[str]): Filter schedules by dataset name. + + Returns: + List[Dict[str, Any]]: The list of Ape schedules. + """ + if self.testing: + return [] + + return await self._worker.alist_ape_schedule( + prompt_name=prompt_name, + dataset_name=dataset_name, + ) def test( self,